use TimesDates and Variable serialization WIP

Author: Affie

Project.toml

@@ -16,7 +16,6 @@ InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 ManifoldsBase = "3362f125-f0bb-47a3-aa74-596ffd7ef2fb"
-NanoDates = "46f1a544-deae-4307-8689-c12aa3c955c6"
 OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
@@ -29,6 +28,7 @@ Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
 Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
 TensorCast = "02d47bb6-7ce6-556a-be16-bb1710789e2b"
 TimeZones = "f269a46b-ccf7-5d73-abea-4c690281aa53"
+TimesDates = "bdfc003b-8df8-5c39-adcd-3a9087f5df4a"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 
 [weakdeps]
@@ -57,7 +57,6 @@ JSON = "1.0.0"
 LieGroups = "0.1"
 LinearAlgebra = "1.10"
 ManifoldsBase = "1, 2"
-NanoDates = "1.0.3"
 OrderedCollections = "1.4"
 Pkg = "1.4, 1.5"
 ProgressMeter = "1"
@@ -72,6 +71,7 @@ Tar = "1.9"
 TensorCast = "0.3.3, 0.4"
 Test = "1.10"
 TimeZones = "1.3.1"
+TimesDates = "0.3.3"
 UUIDs = "1.10"
 julia = "1.10"
 

src/DataBlobs/entities/BlobEntry.jl

@@ -40,18 +40,21 @@ StructUtils.@kwarg struct Blobentry
     """ Additional information that can help a different user of the Blob. """
     description::String = ""
     """ MIME description describing the format of binary data in the `Blob`, e.g. 'image/png' or 'application/json'. """
-    mimetype::String = "application/octet-stream" #FIXME ::MIME = MIME("application/octet-stream")
+    mimetype::MIME = MIME("application/octet-stream")
     """ Storage for a couple of bytes directly in the graph. Use with caution and keep it small and simple."""
     metadata::JSONText = JSONText("{}")
     """ When the Blob itself was first created. Serialized as an ISO 8601 string."""
-    timestamp::NanoDate = ndnow(UTC) & (json = (lower = timestamp,),)
+    timestamp::TimeDateZone = TimeDateZone(now(localzone()))
     """ Type version of this Blobentry."""
-    version::VersionNumber = version(Blobentry)
+    version::VersionNumber = DFG.version(Blobentry)
 end
 version(::Type{Blobentry}) = v"0.1.0"
 
-function Blobentry(label::Symbol, blobstore = :default; kwargs...)
-    return Blobentry(; label, blobstore, kwargs...)
+function Blobentry(label::Symbol, blobstore = :default; metadata, kwargs...)
+    if !(metadata isa JSONText)
+        metadata = JSONText(JSON.json(metadata))
+    end
+    return Blobentry(; label, blobstore, metadata, kwargs...)
 end
 # construction helper from existing Blobentry for user overriding via kwargs
 function Blobentry(
@@ -123,3 +126,25 @@ function Base.setproperty!(x::Blobentry, f::Symbol, val)
 end
 
 const Blobentries = OrderedDict{Symbol, Blobentry}
+
+#TODO write Blobentries as array
+# StructUtils.dictlike(::Type{Blobentries}) = false
+# StructUtils.structlike(::Type{Blobentries}) = false
+# StructUtils.arraylike(::Type{Blobentries}) = false
+
+# function StructUtils.lower(entries::Blobentries)
+#     return map(collect(values(entries))) do (entry)
+#         return StructUtils.lower(entry)
+#     end
+# end
+
+# function StructUtils.lift(s::StructUtils.StructStyle, T::Type{Blobentries}, json_vector)
+#     entries = T()
+#     @warn "lifting" s T json_vector
+#     foreach(json_vector) do obj
+#         global gobj = obj
+#         return push!(entries, Symbol(obj.label) => StructUtils.make(s, Blobentry, obj))
+#     end
+#     @warn "lifted Blobentries" entries
+#     return entries, nothing
+# end

src/DataBlobs/services/BlobEntry.jl

@@ -67,17 +67,17 @@ Also see: [`addBlobentry!`](@ref), [`getBlob`](@ref), [`listBlobentries`](@ref)
 """
 function getBlobentry(var::AbstractGraphVariable, key::Symbol)
     if !hasBlobentry(var, key)
-        throw(LabelNotFoundError("Blobentry", key, collect(keys(var.dataDict))))
+        throw(LabelNotFoundError("Blobentry", key, collect(keys(var.blobentries))))
     end
-    return var.dataDict[key]
+    return var.blobentries[key]
 end
 
-function getBlobentry(var::VariableDFG, key::Symbol)
-    if !hasBlobentry(var, key)
-        throw(LabelNotFoundError("Blobentry", key))
-    end
-    return var.blobEntries[findfirst(x -> x.label == key, var.blobEntries)]
-end
+# function getBlobentry(var::VariableDFG, key::Symbol)
+#     if !hasBlobentry(var, key)
+#         throw(LabelNotFoundError("Blobentry", key))
+#     end
+#     return var.blobEntries[findfirst(x -> x.label == key, var.blobEntries)]
+# end
 
 """
     $(SIGNATURES)
@@ -131,17 +131,17 @@ Should be extended if DFG variable is not returned by reference.
 Also see: [`getBlobentry`](@ref), [`addBlob!`](@ref), [`mergeBlobentry!`](@ref)
 """
 function addBlobentry!(var::VariableCompute, entry::Blobentry)
-    haskey(var.dataDict, entry.label) && throw(LabelExistsError("Blobentry", entry.label))
-    var.dataDict[entry.label] = entry
+    haskey(var.blobentries, entry.label) && throw(LabelExistsError("Blobentry", entry.label))
+    var.blobentries[entry.label] = entry
     return entry
 end
 
-function addBlobentry!(var::VariableDFG, entry::Blobentry)
-    entry.label in getproperty.(var.blobEntries, :label) &&
-        throw(LabelExistsError("Blobentry", entry.label))
-    push!(var.blobEntries, entry)
-    return entry
-end
+# function addBlobentry!(var::VariableDFG, entry::Blobentry)
+#     entry.label in getproperty.(var.blobEntries, :label) &&
+#         throw(LabelExistsError("Blobentry", entry.label))
+#     push!(var.blobEntries, entry)
+#     return entry
+# end
 
 function addBlobentry!(dfg::AbstractDFG, vLbl::Symbol, entry::Blobentry)
     return addBlobentry!(getVariable(dfg, vLbl), entry)
@@ -158,10 +158,10 @@ If the Blobentry does not exist, it will be added.
 Notes:
 """
 function mergeBlobentry!(var::AbstractGraphVariable, bde::Blobentry)
-    if !haskey(var.dataDict, bde.label)
+    if !haskey(var.blobentries, bde.label)
         addBlobentry!(var, bde)
     else
-        var.dataDict[bde.label] = bde
+        var.blobentries[bde.label] = bde
     end
     return 1
 end
@@ -179,15 +179,15 @@ Notes:
 """
 function deleteBlobentry!(var::VariableCompute, key::Symbol)
     !hasBlobentry(var, key) && throw(LabelNotFoundError("Blobentry", key))
-    delete!(var.dataDict, key)
+    delete!(var.blobentries, key)
     return 1
 end
 
-function deleteBlobentry!(var::VariableDFG, key::Symbol)
-    !hasBlobentry(var, key) && throw(LabelNotFoundError("Blobentry", key))
-    deleteat!(var.blobEntries, findfirst(x -> x.label == key, var.blobEntries))
-    return 1
-end
+# function deleteBlobentry!(var::VariableDFG, key::Symbol)
+#     !hasBlobentry(var, key) && throw(LabelNotFoundError("Blobentry", key))
+#     deleteat!(var.blobEntries, findfirst(x -> x.label == key, var.blobEntries))
+#     return 1
+# end
 
 function deleteBlobentry!(dfg::AbstractDFG, label::Symbol, key::Symbol)
     return deleteBlobentry!(getVariable(dfg, label), key)
@@ -223,24 +223,24 @@ end
 
 Does a blob entry exist with `blobLabel`.
 """
-hasBlobentry(v::VariableCompute, blobLabel::Symbol) = haskey(v.dataDict, blobLabel)
+hasBlobentry(v::VariableCompute, blobLabel::Symbol) = haskey(v.blobentries, blobLabel)
 
-function hasBlobentry(v::VariableDFG, label::Symbol)
-    return label in getproperty.(v.blobEntries, :label)
-end
+# function hasBlobentry(v::VariableDFG, label::Symbol)
+#     return label in getproperty.(v.blobEntries, :label)
+# end
 
 """
     $(SIGNATURES)
 
 Get blob entries, returns a `Vector{Blobentry}`.
 """
 function getBlobentries(v::VariableCompute)
-    return collect(values(v.dataDict))
+    return collect(values(v.blobentries))
 end
 
-function getBlobentries(v::VariableDFG)
-    return copy(v.blobEntries)
-end
+# function getBlobentries(v::VariableDFG)
+#     return copy(v.blobEntries)
+# end
 
 function getBlobentries(
     v::AbstractGraphVariable;
@@ -289,12 +289,12 @@ const collectBlobentries = gatherBlobentries
 List the blob entries associated with a particular variable.
 """
 function listBlobentries(var::AbstractGraphVariable)
-    return collect(keys(var.dataDict))
+    return collect(keys(var.blobentries))
 end
 
-function listBlobentries(var::VariableDFG)
-    return getproperty.(var.blobEntries, :label)
-end
+# function listBlobentries(var::VariableDFG)
+#     return getproperty.(var.blobEntries, :label)
+# end
 
 function listBlobentries(dfg::AbstractDFG, label::Symbol)
     return listBlobentries(getVariable(dfg, label))

src/Deprecated.jl

@@ -193,7 +193,7 @@ function setSolverData!(v::VariableCompute, data::State, key::Symbol = :default)
         :setSolverData!,
     )
     @assert key == data.solveKey "State.solveKey=:$(data.solveKey) does not match requested :$(key)"
-    return v.solverDataDict[key] = data
+    return v.states[key] = data
 end
 
 @deprecate mergeVariableSolverData!(args...; kwargs...) mergeState!(args...; kwargs...)

src/DistributedFactorGraphs.jl

@@ -20,7 +20,7 @@ using DocStringExtensions
 using Dates
 using Random
 using TimeZones
-using NanoDates
+using TimesDates
 using JSON
 export StructUtils # export for use in macros
 using LinearAlgebra
@@ -83,7 +83,7 @@ export FactorDFG, FactorSummary, FactorSkeleton
 
 export Blobentry
 
-export State, PackedState
+export State
 
 ##------------------------------------------------------------------------------
 ## Functions
@@ -531,6 +531,7 @@ include("entities/Bloblet.jl")
 include("DataBlobs/entities/BlobEntry.jl")
 include("DataBlobs/entities/BlobStores.jl")
 
+include("serialization/DFGStructStyles.jl")
 include("serialization/PackedSerialization.jl")
 include("serialization/DistributionSerialization.jl")
 

src/FileDFG/services/FileDFG.jl

@@ -40,19 +40,19 @@ function saveDFG(folder::AbstractString, dfg::AbstractDFG; saveMetadata::Bool =
     map(f -> rm("$factorFolder/$f"), readdir(factorFolder))
     # Variables
     @showprogress "saving variables" for v in variables
-        vPacked = packVariable(v)
-        JSON.json("$varFolder/$(v.label).json", vPacked)
+        # vPacked = packVariable(v)
+        JSON.json("$varFolder/$(v.label).json", v; style = DFGJSONStyle())
     end
     # Factors
     @showprogress "saving factors" for f in factors
-        JSON.json("$factorFolder/$(f.label).json", f)
+        JSON.json("$factorFolder/$(f.label).json", f; style = DFGJSONStyle())
     end
     #GraphsDFG metadata
     if saveMetadata
         @assert isa(dfg, GraphsDFG) "only metadata for GraphsDFG are supported"
         @info "saving dfg metadata"
         fgPacked = GraphsDFGs.packDFGMetadata(dfg)
-        JSON.json("$savepath/dfg.json", fgPacked)
+        JSON.json("$savepath/dfg.json", fgPacked; style = DFGJSONStyle())
     end
 
     savedir = dirname(savepath) # is this a path of just local name? #344 -- workaround with unique names
@@ -138,7 +138,7 @@ function loadDFG!(
         @assert isa(dfgLoadInto, GraphsDFG) "Only GraphsDFG metadata are supported"
         @info "loading dfg metadata"
         jstr = read("$folder/dfg.json", String)
-        fgPacked = JSON.parse(jstr, GraphsDFGs.PackedGraphsDFG)
+        fgPacked = JSON.parse(jstr, GraphsDFGs.PackedGraphsDFG; style = DFGJSONStyle())
         GraphsDFGs.unpackDFGMetadata!(dfgLoadInto, fgPacked)
     end
 
@@ -164,7 +164,7 @@ function loadDFG!(
     # type instability on `variables` as either `::Vector{Variable}` or `::Vector{VariableCompute{<:}}` (vector of abstract)
     variables = @showprogress 1 "loading variables" asyncmap(varFiles) do varFile
         jstr = read("$varFolder/$varFile", String)
-        packedvar = JSON.parse(jstr, VariableDFG)
+        packedvar = JSON.parse(jstr, VariableDFG; style = DFGJSONStyle())
         v = usePackedVariable ? packedvar : unpackVariable(packedvar)
         return addVariable!(dfgLoadInto, v)
     end
@@ -176,7 +176,7 @@ function loadDFG!(
 
     # `factors` is not type stable `::Vector{Factor}` or `::Vector{FactorCompute{<:}}` (vector of abstract)
     factors = @showprogress 1 "loading factors" asyncmap(factorFiles) do factorFile
-        f = JSON.parsefile("$factorFolder/$factorFile", FactorDFG)
+        f = JSON.parsefile("$factorFolder/$factorFile", FactorDFG; style = DFGJSONStyle())
         return addFactor!(dfgLoadInto, f)
     end
 
@@ -228,12 +228,12 @@ function loadDFG(file::AbstractString)
     #TODO deprecate old format, v0.28
     local fgPacked
     try
-        fgPacked = JSON.parse(jstr, GraphsDFGs.PackedGraphsDFG)
+        fgPacked = JSON.parse(jstr, GraphsDFGs.PackedGraphsDFG; style = DFGJSONStyle())
     catch e
         if e isa MethodError
             @warn "Deprecated serialization: Failed to read DFG metadata. Attempting to load using the old format. Error:" e
             fgPacked =
-                GraphsDFGs.PackedGraphsDFG(JSON.parse(jstr, GraphsDFGs._OldPackedGraphsDFG))
+                GraphsDFGs.PackedGraphsDFG(JSON.parse(jstr, GraphsDFGs._OldPackedGraphsDFG; style = DFGJSONStyle()))
         else
             rethrow(e)
         end

src/entities/Bloblet.jl

@@ -21,3 +21,46 @@ function StructUtils.lift(::Type{Bloblets}, json_vector::Vector)
         Symbol(x["label"]) => Bloblet(Symbol(x["label"]), x["val"]) for x in json_vector
     )
 end
+
+"""
+    $(SIGNATURES)
+"""
+function getBloblet(node, label::Symbol)
+    !haskey(refBloblets(node), label) && throw(LabelNotFoundError("Bloblet", label))
+    return refBloblets(node)[label]
+end
+
+"""
+    $(SIGNATURES)
+"""
+function addBloblet!(node, bloblet::Bloblet)
+    label = getLabel(bloblet)
+    haskey(refBloblets(node), label) && throw(LabelExistsError("Bloblet", label))
+    refBloblets(node)[label] = bloblet
+    return bloblet
+end
+
+"""
+    $(SIGNATURES)
+"""
+function mergeBloblet!(node, bloblet::Bloblet)
+    refBloblets(node)[getLabel(bloblet)] = bloblet
+    return 1
+end
+
+"""
+    $(SIGNATURES)
+"""
+function deleteBloblet!(node, label::Symbol)
+    !haskey(refBloblets(node), label) && throw(LabelNotFoundError("Bloblet", label))
+    pop!(refBloblets(node), label)
+    return 1
+end
+
+"""
+    $(SIGNATURES)
+List all Bloblet keys for a variable `label` in `dfg`
+"""
+function listBloblets(node)
+    return collect(keys(refBloblets(node)))
+end