diff --git a/Project.toml b/Project.toml index 2e7ba834..9252a4e2 100644 --- a/Project.toml +++ b/Project.toml @@ -13,7 +13,7 @@ DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6" InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" -JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" ManifoldsBase = "3362f125-f0bb-47a3-aa74-596ffd7ef2fb" OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" @@ -24,7 +24,6 @@ RecursiveArrayTools = "731186ca-8d62-57ce-b412-fbd966d074cd" SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" -StructTypes = "856f2bd8-1eba-4b0a-8007-ebc267875bd4" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" TensorCast = "02d47bb6-7ce6-556a-be16-bb1710789e2b" @@ -46,7 +45,6 @@ Base64 = "1.10" CSV = "0.10" CodecZlib = "0.7" Colors = "0.10, 0.11, 0.12, 0.13" -DataStructures = "0.18" Dates = "1.10" Distributions = "0.23, 0.24, 0.25" DocStringExtensions = "0.8, 0.9" @@ -54,7 +52,8 @@ FileIO = "1" GraphMakie = "0.5.14, 0.6" Graphs = "1.4" InteractiveUtils = "1.10" -JSON3 = "1" +JSON = "1.0.0" +LieGroups = "0.1" LinearAlgebra = "1.10" Manifolds = "0.9, 0.10" ManifoldsBase = "0.14, 0.15, 1" @@ -67,7 +66,6 @@ SHA = "0.7, 1" SparseArrays = "1.10" StaticArrays = "1" Statistics = "1.10" -StructTypes = "1" Tables = "v1.11.1" Tar = "1.9" TensorCast = "0.3.3, 0.4" @@ -75,11 +73,9 @@ Test = "1.10" TimeZones = "1.3.1" UUIDs = "1.10" julia = "1.10" -LieGroups = "0.1" [extras] Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" -DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" GraphMakie = "1ecd5474-83a3-4783-bb4f-06765db800d2" LieGroups = "6774de46-80ba-43f8-ba42-e41071ccfc5f" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" @@ -89,4 +85,4 @@ Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Aqua", "Test", "DataStructures", "GraphMakie", "LinearAlgebra", "Manifolds", "Pkg", "Statistics", "LieGroups"] +test = ["Aqua", "Test", "GraphMakie", "LinearAlgebra", "Manifolds", "Pkg", "Statistics", "LieGroups"] diff --git a/src/DataBlobs/entities/BlobEntry.jl b/src/DataBlobs/entities/BlobEntry.jl index f8018c2f..d0c62cc2 100644 --- a/src/DataBlobs/entities/BlobEntry.jl +++ b/src/DataBlobs/entities/BlobEntry.jl @@ -46,10 +46,6 @@ Base.@kwdef struct Blobentry _version::VersionNumber = _getDFGVersion() end -StructTypes.StructType(::Type{Blobentry}) = StructTypes.UnorderedStruct() -StructTypes.idproperty(::Type{Blobentry}) = :id -StructTypes.omitempties(::Type{Blobentry}) = (:id,) - function Blobentry(label::Symbol, blobstore = :default; kwargs...) return Blobentry(; label, blobstore, kwargs...) end diff --git a/src/Deprecated.jl b/src/Deprecated.jl index a2dcaa6c..8536daf1 100644 --- a/src/Deprecated.jl +++ b/src/Deprecated.jl @@ -1,3 +1,7 @@ +## ================================================================================ +## Deprecated in v0.29 +##================================================================================= + ## ================================================================================ ## Deprecated in v0.28 ##================================================================================= @@ -607,412 +611,412 @@ end ## Deprecated in v0.27 ##================================================================================= -const AbstractFactor = AbstractObservation -const AbstractPackedFactor = AbstractPackedObservation -const FactorOperationalMemory = FactorCache -const VariableNodeData = State - -@deprecate getNeighborhood(args...; kwargs...) listNeighborhood(args...; kwargs...) -@deprecate addBlob!(store::AbstractBlobstore, blobId::UUID, data, ::String) addBlob!( - store, - blobId, - data, -) -@deprecate addBlob!(store::AbstractBlobstore{T}, data::T, ::String) where {T} addBlob!( - store, - uuid4(), - data, -) - -@deprecate updateVariable!(args...) mergeVariable!(args...) -@deprecate updateFactor!(args...) mergeFactor!(args...) - -@deprecate updateBlobEntry!(args...) mergeBlobentry!(args...) -@deprecate updateGraphBlobEntry!(args...) mergeGraphBlobentry!(args...) -@deprecate updateAgentBlobEntry!(args...) mergeAgentBlobentry!(args...) - -@deprecate getBlobStore(args...) getBlobstore(args...) -@deprecate addBlobStore!(args...) addBlobstore!(args...) -@deprecate updateBlobStore!(args...) updateBlobstore!(args...) -@deprecate deleteBlobStore!(args...) deleteBlobstore!(args...) -@deprecate emptyBlobStore!(args...) emptyBlobstore!(args...) -@deprecate listBlobStores(args...) listBlobstores(args...) - -@deprecate BlobEntry(args...; kwargs...) Blobentry(args...; kwargs...) -@deprecate getGraphBlobEntry(args...; kwargs...) getGraphBlobentry(args...; kwargs...) -@deprecate getGraphBlobEntries(args...; kwargs...) getGraphBlobentries(args...; kwargs...) -@deprecate addGraphBlobEntry!(args...; kwargs...) addGraphBlobentry!(args...; kwargs...) -@deprecate addGraphBlobEntries!(args...; kwargs...) addGraphBlobentries!(args...; kwargs...) -@deprecate mergeGraphBlobEntry!(args...; kwargs...) mergeGraphBlobentry!(args...; kwargs...) -@deprecate deleteGraphBlobEntry!(args...; kwargs...) deleteGraphBlobentry!( - args...; - kwargs..., -) -@deprecate getAgentBlobEntry(args...; kwargs...) getAgentBlobentry(args...; kwargs...) -@deprecate getAgentBlobEntries(args...; kwargs...) getAgentBlobentries(args...; kwargs...) -@deprecate addAgentBlobEntry!(args...; kwargs...) addAgentBlobentry!(args...; kwargs...) -@deprecate addAgentBlobEntries!(args...; kwargs...) addAgentBlobentries!(args...; kwargs...) -@deprecate mergeAgentBlobEntry!(args...; kwargs...) mergeAgentBlobentry!(args...; kwargs...) -@deprecate deleteAgentBlobEntry!(args...; kwargs...) deleteAgentBlobentry!( - args...; - kwargs..., -) -@deprecate listGraphBlobEntries(args...; kwargs...) listGraphBlobentries(args...; kwargs...) -@deprecate listAgentBlobEntries(args...; kwargs...) listAgentBlobentries(args...; kwargs...) -@deprecate hasBlobEntry(args...; kwargs...) hasBlobentry(args...; kwargs...) -@deprecate getBlobEntry(args...; kwargs...) getBlobentry(args...; kwargs...) -@deprecate getBlobEntryFirst(args...; kwargs...) getfirstBlobentry(args...; kwargs...) -@deprecate getBlobentry(var::AbstractGraphVariable, blobId::UUID) getfirstBlobentry( - var::AbstractGraphVariable, - blobId::UUID, -) -@deprecate addBlobEntry!(args...; kwargs...) addBlobentry!(args...; kwargs...) -@deprecate addBlobEntries!(args...; kwargs...) addBlobentries!(args...; kwargs...) -@deprecate mergeBlobEntry!(args...; kwargs...) mergeBlobentry!(args...; kwargs...) -@deprecate deleteBlobEntry!(args...; kwargs...) deleteBlobentry!(args...; kwargs...) -@deprecate listBlobEntrySequence(args...; kwargs...) listBlobentrySequence( - args...; - kwargs..., -) -@deprecate mergeBlobEntries!(args...; kwargs...) mergeBlobentries!(args...; kwargs...) - -@deprecate getVariableSolverData(args...; kwargs...) getState(args...; kwargs...) -@deprecate addVariableSolverData!(args...; kwargs...) addState!(args...; kwargs...) -@deprecate deleteVariableSolverData!(args...; kwargs...) deleteState!(args...; kwargs...) -@deprecate listVariableSolverData(args...; kwargs...) listStates(args...; kwargs...) -@deprecate getVariableSolverDataAll(args...; kwargs...) getStates(args...; kwargs...) - -@deprecate getSolverData(v::VariableCompute, solveKey::Symbol = :default) getState( - v, - solveKey, -) false - -@deprecate packVariableNodeData(args...; kwargs...) packState(args...; kwargs...) -@deprecate unpackVariableNodeData(args...; kwargs...) unpackState(args...; kwargs...) - -#TODO possibly completely deprecated or not exported until update verb is standardized -function updateVariableSolverData!( - dfg::AbstractDFG, - variablekey::Symbol, - vnd::State, - useCopy::Bool = false, - fields::Vector{Symbol} = Symbol[]; - warn_if_absent::Bool = true, -) - Base.depwarn( - "updateVariableSolverData! is deprecated, use mergeState! or copytoState! instead", - :updateVariableSolverData!, - ) - #This is basically just setSolverData - var = getVariable(dfg, variablekey) - warn_if_absent && - !haskey(var.solverDataDict, vnd.solveKey) && - @warn "State '$(vnd.solveKey)' does not exist, adding" - - # for InMemoryDFGTypes do memory copy or repointing, for cloud this would be an different kind of update. - usevnd = vnd # useCopy ? deepcopy(vnd) : vnd - # should just one, or many pointers be updated? - useExisting = - haskey(var.solverDataDict, vnd.solveKey) && - isa(var.solverDataDict[vnd.solveKey], State) && - length(fields) != 0 - # @error useExisting vnd.solveKey - if useExisting - # change multiple pointers inside the VND var.solverDataDict[solvekey] - for field in fields - destField = getfield(var.solverDataDict[vnd.solveKey], field) - srcField = getfield(usevnd, field) - if isa(destField, Array) && size(destField) == size(srcField) - # use broadcast (in-place operation) - destField .= srcField - else - # change pointer of destination VND object member - setfield!(var.solverDataDict[vnd.solveKey], field, srcField) - end - end - else - # change a single pointer in var.solverDataDict - var.solverDataDict[vnd.solveKey] = usevnd - end - - return var.solverDataDict[vnd.solveKey] -end - -function updateVariableSolverData!( - dfg::AbstractDFG, - variablekey::Symbol, - vnd::State, - solveKey::Symbol, - useCopy::Bool = false, - fields::Vector{Symbol} = Symbol[]; - warn_if_absent::Bool = true, -) - # TODO not very clean - if vnd.solveKey != solveKey - Base.depwarn( - "updateVariableSolverData with solveKey is deprecated use copytoState! instead.", - :updateVariableSolverData!, - ) - usevnd = useCopy ? deepcopy(vnd) : vnd - usevnd.solveKey = solveKey - return updateVariableSolverData!( - dfg, - variablekey, - usevnd, - useCopy, - fields; - warn_if_absent = warn_if_absent, - ) - else - return updateVariableSolverData!( - dfg, - variablekey, - vnd, - useCopy, - fields; - warn_if_absent = warn_if_absent, - ) - end -end - -function updateVariableSolverData!( - dfg::AbstractDFG, - sourceVariable::VariableCompute, - solveKey::Symbol = :default, - useCopy::Bool = false, - fields::Vector{Symbol} = Symbol[]; - warn_if_absent::Bool = true, -) - # - vnd = getSolverData(sourceVariable, solveKey) - # toshow = listSolveKeys(sourceVariable) |> collect - # @info "update DFGVar solveKey" solveKey vnd.solveKey - # @show toshow - @assert solveKey == vnd.solveKey "State's solveKey=:$(vnd.solveKey) does not match requested :$solveKey" - return updateVariableSolverData!( - dfg, - sourceVariable.label, - vnd, - useCopy, - fields; - warn_if_absent = warn_if_absent, - ) -end - -function updateVariableSolverData!( - dfg::AbstractDFG, - sourceVariables::Vector{<:VariableCompute}, - solveKey::Symbol = :default, - useCopy::Bool = false, - fields::Vector{Symbol} = Symbol[]; - warn_if_absent::Bool = true, -) - #I think cloud would do this in bulk for speed - for var in sourceVariables - updateVariableSolverData!( - dfg, - var.label, - getSolverData(var, solveKey), - useCopy, - fields; - warn_if_absent = warn_if_absent, - ) - end -end - -## factor refactor deprecations -Base.@kwdef mutable struct GenericFunctionNodeData{ - T <: Union{<:AbstractPackedObservation, <:AbstractObservation, <:FactorCache}, -} - eliminated::Bool = false - potentialused::Bool = false - edgeIDs::Vector{Int} = Int[] - fnc::T - multihypo::Vector{Float64} = Float64[] # TODO re-evaluate after refactoring w #477 - certainhypo::Vector{Int} = Int[] - nullhypo::Float64 = 0.0 - solveInProgress::Int = 0 - inflation::Float64 = 0.0 -end - -function FactorCompute( - label::Symbol, - timestamp::Union{DateTime, ZonedDateTime}, - nstime::Nanosecond, - tags::Set{Symbol}, - solverData::GenericFunctionNodeData, - solvable::Int, - variableOrder::Union{Vector{Symbol}, Tuple}; - observation = getFactorType(solverData), - state::FactorState = FactorState(), - solvercache::Base.RefValue{<:FactorCache} = Ref{FactorCache}(), - id::Union{UUID, Nothing} = nothing, - smallData::Dict{Symbol, MetadataTypes} = Dict{Symbol, MetadataTypes}(), -) - error( - "This constructor is deprecated, use FactorCompute(label, variableOrder, solverData; ...) instead", - ) - return FactorCompute( - id, - label, - tags, - Tuple(variableOrder), - timestamp, - nstime, - Ref(solverData), - Ref(solvable), - smallData, - observation, - state, - solvercache, - ) -end - -function getSolverData(f::FactorCompute) - return error( - "getSolverData(f::FactorCompute) is obsolete, use getFactorState, getObservation, or getCache instead", - ) -end - -function setSolverData!(f::FactorCompute, data::GenericFunctionNodeData) - return error( - "setSolverData!(f::FactorCompute, data::GenericFunctionNodeData) is obsolete, use setState!, or setCache! instead", - ) -end - -@deprecate unpackFactor(dfg::AbstractDFG, factor::FactorDFG; skipVersionCheck::Bool = false) unpackFactor( - factor; - skipVersionCheck, -) false - -@deprecate rebuildFactorMetadata!(args...; kwargs...) rebuildFactorCache!( - args...; - kwargs..., -) - -function reconstFactorData end - -function decodePackedType( - dfg::AbstractDFG, - varOrder::AbstractVector{Symbol}, - ::Type{T}, - packeddata::GenericFunctionNodeData{PT}, -) where {T <: FactorCache, PT} - error("decodePackedType is obsolete") - # - # TODO, to solve IIF 1424 - # variables = map(lb->getVariable(dfg, lb), varOrder) - - # Also look at parentmodule - usrtyp = convertStructType(PT) - fulltype = DFG.FunctionNodeData{T{usrtyp}} - factordata = reconstFactorData(dfg, varOrder, fulltype, packeddata) - return factordata -end - -function _packSolverData(f::FactorCompute, fnctype::AbstractObservation) - # - error("_packSolverData is deprecated, use seperate packing of observation #TODO") - packtype = convertPackedType(fnctype) - try - packed = convert(PackedFunctionNodeData{packtype}, getSolverData(f)) #TODO getSolverData - packedJson = packed - return packedJson - catch ex - io = IOBuffer() - showerror(io, ex, catch_backtrace()) - err = String(take!(io)) - msg = "Error while packing '$(f.label)' as '$fnctype', please check the unpacking/packing converters for this factor - \r\n$err" - error(msg) - end -end - -const PackedFunctionNodeData{T} = - GenericFunctionNodeData{T} where {T <: AbstractPackedObservation} -function PackedFunctionNodeData(args...; kw...) - error("PackedFunctionNodeData is obsolete") - return PackedFunctionNodeData{typeof(args[4])}(args...; kw...) -end - -const FunctionNodeData{T} = - GenericFunctionNodeData{T} where {T <: Union{<:AbstractObservation, <:FactorCache}} -FunctionNodeData(args...; kw...) = FunctionNodeData{typeof(args[4])}(args...; kw...) - -# this is the GenericFunctionNodeData for packed types -#TODO deprecate FactorData in favor of FactorState (with no more distinction between packed and compute) -const FactorData = PackedFunctionNodeData{AbstractPackedObservation} - -function FactorCompute( - label::Symbol, - variableOrder::Union{Vector{Symbol}, Tuple}, - solverData::GenericFunctionNodeData; - tags::Set{Symbol} = Set{Symbol}(), - timestamp::Union{DateTime, ZonedDateTime} = now(localzone()), - solvable::Int = 1, - nstime::Nanosecond = Nanosecond(0), - id::Union{UUID, Nothing} = nothing, - smallData::Dict{Symbol, MetadataTypes} = Dict{Symbol, MetadataTypes}(), -) - Base.depwarn( - "`FactorCompute` constructor with `GenericFunctionNodeData` is deprecated. observation, state, and solvercache should be provided explicitly.", - :FactorCompute, - ) - observation = getFactorType(solverData) - state = FactorState( - solverData.eliminated, - solverData.potentialused, - solverData.multihypo, - solverData.certainhypo, - solverData.nullhypo, - solverData.solveInProgress, - solverData.inflation, - ) - - if solverData.fnc isa FactorCache - solvercache = solverData.fnc - else - solvercache = nothing - end - - return FactorCompute( - label, - Tuple(variableOrder), - observation, - state, - solvercache; - id, - timestamp, - nstime, - tags, - smallData, - solvable, - ) -end - -# Deprecated check usefull? # packedFnc = fncStringToData(factor.fnctype, factor.data) -# Deprecated check usefull? # decodeType = getFactorOperationalMemoryType(dfg) -# Deprecated check usefull? # fullFactorData = decodePackedType(dfg, factor._variableOrderSymbols, decodeType, packedFnc) -function fncStringToData(args...; kwargs...) - @warn "fncStringToData is obsolete, called with" args kwargs - return error("fncStringToData is obsolete.") -end - -#TODO make sure getFactorOperationalMemoryType is obsolete -function getFactorOperationalMemoryType(dummy) - return error( - "Please extend your workspace with function getFactorOperationalMemoryType(<:AbstractParams) for your usecase, e.g. IncrementalInference uses `CommonConvWrapper <: FactorCache`", - ) -end -function getFactorOperationalMemoryType(dfg::AbstractDFG) - return getFactorOperationalMemoryType(getSolverParams(dfg)) -end - -function typeModuleName(variableType::StateType) - Base.depwarn("typeModuleName is obsolete", :typeModuleName) - io = IOBuffer() - ioc = IOContext(io, :module => DistributedFactorGraphs) - show(ioc, typeof(variableType)) - return String(take!(io)) -end - -typeModuleName(varT::Type{<:StateType}) = typeModuleName(varT()) +# const AbstractFactor = AbstractObservation +# const AbstractPackedFactor = AbstractPackedObservation +# const FactorOperationalMemory = FactorCache +# const VariableNodeData = State + +# @deprecate getNeighborhood(args...; kwargs...) listNeighborhood(args...; kwargs...) +# @deprecate addBlob!(store::AbstractBlobstore, blobId::UUID, data, ::String) addBlob!( +# store, +# blobId, +# data, +# ) +# @deprecate addBlob!(store::AbstractBlobstore{T}, data::T, ::String) where {T} addBlob!( +# store, +# uuid4(), +# data, +# ) + +# @deprecate updateVariable!(args...) mergeVariable!(args...) +# @deprecate updateFactor!(args...) mergeFactor!(args...) + +# @deprecate updateBlobEntry!(args...) mergeBlobentry!(args...) +# @deprecate updateGraphBlobEntry!(args...) mergeGraphBlobentry!(args...) +# @deprecate updateAgentBlobEntry!(args...) mergeAgentBlobentry!(args...) + +# @deprecate getBlobStore(args...) getBlobstore(args...) +# @deprecate addBlobStore!(args...) addBlobstore!(args...) +# @deprecate updateBlobStore!(args...) updateBlobstore!(args...) +# @deprecate deleteBlobStore!(args...) deleteBlobstore!(args...) +# @deprecate emptyBlobStore!(args...) emptyBlobstore!(args...) +# @deprecate listBlobStores(args...) listBlobstores(args...) + +# @deprecate BlobEntry(args...; kwargs...) Blobentry(args...; kwargs...) +# @deprecate getGraphBlobEntry(args...; kwargs...) getGraphBlobentry(args...; kwargs...) +# @deprecate getGraphBlobEntries(args...; kwargs...) getGraphBlobentries(args...; kwargs...) +# @deprecate addGraphBlobEntry!(args...; kwargs...) addGraphBlobentry!(args...; kwargs...) +# @deprecate addGraphBlobEntries!(args...; kwargs...) addGraphBlobentries!(args...; kwargs...) +# @deprecate mergeGraphBlobEntry!(args...; kwargs...) mergeGraphBlobentry!(args...; kwargs...) +# @deprecate deleteGraphBlobEntry!(args...; kwargs...) deleteGraphBlobentry!( +# args...; +# kwargs..., +# ) +# @deprecate getAgentBlobEntry(args...; kwargs...) getAgentBlobentry(args...; kwargs...) +# @deprecate getAgentBlobEntries(args...; kwargs...) getAgentBlobentries(args...; kwargs...) +# @deprecate addAgentBlobEntry!(args...; kwargs...) addAgentBlobentry!(args...; kwargs...) +# @deprecate addAgentBlobEntries!(args...; kwargs...) addAgentBlobentries!(args...; kwargs...) +# @deprecate mergeAgentBlobEntry!(args...; kwargs...) mergeAgentBlobentry!(args...; kwargs...) +# @deprecate deleteAgentBlobEntry!(args...; kwargs...) deleteAgentBlobentry!( +# args...; +# kwargs..., +# ) +# @deprecate listGraphBlobEntries(args...; kwargs...) listGraphBlobentries(args...; kwargs...) +# @deprecate listAgentBlobEntries(args...; kwargs...) listAgentBlobentries(args...; kwargs...) +# @deprecate hasBlobEntry(args...; kwargs...) hasBlobentry(args...; kwargs...) +# @deprecate getBlobEntry(args...; kwargs...) getBlobentry(args...; kwargs...) +# @deprecate getBlobEntryFirst(args...; kwargs...) getfirstBlobentry(args...; kwargs...) +# @deprecate getBlobentry(var::AbstractGraphVariable, blobId::UUID) getfirstBlobentry( +# var::AbstractGraphVariable, +# blobId::UUID, +# ) +# @deprecate addBlobEntry!(args...; kwargs...) addBlobentry!(args...; kwargs...) +# @deprecate addBlobEntries!(args...; kwargs...) addBlobentries!(args...; kwargs...) +# @deprecate mergeBlobEntry!(args...; kwargs...) mergeBlobentry!(args...; kwargs...) +# @deprecate deleteBlobEntry!(args...; kwargs...) deleteBlobentry!(args...; kwargs...) +# @deprecate listBlobEntrySequence(args...; kwargs...) listBlobentrySequence( +# args...; +# kwargs..., +# ) +# @deprecate mergeBlobEntries!(args...; kwargs...) mergeBlobentries!(args...; kwargs...) + +# @deprecate getVariableSolverData(args...; kwargs...) getState(args...; kwargs...) +# @deprecate addVariableSolverData!(args...; kwargs...) addState!(args...; kwargs...) +# @deprecate deleteVariableSolverData!(args...; kwargs...) deleteState!(args...; kwargs...) +# @deprecate listVariableSolverData(args...; kwargs...) listStates(args...; kwargs...) +# @deprecate getVariableSolverDataAll(args...; kwargs...) getStates(args...; kwargs...) + +# @deprecate getSolverData(v::VariableCompute, solveKey::Symbol = :default) getState( +# v, +# solveKey, +# ) false + +# @deprecate packVariableNodeData(args...; kwargs...) packState(args...; kwargs...) +# @deprecate unpackVariableNodeData(args...; kwargs...) unpackState(args...; kwargs...) + +# #TODO possibly completely deprecated or not exported until update verb is standardized +# function updateVariableSolverData!( +# dfg::AbstractDFG, +# variablekey::Symbol, +# vnd::State, +# useCopy::Bool = false, +# fields::Vector{Symbol} = Symbol[]; +# warn_if_absent::Bool = true, +# ) +# Base.depwarn( +# "updateVariableSolverData! is deprecated, use mergeState! or copytoState! instead", +# :updateVariableSolverData!, +# ) +# #This is basically just setSolverData +# var = getVariable(dfg, variablekey) +# warn_if_absent && +# !haskey(var.solverDataDict, vnd.solveKey) && +# @warn "State '$(vnd.solveKey)' does not exist, adding" + +# # for InMemoryDFGTypes do memory copy or repointing, for cloud this would be an different kind of update. +# usevnd = vnd # useCopy ? deepcopy(vnd) : vnd +# # should just one, or many pointers be updated? +# useExisting = +# haskey(var.solverDataDict, vnd.solveKey) && +# isa(var.solverDataDict[vnd.solveKey], State) && +# length(fields) != 0 +# # @error useExisting vnd.solveKey +# if useExisting +# # change multiple pointers inside the VND var.solverDataDict[solvekey] +# for field in fields +# destField = getfield(var.solverDataDict[vnd.solveKey], field) +# srcField = getfield(usevnd, field) +# if isa(destField, Array) && size(destField) == size(srcField) +# # use broadcast (in-place operation) +# destField .= srcField +# else +# # change pointer of destination VND object member +# setfield!(var.solverDataDict[vnd.solveKey], field, srcField) +# end +# end +# else +# # change a single pointer in var.solverDataDict +# var.solverDataDict[vnd.solveKey] = usevnd +# end + +# return var.solverDataDict[vnd.solveKey] +# end + +# function updateVariableSolverData!( +# dfg::AbstractDFG, +# variablekey::Symbol, +# vnd::State, +# solveKey::Symbol, +# useCopy::Bool = false, +# fields::Vector{Symbol} = Symbol[]; +# warn_if_absent::Bool = true, +# ) +# # TODO not very clean +# if vnd.solveKey != solveKey +# Base.depwarn( +# "updateVariableSolverData with solveKey is deprecated use copytoState! instead.", +# :updateVariableSolverData!, +# ) +# usevnd = useCopy ? deepcopy(vnd) : vnd +# usevnd.solveKey = solveKey +# return updateVariableSolverData!( +# dfg, +# variablekey, +# usevnd, +# useCopy, +# fields; +# warn_if_absent = warn_if_absent, +# ) +# else +# return updateVariableSolverData!( +# dfg, +# variablekey, +# vnd, +# useCopy, +# fields; +# warn_if_absent = warn_if_absent, +# ) +# end +# end + +# function updateVariableSolverData!( +# dfg::AbstractDFG, +# sourceVariable::VariableCompute, +# solveKey::Symbol = :default, +# useCopy::Bool = false, +# fields::Vector{Symbol} = Symbol[]; +# warn_if_absent::Bool = true, +# ) +# # +# vnd = getSolverData(sourceVariable, solveKey) +# # toshow = listSolveKeys(sourceVariable) |> collect +# # @info "update DFGVar solveKey" solveKey vnd.solveKey +# # @show toshow +# @assert solveKey == vnd.solveKey "State's solveKey=:$(vnd.solveKey) does not match requested :$solveKey" +# return updateVariableSolverData!( +# dfg, +# sourceVariable.label, +# vnd, +# useCopy, +# fields; +# warn_if_absent = warn_if_absent, +# ) +# end + +# function updateVariableSolverData!( +# dfg::AbstractDFG, +# sourceVariables::Vector{<:VariableCompute}, +# solveKey::Symbol = :default, +# useCopy::Bool = false, +# fields::Vector{Symbol} = Symbol[]; +# warn_if_absent::Bool = true, +# ) +# #I think cloud would do this in bulk for speed +# for var in sourceVariables +# updateVariableSolverData!( +# dfg, +# var.label, +# getSolverData(var, solveKey), +# useCopy, +# fields; +# warn_if_absent = warn_if_absent, +# ) +# end +# end + +# ## factor refactor deprecations +# Base.@kwdef mutable struct GenericFunctionNodeData{ +# T <: Union{<:AbstractPackedObservation, <:AbstractObservation, <:FactorCache}, +# } +# eliminated::Bool = false +# potentialused::Bool = false +# edgeIDs::Vector{Int} = Int[] +# fnc::T +# multihypo::Vector{Float64} = Float64[] # TODO re-evaluate after refactoring w #477 +# certainhypo::Vector{Int} = Int[] +# nullhypo::Float64 = 0.0 +# solveInProgress::Int = 0 +# inflation::Float64 = 0.0 +# end + +# function FactorCompute( +# label::Symbol, +# timestamp::Union{DateTime, ZonedDateTime}, +# nstime::Nanosecond, +# tags::Set{Symbol}, +# solverData::GenericFunctionNodeData, +# solvable::Int, +# variableOrder::Union{Vector{Symbol}, Tuple}; +# observation = getFactorType(solverData), +# state::FactorState = FactorState(), +# solvercache::Base.RefValue{<:FactorCache} = Ref{FactorCache}(), +# id::Union{UUID, Nothing} = nothing, +# smallData::Dict{Symbol, MetadataTypes} = Dict{Symbol, MetadataTypes}(), +# ) +# error( +# "This constructor is deprecated, use FactorCompute(label, variableOrder, solverData; ...) instead", +# ) +# return FactorCompute( +# id, +# label, +# tags, +# Tuple(variableOrder), +# timestamp, +# nstime, +# Ref(solverData), +# Ref(solvable), +# smallData, +# observation, +# state, +# solvercache, +# ) +# end + +# function getSolverData(f::FactorCompute) +# return error( +# "getSolverData(f::FactorCompute) is obsolete, use getFactorState, getObservation, or getCache instead", +# ) +# end + +# function setSolverData!(f::FactorCompute, data::GenericFunctionNodeData) +# return error( +# "setSolverData!(f::FactorCompute, data::GenericFunctionNodeData) is obsolete, use setState!, or setCache! instead", +# ) +# end + +# @deprecate unpackFactor(dfg::AbstractDFG, factor::FactorDFG; skipVersionCheck::Bool = false) unpackFactor( +# factor; +# skipVersionCheck, +# ) false + +# @deprecate rebuildFactorMetadata!(args...; kwargs...) rebuildFactorCache!( +# args...; +# kwargs..., +# ) + +# function reconstFactorData end + +# function decodePackedType( +# dfg::AbstractDFG, +# varOrder::AbstractVector{Symbol}, +# ::Type{T}, +# packeddata::GenericFunctionNodeData{PT}, +# ) where {T <: FactorCache, PT} +# error("decodePackedType is obsolete") +# # +# # TODO, to solve IIF 1424 +# # variables = map(lb->getVariable(dfg, lb), varOrder) + +# # Also look at parentmodule +# usrtyp = convertStructType(PT) +# fulltype = DFG.FunctionNodeData{T{usrtyp}} +# factordata = reconstFactorData(dfg, varOrder, fulltype, packeddata) +# return factordata +# end + +# function _packSolverData(f::FactorCompute, fnctype::AbstractObservation) +# # +# error("_packSolverData is deprecated, use seperate packing of observation #TODO") +# packtype = convertPackedType(fnctype) +# try +# packed = convert(PackedFunctionNodeData{packtype}, getSolverData(f)) #TODO getSolverData +# packedJson = packed +# return packedJson +# catch ex +# io = IOBuffer() +# showerror(io, ex, catch_backtrace()) +# err = String(take!(io)) +# msg = "Error while packing '$(f.label)' as '$fnctype', please check the unpacking/packing converters for this factor - \r\n$err" +# error(msg) +# end +# end + +# const PackedFunctionNodeData{T} = +# GenericFunctionNodeData{T} where {T <: AbstractPackedObservation} +# function PackedFunctionNodeData(args...; kw...) +# error("PackedFunctionNodeData is obsolete") +# return PackedFunctionNodeData{typeof(args[4])}(args...; kw...) +# end + +# const FunctionNodeData{T} = +# GenericFunctionNodeData{T} where {T <: Union{<:AbstractObservation, <:FactorCache}} +# FunctionNodeData(args...; kw...) = FunctionNodeData{typeof(args[4])}(args...; kw...) + +# # this is the GenericFunctionNodeData for packed types +# #TODO deprecate FactorData in favor of FactorState (with no more distinction between packed and compute) +# const FactorData = PackedFunctionNodeData{AbstractPackedObservation} + +# function FactorCompute( +# label::Symbol, +# variableOrder::Union{Vector{Symbol}, Tuple}, +# solverData::GenericFunctionNodeData; +# tags::Set{Symbol} = Set{Symbol}(), +# timestamp::Union{DateTime, ZonedDateTime} = now(localzone()), +# solvable::Int = 1, +# nstime::Nanosecond = Nanosecond(0), +# id::Union{UUID, Nothing} = nothing, +# smallData::Dict{Symbol, MetadataTypes} = Dict{Symbol, MetadataTypes}(), +# ) +# Base.depwarn( +# "`FactorCompute` constructor with `GenericFunctionNodeData` is deprecated. observation, state, and solvercache should be provided explicitly.", +# :FactorCompute, +# ) +# observation = getFactorType(solverData) +# state = FactorState( +# solverData.eliminated, +# solverData.potentialused, +# solverData.multihypo, +# solverData.certainhypo, +# solverData.nullhypo, +# solverData.solveInProgress, +# solverData.inflation, +# ) + +# if solverData.fnc isa FactorCache +# solvercache = solverData.fnc +# else +# solvercache = nothing +# end + +# return FactorCompute( +# label, +# Tuple(variableOrder), +# observation, +# state, +# solvercache; +# id, +# timestamp, +# nstime, +# tags, +# smallData, +# solvable, +# ) +# end + +# # Deprecated check usefull? # packedFnc = fncStringToData(factor.fnctype, factor.data) +# # Deprecated check usefull? # decodeType = getFactorOperationalMemoryType(dfg) +# # Deprecated check usefull? # fullFactorData = decodePackedType(dfg, factor._variableOrderSymbols, decodeType, packedFnc) +# function fncStringToData(args...; kwargs...) +# @warn "fncStringToData is obsolete, called with" args kwargs +# return error("fncStringToData is obsolete.") +# end + +# #TODO make sure getFactorOperationalMemoryType is obsolete +# function getFactorOperationalMemoryType(dummy) +# return error( +# "Please extend your workspace with function getFactorOperationalMemoryType(<:AbstractParams) for your usecase, e.g. IncrementalInference uses `CommonConvWrapper <: FactorCache`", +# ) +# end +# function getFactorOperationalMemoryType(dfg::AbstractDFG) +# return getFactorOperationalMemoryType(getSolverParams(dfg)) +# end + +# function typeModuleName(variableType::StateType) +# Base.depwarn("typeModuleName is obsolete", :typeModuleName) +# io = IOBuffer() +# ioc = IOContext(io, :module => DistributedFactorGraphs) +# show(ioc, typeof(variableType)) +# return String(take!(io)) +# end + +# typeModuleName(varT::Type{<:StateType}) = typeModuleName(varT()) diff --git a/src/DistributedFactorGraphs.jl b/src/DistributedFactorGraphs.jl index 4f82fc83..1632ff8c 100644 --- a/src/DistributedFactorGraphs.jl +++ b/src/DistributedFactorGraphs.jl @@ -20,8 +20,7 @@ using DocStringExtensions using Dates using Random using TimeZones -using JSON3 -using StructTypes +using JSON using LinearAlgebra using SparseArrays using UUIDs @@ -488,18 +487,18 @@ const unstable_functions::Vector{Symbol} = [ :NoSolverParams, :AbstractParams, # Deprecated in v0.27 - :AbstractFactor, - :AbstractPackedFactor, - :FactorOperationalMemory, - :VariableNodeData, - :updateVariableSolverData!, - :getSolverData, - :setSolverData!, - :reconstFactorData, - :_packSolverData, - :GenericFunctionNodeData, - :PackedFunctionNodeData, - :FunctionNodeData, + # :AbstractFactor, + # :AbstractPackedFactor, + # :FactorOperationalMemory, + # :VariableNodeData, + # :updateVariableSolverData!, + # :getSolverData, + # :setSolverData!, + # :reconstFactorData, + # :_packSolverData, + # :GenericFunctionNodeData, + # :PackedFunctionNodeData, + # :FunctionNodeData, ] macro usingDFG(unstable = false) diff --git a/src/FileDFG/services/FileDFG.jl b/src/FileDFG/services/FileDFG.jl index fbffbfaa..16c3dd75 100644 --- a/src/FileDFG/services/FileDFG.jl +++ b/src/FileDFG/services/FileDFG.jl @@ -41,19 +41,19 @@ function saveDFG(folder::AbstractString, dfg::AbstractDFG; saveMetadata::Bool = # Variables @showprogress "saving variables" for v in variables vPacked = packVariable(v) - JSON3.write("$varFolder/$(v.label).json", vPacked) + JSON.json("$varFolder/$(v.label).json", vPacked) end # Factors @showprogress "saving factors" for f in factors fPacked = packFactor(f) - JSON3.write("$factorFolder/$(f.label).json", fPacked) + JSON.json("$factorFolder/$(f.label).json", fPacked) end #GraphsDFG metadata if saveMetadata @assert isa(dfg, GraphsDFG) "only metadata for GraphsDFG are supported" @info "saving dfg metadata" fgPacked = GraphsDFGs.packDFGMetadata(dfg) - JSON3.write("$savepath/dfg.json", fgPacked) + JSON.json("$savepath/dfg.json", fgPacked) end savedir = dirname(savepath) # is this a path of just local name? #344 -- workaround with unique names @@ -139,7 +139,7 @@ function loadDFG!( @assert isa(dfgLoadInto, GraphsDFG) "Only GraphsDFG metadata are supported" @info "loading dfg metadata" jstr = read("$folder/dfg.json", String) - fgPacked = JSON3.read(jstr, GraphsDFGs.PackedGraphsDFG) + fgPacked = JSON.parse(jstr, GraphsDFGs.PackedGraphsDFG) GraphsDFGs.unpackDFGMetadata!(dfgLoadInto, fgPacked) end @@ -165,7 +165,7 @@ function loadDFG!( # type instability on `variables` as either `::Vector{Variable}` or `::Vector{VariableCompute{<:}}` (vector of abstract) variables = @showprogress 1 "loading variables" asyncmap(varFiles) do varFile jstr = read("$varFolder/$varFile", String) - packedvar = JSON3.read(jstr, VariableDFG) + packedvar = JSON.parse(jstr, VariableDFG) v = usePackedVariable ? packedvar : unpackVariable(packedvar) return addVariable!(dfgLoadInto, v) end @@ -178,7 +178,7 @@ function loadDFG!( # `factors` is not type stable `::Vector{Factor}` or `::Vector{FactorCompute{<:}}` (vector of abstract) factors = @showprogress 1 "loading factors" asyncmap(factorFiles) do factorFile jstr = read("$factorFolder/$factorFile", String) - packedfact = JSON3.read(jstr, FactorDFG) + packedfact = JSON.parse(jstr, FactorDFG) f = usePackedFactor ? packedfact : unpackFactor(packedfact) return addFactor!(dfgLoadInto, f) end @@ -231,12 +231,12 @@ function loadDFG(file::AbstractString) #TODO deprecate old format, v0.28 local fgPacked try - fgPacked = JSON3.read(jstr, GraphsDFGs.PackedGraphsDFG) + fgPacked = JSON.parse(jstr, GraphsDFGs.PackedGraphsDFG) catch e if e isa MethodError @warn "Deprecated serialization: Failed to read DFG metadata. Attempting to load using the old format. Error:" e fgPacked = - GraphsDFGs.PackedGraphsDFG(JSON3.read(jstr, GraphsDFGs._OldPackedGraphsDFG)) + GraphsDFGs.PackedGraphsDFG(JSON.parse(jstr, GraphsDFGs._OldPackedGraphsDFG)) else rethrow(e) end diff --git a/src/GraphsDFG/GraphsDFG.jl b/src/GraphsDFG/GraphsDFG.jl index 8e11b40a..fa66142b 100644 --- a/src/GraphsDFG/GraphsDFG.jl +++ b/src/GraphsDFG/GraphsDFG.jl @@ -3,9 +3,8 @@ module GraphsDFGs using Graphs using DocStringExtensions using UUIDs -using JSON3 +using JSON using OrderedCollections -using StructTypes using ...DistributedFactorGraphs using ...DistributedFactorGraphs: diff --git a/src/GraphsDFG/services/GraphsDFGSerialization.jl b/src/GraphsDFG/services/GraphsDFGSerialization.jl index 87e09352..f0fa4f8a 100644 --- a/src/GraphsDFG/services/GraphsDFGSerialization.jl +++ b/src/GraphsDFG/services/GraphsDFGSerialization.jl @@ -14,19 +14,6 @@ using InteractiveUtils graphBlobEntries::OrderedDict{Symbol, Blobentry} agent::Agent end -StructTypes.StructType(::Type{_OldPackedGraphsDFG}) = StructTypes.AbstractType() -function StructTypes.StructType( - ::Type{_OldPackedGraphsDFG{T}}, -) where {T <: AbstractDFGParams} - return StructTypes.Struct() -end -StructTypes.subtypekey(::Type{_OldPackedGraphsDFG}) = :solverParams_type -#TODO look at StructTypes.@register_struct_subtype when new StructTypes.jl is tagged (for type field) - -function StructTypes.subtypes(::Type{_OldPackedGraphsDFG}) - subs = subtypes(AbstractDFGParams) - return NamedTuple(map(s -> nameof(s) => _OldPackedGraphsDFG{s}, subs)) -end @kwdef struct PackedGraphsDFG{T <: AbstractDFGParams} addHistory::Vector{Symbol} @@ -59,17 +46,24 @@ function PackedGraphsDFG(old::_OldPackedGraphsDFG) ) end -StructTypes.StructType(::Type{PackedGraphsDFG}) = StructTypes.AbstractType() -function StructTypes.StructType(::Type{PackedGraphsDFG{T}}) where {T <: AbstractDFGParams} - return StructTypes.Struct() +function getPackedGraphsDFGSubtype(s) + subs = subtypes(AbstractDFGParams) + idx = findfirst(x -> nameof(x) == Symbol(s.solverParams_type[]), subs) + isnothing(idx) && throw( + DFG.SerializationError("Unknown solver parameters type `$(s.solverParams_type[])`"), + ) + return PackedGraphsDFG{subs[idx]} end -StructTypes.subtypekey(::Type{PackedGraphsDFG}) = :solverParams_type -#TODO look at StructTypes.@register_struct_subtype when new StructTypes.jl is tagged (for type field) - -function StructTypes.subtypes(::Type{PackedGraphsDFG}) +function getOldPackedGraphsDFGSubtype(s) subs = subtypes(AbstractDFGParams) - return NamedTuple(map(s -> nameof(s) => PackedGraphsDFG{s}, subs)) + idx = findfirst(x -> nameof(x) == Symbol(s.solverParams_type[]), subs) + isnothing(idx) && throw( + DFG.SerializationError("Unknown solver parameters type `$(s.solverParams_type[])`"), + ) + return _OldPackedGraphsDFG{subs[idx]} end +JSON.@choosetype PackedGraphsDFG getPackedGraphsDFGSubtype +JSON.@choosetype _OldPackedGraphsDFG getOldPackedGraphsDFGSubtype function getTypeDFGVariables( fg::GraphsDFG{<:AbstractDFGParams, T, <:AbstractGraphFactor}, diff --git a/src/entities/AbstractDFG.jl b/src/entities/AbstractDFG.jl index 1609d384..313113ba 100644 --- a/src/entities/AbstractDFG.jl +++ b/src/entities/AbstractDFG.jl @@ -52,8 +52,6 @@ Empty structure for solver parameters. d::Int = 0#FIXME JSON3.jl error MethodError: no method matching read(::StructTypes.SingletonType, ... end -StructTypes.StructType(::NoSolverParams) = StructTypes.Struct() - """ Types valid for small data. """ diff --git a/src/entities/DFGFactor.jl b/src/entities/DFGFactor.jl index 4813c77f..b5a371d4 100644 --- a/src/entities/DFGFactor.jl +++ b/src/entities/DFGFactor.jl @@ -56,7 +56,7 @@ end The Factor information packed in a way that accomdates multi-lang using json. """ -Base.@kwdef struct FactorDFG <: AbstractGraphFactor +StructUtils.@kwarg struct FactorDFG <: AbstractGraphFactor id::Union{UUID, Nothing} = nothing label::Symbol tags::Set{Symbol} @@ -65,7 +65,6 @@ Base.@kwdef struct FactorDFG <: AbstractGraphFactor nstime::String fnctype::String solvable::Int - data::Union{Nothing, String} = nothing #TODO v0.27 deprecate data completely, left as a bridge to old serialization structure metadata::String _version::VersionNumber = _getDFGVersion() state::FactorState @@ -78,56 +77,6 @@ end # createdTimestamp::DateTime # lastUpdatedTimestamp::DateTime -StructTypes.StructType(::Type{FactorDFG}) = StructTypes.UnorderedStruct() -StructTypes.idproperty(::Type{FactorDFG}) = :id -StructTypes.omitempties(::Type{FactorDFG}) = (:id, :data) - -#TODO deprecate, added in v0.27 as a bridge to new serialization structure -function FactorDFG( - id::Union{UUID, Nothing}, - label::Symbol, - tags::Set{Symbol}, - _variableOrderSymbols::Vector{Symbol}, - timestamp::ZonedDateTime, - nstime::String, - fnctype::String, - solvable::Int, - data::Union{Nothing, String}, - metadata::String, - _version, - state::Union{Nothing, FactorState} = nothing, - observJSON::Union{Nothing, String} = nothing, -) - if isnothing(state) || isnothing(observJSON) - fd = JSON3.read(data) - state = FactorState( - fd.eliminated, - fd.potentialused, - fd.multihypo, - fd.certainhypo, - fd.nullhypo, - fd.solveInProgress, - fd.inflation, - ) - observJSON = JSON3.write(fd.fnc) - end - return FactorDFG( - id, - label, - tags, - _variableOrderSymbols, - timestamp, - nstime, - fnctype, - solvable, - nothing, #TODO v0.27 deprecate data completely - metadata, - _version, - state, - observJSON, - ) -end - # Packed Factor constructor function assembleFactorName(xisyms::Union{Vector{String}, Vector{Symbol}}) return Symbol(xisyms..., "_f", randstring(4)) @@ -163,10 +112,9 @@ function FactorDFG( nstime = string(nstime), fnctype, solvable, - metadata = base64encode(JSON3.write(metadata)), + metadata = base64encode(JSON.json(metadata)), state, - observJSON = JSON3.write(fnc), - data = "", #TODO v0.27 deprecate data completely + observJSON = JSON.json(fnc), ) return factor @@ -272,11 +220,6 @@ end function Base.getproperty(x::FactorCompute, f::Symbol) if f == :solvable getfield(x, f)[] - elseif f == :solverData - # TODO remove, deprecated in v0.27 - error( - "`solverData` is obsolete in `FactorCompute`. Use `getObservation`, `getFactorState` or `getCache` instead.", - ) elseif f == :_variableOrderSymbols [getfield(x, f)...] else @@ -287,10 +230,6 @@ end function Base.setproperty!(x::FactorCompute, f::Symbol, val) if f == :solvable getfield(x, f)[] = val - elseif f == :solverData - error( - "`solverData` is obsolete in `FactorCompute`. Use `Observation`, `State` or `Cache` instead.", - ) else setfield!(x, f, val) end diff --git a/src/entities/DFGVariable.jl b/src/entities/DFGVariable.jl index dfd3b98b..9c8338ce 100644 --- a/src/entities/DFGVariable.jl +++ b/src/entities/DFGVariable.jl @@ -139,10 +139,6 @@ getLabel(packedstate::PackedState) = packedstate.solveKey # createdTimestamp::DateTime#! # lastUpdatedTimestamp::DateTime#! -StructTypes.StructType(::Type{PackedState}) = StructTypes.UnorderedStruct() -StructTypes.idproperty(::Type{PackedState}) = :id -StructTypes.omitempties(::Type{PackedState}) = (:id,) - ##============================================================================== ## PointParametricEst ##============================================================================== @@ -173,12 +169,6 @@ Base.@kwdef struct MeanMaxPPE <: AbstractPointParametricEst lastUpdatedTimestamp::Union{ZonedDateTime, Nothing} = nothing end -StructTypes.StructType(::Type{MeanMaxPPE}) = StructTypes.UnorderedStruct() -StructTypes.idproperty(::Type{MeanMaxPPE}) = :id -function StructTypes.omitempties(::Type{MeanMaxPPE}) - return (:id, :createdTimestamp, :lastUpdatedTimestamp) -end - ##------------------------------------------------------------------------------ ## Constructors @@ -261,7 +251,7 @@ function VariableDFG( nstime = string(nanosecondtime), solvable, tags, - metadata = base64encode(JSON3.write(smalldata)), + metadata = base64encode(JSON.json(smalldata)), timestamp, kwargs..., ) @@ -269,12 +259,8 @@ function VariableDFG( return pacvar end -StructTypes.StructType(::Type{VariableDFG}) = StructTypes.UnorderedStruct() -StructTypes.idproperty(::Type{VariableDFG}) = :id -StructTypes.omitempties(::Type{VariableDFG}) = (:id,) - function getMetadata(v::VariableDFG) - return JSON3.read(base64decode(v.metadata), Dict{Symbol, MetadataTypes}) + return JSON.parse(base64decode(v.metadata), Dict{Symbol, MetadataTypes}) end ##------------------------------------------------------------------------------ @@ -387,7 +373,7 @@ Summary variable structure for a DistributedFactorGraph variable. Fields: $(TYPEDFIELDS) """ -Base.@kwdef struct VariableSummary <: AbstractGraphVariable +@tags struct VariableSummary <: AbstractGraphVariable """The ID for the variable""" id::Union{UUID, Nothing} """Variable label, e.g. :x1. @@ -404,7 +390,7 @@ Base.@kwdef struct VariableSummary <: AbstractGraphVariable ppeDict::Dict{Symbol, <:AbstractPointParametricEst} """Symbol for the variableType for the underlying variable. Accessor: [`getVariableType`](@ref)""" - variableTypeName::Symbol + variableTypeName::Symbol & (json = (name = "variableType",)) # TODO check from StructTypes.names(::Type{VariableSummary}) = ((:variableTypeName, :variableType),) """Dictionary of large data associated with this variable. Accessors: [`addBlobentry!`](@ref), [`getBlobentry`](@ref), [`mergeBlobentry!`](@ref), and [`deleteBlobentry!`](@ref)""" dataDict::Dict{Symbol, Blobentry} @@ -422,8 +408,6 @@ function VariableSummary(id, label, timestamp, tags, ::Nothing, variableTypeName ) end -StructTypes.names(::Type{VariableSummary}) = ((:variableTypeName, :variableType),) - ##------------------------------------------------------------------------------ ## VariableSkeleton.jl ##------------------------------------------------------------------------------ diff --git a/src/services/DFGFactor.jl b/src/services/DFGFactor.jl index 8577e9b6..be7a19d6 100644 --- a/src/services/DFGFactor.jl +++ b/src/services/DFGFactor.jl @@ -3,7 +3,7 @@ ##============================================================================== function getMetadata(f::FactorDFG) - return JSON3.read(base64decode(f.metadata), Dict{Symbol, MetadataTypes}) + return JSON.parse(base64decode(f.metadata), Dict{Symbol, MetadataTypes}) end ## COMMON @@ -27,7 +27,7 @@ that contains the information about the factor, such as the measurement, prior, """ getObservation(f::FactorCompute) = f.observation function getObservation(f::FactorDFG) - #FIXME completely refactor to not need getTypeFromSerializationModule and just use StructTypes + #FIXME completely refactor to not need getTypeFromSerializationModule and just use StructUtils if contains(f.fnctype, ".") # packed factor contains a module name, just extracting type and ignoring module @@ -37,7 +37,7 @@ function getObservation(f::FactorDFG) end packtype = DFG.getTypeFromSerializationModule("Packed" * fnctype) - return packtype(; JSON3.read(f.observJSON)...) + return packtype(; JSON.parse(f.observJSON)...) # return packtype(JSON3.read(f.observJSON)) end @@ -95,10 +95,8 @@ function unpack end function packDistribution end function unpackDistribution end -StructTypes.StructType(::Type{<:PackedBelief}) = StructTypes.UnorderedStruct() - #TODO remove, rather use StructTypes.jl properly -function Base.convert(::Type{<:PackedBelief}, nt::Union{NamedTuple, JSON3.Object}) +function Base.convert(::Type{<:PackedBelief}, nt::Union{NamedTuple, JSON.Object}) distrType = getTypeFromSerializationModule(nt._type) return distrType(; nt...) end diff --git a/src/services/Serialization.jl b/src/services/Serialization.jl index f2598ecf..d6251508 100644 --- a/src/services/Serialization.jl +++ b/src/services/Serialization.jl @@ -213,7 +213,7 @@ function packVariable( tags = collect(v.tags), # Symbol.() ppes = collect(values(v.ppeDict)), solverData = packState.(collect(values(v.solverDataDict))), - metadata = base64encode(JSON3.write(v.smallData)), + metadata = base64encode(JSON.json(v.smallData)), solvable = v.solvable, variableType = stringVariableType(DFG.getVariableType(v)), blobEntries = collect(values(v.dataDict)), @@ -251,7 +251,7 @@ function unpackVariable(variable::VariableDFG; skipVersionCheck::Bool = false) dataDict = Dict{Symbol, Blobentry}( map(de -> de.label, variable.blobEntries) .=> variable.blobEntries, ) - metadata = JSON3.read(base64decode(variable.metadata), Dict{Symbol, DFG.MetadataTypes}) + metadata = JSON.parse(base64decode(variable.metadata), Dict{Symbol, DFG.MetadataTypes}) return VariableCompute( variable.label, @@ -293,11 +293,11 @@ function packFactor(f::FactorCompute) # fnctype = String(_getname(getObservation(f))), fnctype, solvable = getSolvable(f), - metadata = base64encode(JSON3.write(f.smallData)), + metadata = base64encode(JSON.json(f.smallData)), # Pack the node data _version = _getDFGVersion(), state = f.state, - observJSON = JSON3.write(packObservation(f)), + observJSON = JSON.json(packObservation(f)), ) return props end @@ -314,7 +314,7 @@ function unpackObservation(factor::FactorDFG) Falling back to deprecated convert method.""", :unpackObservation, ) - #FIXME completely refactor to not need getTypeFromSerializationModule and just use StructTypes + #FIXME completely refactor to not need getTypeFromSerializationModule and just use StructUtils #TODO change to unpack: observ = unpack(observpacked) # currently the observation type is stored in the factor and this complicates unpacking of seperate observations observpacked = getObservation(factor) diff --git a/test/data/0_25_0.tar.gz b/test/data/archive/0_25_0.tar.gz similarity index 100% rename from test/data/0_25_0.tar.gz rename to test/data/archive/0_25_0.tar.gz diff --git a/test/runtests.jl b/test/runtests.jl index 3319bd72..6a7adf04 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -16,158 +16,179 @@ DFG.@usingDFG true # using Logging # logger = SimpleLogger(stdout, Logging.Debug) # global_logger(logger) +@testset "DFG Tests" begin + include("test_defVariable.jl") -include("test_defVariable.jl") + include("testBlocks.jl") -include("testBlocks.jl") - -@testset "Test generated ==" begin - include("compareTests.jl") -end - -@testset "Testing GraphsDFG.FactorGraphs functions" begin - include("FactorGraphsTests.jl") -end - -apis = [GraphsDFG] - -for api in apis - @testset "Testing Driver: $(api)" begin - @info "Testing Driver: $(api)" - global testDFGAPI = api - include("interfaceTests.jl") + @testset "Test generated ==" begin + include("compareTests.jl") end -end - -# Test special cases -@testset "Plotting Tests" begin - include("plottingTest.jl") -end - -@testset "Data Store Tests" begin - include("consol_DataEntryBlobTests.jl") -end -@testset "GraphsDFG subtype tests" begin - for type in [ - (var = VariableSummary, fac = FactorSummary), - (var = VariableSkeleton, fac = FactorSkeleton), - ] - @testset "$(type.var) and $(type.fac) tests" begin - @info "Testing $(type.var) and $(type.fac)" - global VARTYPE = type.var - global FACTYPE = type.fac - include("GraphsDFGSummaryTypes.jl") - end + @testset "Testing GraphsDFG.FactorGraphs functions" begin + include("FactorGraphsTests.jl") end -end -if get(ENV, "IIF_TEST", "true") == "true" - - # Switch to our upstream test branch. - #FIXME This is a temporary fix to use the develop branch of AMP. - Pkg.add(PackageSpec(; name = "ApproxManifoldProducts", rev = "develop")) - #FIXME This is a temporary fix to use the develop branch of IIF. - # Pkg.add(PackageSpec(; name = "IncrementalInference", rev = "upstream/dfg_integration_test")) - # Pkg.add(PackageSpec(; name = "IncrementalInference", rev = "develop")) - Pkg.add( - PackageSpec(; - url = "https://github.com/JuliaRobotics/IncrementalInference.jl.git", - subdir = "IncrementalInferenceTypes", - rev = "develop", - ), - ) - Pkg.add( - PackageSpec(; - url = "https://github.com/JuliaRobotics/IncrementalInference.jl.git", - subdir = "IncrementalInference", - rev = "develop", - ), - ) - @info "------------------------------------------------------------------------" - @info "These tests are using IncrementalInference to do additional driver tests" - @info "------------------------------------------------------------------------" - - using IncrementalInference - - apis = Vector{AbstractDFG}() - push!(apis, GraphsDFG(; solverParams = SolverParams())) + apis = [GraphsDFG] for api in apis - @testset "Testing Driver: $(typeof(api))" begin + @testset "Testing Driver: $(api)" begin @info "Testing Driver: $(api)" - global dfg = deepcopy(api) - include("iifInterfaceTests.jl") + global testDFGAPI = api + include("interfaceTests.jl") end + end - @testset "FileDFG Testing Driver: $(typeof(api))" begin - @info "FileDFG Testing Driver: $(typeof(api))" - global dfg = deepcopy(api) - include("fileDFGTests.jl") - end + # Test special cases + @testset "Plotting Tests" begin + include("plottingTest.jl") end - @testset "IIF Compare Tests" begin - #run a copy of compare tests from IIF - include("iifCompareTests.jl") + @testset "Data Store Tests" begin + include("consol_DataEntryBlobTests.jl") end - # Simple graph solving test - @testset "Simple graph solving test" begin - # This is just to validate we're not going to blow up downstream. - apis = [ - # GraphsDFG{SolverParams}(), - GraphsDFG(; solverParams = SolverParams()), + @testset "GraphsDFG subtype tests" begin + for type in [ + (var = VariableSummary, fac = FactorSummary), + (var = VariableSkeleton, fac = FactorSkeleton), ] + @testset "$(type.var) and $(type.fac) tests" begin + @info "Testing $(type.var) and $(type.fac)" + global VARTYPE = type.var + global FACTYPE = type.fac + include("GraphsDFGSummaryTypes.jl") + end + end + end + + if get(ENV, "IIF_TEST", "false") == "true" + + # Switch to our upstream test branch. + #FIXME This is a temporary fix to use the develop branch of AMP. + Pkg.add(PackageSpec(; name = "ApproxManifoldProducts", rev = "develop")) + #FIXME This is a temporary fix to use the develop branch of IIF. + # Pkg.add(PackageSpec(; name = "IncrementalInference", rev = "upstream/dfg_integration_test")) + # Pkg.add(PackageSpec(; name = "IncrementalInference", rev = "develop")) + if get(ENV, "IIF_TEST_DEV", "true") == "true" + Pkg.add( + PackageSpec(; + url = "https://github.com/JuliaRobotics/IncrementalInference.jl.git", + subdir = "IncrementalInferenceTypes", + rev = "develop", + ), + ) + Pkg.add( + PackageSpec(; + url = "https://github.com/JuliaRobotics/IncrementalInference.jl.git", + subdir = "IncrementalInference", + rev = "develop", + ), + ) + else + Pkg.develop(; + path = joinpath( + DEPOT_PATH[1], + "dev", + "IncrementalInference", + "IncrementalInferenceTypes", + ), + ) + Pkg.develop(; + path = joinpath( + DEPOT_PATH[1], + "dev", + "IncrementalInference", + "IncrementalInference", + ), + ) + end + + @info "------------------------------------------------------------------------" + @info "These tests are using IncrementalInference to do additional driver tests" + @info "------------------------------------------------------------------------" + + using IncrementalInference + + apis = Vector{AbstractDFG}() + push!(apis, GraphsDFG(; solverParams = SolverParams())) + for api in apis - @info "Running simple solver test: $(typeof(api))" - global dfg = deepcopy(api) - include("solveTest.jl") + @testset "Testing Driver: $(typeof(api))" begin + @info "Testing Driver: $(api)" + global dfg = deepcopy(api) + include("iifInterfaceTests.jl") + end + + @testset "FileDFG Testing Driver: $(typeof(api))" begin + @info "FileDFG Testing Driver: $(typeof(api))" + global dfg = deepcopy(api) + include("fileDFGTests.jl") + end end + + @testset "IIF Compare Tests" begin + #run a copy of compare tests from IIF + include("iifCompareTests.jl") + end + + # Simple graph solving test + @testset "Simple graph solving test" begin + # This is just to validate we're not going to blow up downstream. + apis = [ + # GraphsDFG{SolverParams}(), + GraphsDFG(; solverParams = SolverParams()), + ] + for api in apis + @info "Running simple solver test: $(typeof(api))" + global dfg = deepcopy(api) + include("solveTest.jl") + end + end + else + @warn "Skipping IncrementalInference driver tests" end -else - @warn "Skipping IncrementalInference driver tests" -end -struct NotImplementedDFG{V, T} <: AbstractDFG{V, T} end + struct NotImplementedDFG{V, T} <: AbstractDFG{V, T} end -@testset "No Interface tests" begin - dfg = NotImplementedDFG{VariableDFG, FactorDFG}() - v1 = VariableSkeleton(:v1) - f1 = FactorSkeleton(:f1, [:v1]) + @testset "No Interface tests" begin + dfg = NotImplementedDFG{VariableDFG, FactorDFG}() + v1 = VariableSkeleton(:v1) + f1 = FactorSkeleton(:f1, [:v1]) - @test_throws MethodError exists(dfg, v1) - @test_throws MethodError exists(dfg, f1) + @test_throws MethodError exists(dfg, v1) + @test_throws MethodError exists(dfg, f1) - @test_throws MethodError exists(dfg, :s) - @test_throws MethodError addVariable!(dfg, v1) + @test_throws MethodError exists(dfg, :s) + @test_throws MethodError addVariable!(dfg, v1) - @test_throws MethodError getVariable(dfg, :a) - @test_throws MethodError getFactor(dfg, :a) - @test_throws MethodError mergeVariable!(dfg, v1) - @test_throws MethodError mergeFactor!(dfg, f1) + @test_throws MethodError getVariable(dfg, :a) + @test_throws MethodError getFactor(dfg, :a) + @test_throws MethodError mergeVariable!(dfg, v1) + @test_throws MethodError mergeFactor!(dfg, f1) - @test_throws MethodError deleteVariable!(dfg, :a) - @test_throws MethodError deleteFactor!(dfg, :a) - @test_throws MethodError getVariables(dfg) - @test_throws MethodError getFactors(dfg) - @test_throws MethodError isConnected(dfg) - @test_throws MethodError listNeighbors(dfg, v1) - @test_throws MethodError listNeighbors(dfg, :a) + @test_throws MethodError deleteVariable!(dfg, :a) + @test_throws MethodError deleteFactor!(dfg, :a) + @test_throws MethodError getVariables(dfg) + @test_throws MethodError getFactors(dfg) + @test_throws MethodError isConnected(dfg) + @test_throws MethodError listNeighbors(dfg, v1) + @test_throws MethodError listNeighbors(dfg, :a) - @test_throws MethodError DFG._getDuplicatedEmptyDFG(dfg) + @test_throws MethodError DFG._getDuplicatedEmptyDFG(dfg) - @test_throws MethodError isVariable(dfg, :a) - @test_throws MethodError isFactor(dfg, :a) -end + @test_throws MethodError isVariable(dfg, :a) + @test_throws MethodError isFactor(dfg, :a) + end -@testset "Testing Code Quality with Aqua" begin - Aqua.test_ambiguities([DistributedFactorGraphs]) - Aqua.test_unbound_args(DistributedFactorGraphs) - Aqua.test_undefined_exports(DistributedFactorGraphs) - Aqua.test_piracies(DistributedFactorGraphs) - Aqua.test_project_extras(DistributedFactorGraphs) - Aqua.test_stale_deps(DistributedFactorGraphs; ignore = [:Colors]) - Aqua.test_deps_compat(DistributedFactorGraphs) - # Aqua.test_project_toml_formatting(DistributedFactorGraphs) # deprecated in Aqua.jl v0.8 + @testset "Testing Code Quality with Aqua" begin + Aqua.test_ambiguities([DistributedFactorGraphs]) + Aqua.test_unbound_args(DistributedFactorGraphs) + Aqua.test_undefined_exports(DistributedFactorGraphs) + Aqua.test_piracies(DistributedFactorGraphs) + Aqua.test_project_extras(DistributedFactorGraphs) + Aqua.test_stale_deps(DistributedFactorGraphs; ignore = [:Colors]) + Aqua.test_deps_compat(DistributedFactorGraphs) + # Aqua.test_project_toml_formatting(DistributedFactorGraphs) # deprecated in Aqua.jl v0.8 + end end