Refactoring Factors: data -> state, observation and cache

NEWS.md

@@ -6,6 +6,12 @@ Listing news on any major breaking changes in DFG.  For regular changes, see int
 - Deprecate `updateFactor!` for `mergeFactor!`, note `merege` returns number of nodes updated/added.
 - Rename BlobEntry to Blobentry, see #1123.
 - Rename BlobStore to Blobstore, see #1124.
+- Refactor the Factor solver data structure, see #1127:
+    - Deprecated GenericFunctionNodeData, PackedFunctionNodeData, FunctionNodeData, and all functions related factor.solverData.
+    - Replaced by 3 seperete types: Observation, State, and Cache
+    - This is used internally be the solver and should not affect the average user of DFG.
+    - Rename FactorOperationalMemory -> FactorSolverCache
+    - Rename AbstractFactor -> AbstractFactorObservation (keeping both around)
 
 # v0.26
 - Graph structure plotting now uses GraphMakie.jl instead of GraphPlot.jl. Update by replacing `using GraphPlot` with `using GraphMakie`.

Project.toml

@@ -1,6 +1,6 @@
 name = "DistributedFactorGraphs"
 uuid = "b5cc3c7e-6572-11e9-2517-99fb8daf2f04"
-version = "0.26.0"
+version = "0.27.0"
 
 [deps]
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
@@ -19,6 +19,7 @@ ManifoldsBase = "3362f125-f0bb-47a3-aa74-596ffd7ef2fb"
 OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 RecursiveArrayTools = "731186ca-8d62-57ce-b412-fbd966d074cd"
 Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
 SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
@@ -61,6 +62,7 @@ ManifoldsBase = "0.14, 0.15, 1"
 OrderedCollections = "1.4"
 Pkg = "1.4, 1.5"
 ProgressMeter = "1"
+Random = "1.10"
 RecursiveArrayTools = "2, 3"
 Reexport = "1"
 SHA = "0.7, 1"

src/Common.jl

@@ -3,10 +3,10 @@
 _getmodule(t::T) where {T} = T.name.module
 _getname(t::T) where {T} = T.name.name
 
-function convertPackedType(t::Union{T, Type{T}}) where {T <: AbstractFactor}
+function convertPackedType(t::Union{T, Type{T}}) where {T <: AbstractFactorObservation}
     return getfield(_getmodule(t), Symbol("Packed$(_getname(t))"))
 end
-function convertStructType(::Type{PT}) where {PT <: AbstractPackedFactor}
+function convertStructType(::Type{PT}) where {PT <: AbstractPackedFactorObservation}
     # see #668 for expanded reasoning.  PT may be ::UnionAll if the type is of template type.
     ptt = PT isa DataType ? PT.name.name : PT
     moduleName = PT isa DataType ? PT.name.module : Main

src/Deprecated.jl

@@ -1,6 +1,14 @@
 ## ================================================================================
 ## Deprecated in v0.27
 ##=================================================================================
+export AbstractFactor
+const AbstractFactor = AbstractFactorObservation
+
+export AbstractPackedFactor
+const AbstractPackedFactor = AbstractPackedFactorObservation
+
+export FactorOperationalMemory
+const FactorOperationalMemory = FactorSolverCache
 
 @deprecate getNeighborhood(args...; kwargs...) listNeighborhood(args...; kwargs...)
 @deprecate addBlob!(store::AbstractBlobstore, blobId::UUID, data, ::String) addBlob!(
@@ -73,6 +81,10 @@
     fields::Vector{Symbol} = Symbol[];
     warn_if_absent::Bool = true,
 )
+    Base.depwarn(
+        "updateVariableSolverData! is deprecated, use mergeVariableState! or copytoVariableState! instead",
+        :updateVariableSolverData!,
+    )
     #This is basically just setSolverData
     var = getVariable(dfg, variablekey)
     warn_if_absent &&
@@ -190,6 +202,194 @@
     end
 end
 
+## factor refactor deprecations
+Base.@kwdef mutable struct GenericFunctionNodeData{
+    T <: Union{
+        <:AbstractPackedFactorObservation,
+        <:AbstractFactorObservation,
+        <:FactorSolverCache,
+    },
+}
+    eliminated::Bool = false
+    potentialused::Bool = false
+    edgeIDs::Vector{Int} = Int[]
+    fnc::T
+    multihypo::Vector{Float64} = Float64[] # TODO re-evaluate after refactoring w #477
+    certainhypo::Vector{Int} = Int[]
+    nullhypo::Float64 = 0.0
+    solveInProgress::Int = 0
+    inflation::Float64 = 0.0
+end
+
+function FactorCompute(
+    label::Symbol,
+    timestamp::Union{DateTime, ZonedDateTime},
+    nstime::Nanosecond,
+    tags::Set{Symbol},
+    solverData::GenericFunctionNodeData,
+    solvable::Int,
+    variableOrder::Union{Vector{Symbol}, Tuple};
+    observation = getFactorType(solverData),
+    state::FactorState = FactorState(),
+    solvercache::Base.RefValue{<:FactorSolverCache} = Ref{FactorSolverCache}(),
+    id::Union{UUID, Nothing} = nothing,
+    smallData::Dict{Symbol, SmallDataTypes} = Dict{Symbol, SmallDataTypes}(),
+)
+    error(
+        "This constructor is deprecated, use FactorCompute(label, variableOrder, solverData; ...) instead",
+    )
+    return FactorCompute(
+        id,
+        label,
+        tags,
+        Tuple(variableOrder),
+        timestamp,
+        nstime,
+        Ref(solverData),
+        Ref(solvable),
+        smallData,
+        observation,
+        state,
+        solvercache,
+    )
+end
+
+export getSolverData, setSolverData!
+
+function getSolverData(f::FactorCompute)
+    return error(
+        "getSolverData(f::FactorCompute) is obsolete, use getState, getObservation, or getCache instead",
+    )
+end
+
+function setSolverData!(f::FactorCompute, data::GenericFunctionNodeData)
+    return error(
+        "setSolverData!(f::FactorCompute, data::GenericFunctionNodeData) is obsolete, use setState!, or setCache! instead",
+    )
+end
+
+@deprecate unpackFactor(dfg::AbstractDFG, factor::FactorDFG; skipVersionCheck::Bool = false) unpackFactor(
+    factor;
+    skipVersionCheck,
+)
+
+@deprecate rebuildFactorMetadata!(args...; kwargs...) rebuildFactorCache!(
+    args...;
+    kwargs...,
+)
+
+export reconstFactorData
+function reconstFactorData end
+
+function decodePackedType(
+    dfg::AbstractDFG,
+    varOrder::AbstractVector{Symbol},
+    ::Type{T},
+    packeddata::GenericFunctionNodeData{PT},
+) where {T <: FactorSolverCache, PT}
+    error("decodePackedType is obsolete")
+    #
+    # TODO, to solve IIF 1424
+    # variables = map(lb->getVariable(dfg, lb), varOrder)
+
+    # Also look at parentmodule
+    usrtyp = convertStructType(PT)
+    fulltype = DFG.FunctionNodeData{T{usrtyp}}
+    factordata = reconstFactorData(dfg, varOrder, fulltype, packeddata)
+    return factordata
+end
+
+export _packSolverData
+function _packSolverData(f::FactorCompute, fnctype::AbstractFactorObservation)
+    #
+    error("_packSolverData is deprecated, use seperate packing of observation #TODO")
+    packtype = convertPackedType(fnctype)
+    try
+        packed = convert(PackedFunctionNodeData{packtype}, getSolverData(f)) #TODO getSolverData 
+        packedJson = packed
+        return packedJson
+    catch ex
+        io = IOBuffer()
+        showerror(io, ex, catch_backtrace())
+        err = String(take!(io))
+        msg = "Error while packing '$(f.label)' as '$fnctype', please check the unpacking/packing converters for this factor - \r\n$err"
+        error(msg)
+    end
+end
+
+export GenericFunctionNodeData, PackedFunctionNodeData, FunctionNodeData
+
+const PackedFunctionNodeData{T} =
+    GenericFunctionNodeData{T} where {T <: AbstractPackedFactorObservation}
+function PackedFunctionNodeData(args...; kw...)
+    error("PackedFunctionNodeData is obsolete")
+    return PackedFunctionNodeData{typeof(args[4])}(args...; kw...)
+end
+
+const FunctionNodeData{T} = GenericFunctionNodeData{
+    T,
+} where {T <: Union{<:AbstractFactorObservation, <:FactorSolverCache}}
+FunctionNodeData(args...; kw...) = FunctionNodeData{typeof(args[4])}(args...; kw...)
+
+# this is the GenericFunctionNodeData for packed types
+#TODO deprecate FactorData in favor of FactorState (with no more distinction between packed and compute)
+const FactorData = PackedFunctionNodeData{AbstractPackedFactorObservation}
+
+function FactorCompute(
+    label::Symbol,
+    variableOrder::Union{Vector{Symbol}, Tuple},
+    solverData::GenericFunctionNodeData;
+    tags::Set{Symbol} = Set{Symbol}(),
+    timestamp::Union{DateTime, ZonedDateTime} = now(localzone()),
+    solvable::Int = 1,
+    nstime::Nanosecond = Nanosecond(0),
+    id::Union{UUID, Nothing} = nothing,
+    smallData::Dict{Symbol, SmallDataTypes} = Dict{Symbol, SmallDataTypes}(),
+)
+    Base.depwarn(
+        "`FactorCompute` constructor with `GenericFunctionNodeData` is deprecated. observation, state, and solvercache should be provided explicitly.",
+        :FactorCompute,
+    )
+    observation = getFactorType(solverData)
+    state = FactorState(
+        solverData.eliminated,
+        solverData.potentialused,
+        solverData.multihypo,
+        solverData.certainhypo,
+        solverData.nullhypo,
+        solverData.solveInProgress,
+        solverData.inflation,
+    )
+
+    if solverData.fnc isa FactorSolverCache
+        solvercache = solverData.fnc
+    else
+        solvercache = nothing
+    end
+
+    return FactorCompute(
+        label,
+        Tuple(variableOrder),
+        observation,
+        state,
+        solvercache;
+        id,
+        timestamp,
+        nstime,
+        tags,
+        smallData,
+        solvable,
+    )
+end
+
+# Deprecated check usefull? # packedFnc = fncStringToData(factor.fnctype, factor.data)
+# Deprecated check usefull? # decodeType = getFactorOperationalMemoryType(dfg)
+# Deprecated check usefull? # fullFactorData = decodePackedType(dfg, factor._variableOrderSymbols, decodeType, packedFnc)
+function fncStringToData(args...; kwargs...)
+    @warn "fncStringToData is obsolete, called with" args kwargs
+    return error("fncStringToData is obsolete.")
+end
+
 ## ================================================================================
 ## Deprecated in v0.25
 ##=================================================================================

src/DistributedFactorGraphs.jl

@@ -18,6 +18,7 @@ using Base
 using Base64
 using DocStringExtensions
 using Dates
+using Random
 using TimeZones
 using Distributions
 using Reexport
@@ -42,7 +43,7 @@ using Tables
 
 # used for @defVariable
 import ManifoldsBase
-import ManifoldsBase: AbstractManifold, manifold_dimension
+using ManifoldsBase: AbstractManifold, manifold_dimension
 export AbstractManifold, manifold_dimension
 
 import RecursiveArrayTools: ArrayPartition
@@ -165,7 +166,8 @@ export InferenceVariable
 export getSolverDataDict, setSolverData!
 export getVariableType, getVariableTypeName
 
-export getSolverData
+export getObservation
+export getState, getFactorState
 
 export getVariableType
 
@@ -246,10 +248,9 @@ export @format_str
 # Factors
 ##------------------------------------------------------------------------------
 # Factor Data
-export GenericFunctionNodeData, PackedFunctionNodeData, FunctionNodeData
-export AbstractFactor, AbstractPackedFactor
+export AbstractFactorObservation, AbstractPackedFactorObservation
 export AbstractPrior, AbstractRelative, AbstractRelativeMinimize, AbstractManifoldMinimize
-export FactorOperationalMemory
+export FactorSolverCache
 
 # accessors
 export getVariableOrder
@@ -261,7 +262,7 @@ export mergeVariableData!, mergeGraphVariableData!
 # Serialization type conversion
 export convertPackedType, convertStructType
 
-export reconstFactorData
+export pack, unpack, packDistribution, unpackDistribution
 
 ##------------------------------------------------------------------------------
 ## Other utility functions
@@ -285,7 +286,7 @@ export findClosestTimestamp, findVariableNearTimestamp
 
 # Serialization
 export packVariable, unpackVariable, packFactor, unpackFactor
-export rebuildFactorMetadata!
+export rebuildFactorCache!
 export @defVariable
 
 # File import and export
@@ -301,7 +302,6 @@ export compare,
     compareField,
     compareFields,
     compareAll,
-    compareAllSpecial,
     compareVariable,
     compareFactor,
     compareAllVariables,