Refactor DFG single agent to multiple agents and rename blobstore->storelabel and :default -> :primary (#1216)

Author: Affie

src/DataBlobs/entities/BlobEntry.jl

@@ -8,13 +8,13 @@ A `Blobentry` is a small about of structured data that holds reference informati
 can exist on different graph nodes spanning Agents and Factor Graphs which can all reference the same `Blob`.
 
 Notes:
-- `blobid`s should be unique within a blobstore and are immutable.
+- `blobid`s should be unique within a Blobstore and are immutable.
 """
 StructUtils.@kwarg struct Blobentry
     """ Human friendly label of the `Blob` and also used as unique identifier per node on which a `Blobentry` is added.  E.g. do "LEFTCAM_1", "LEFTCAM_2", ... of you need to repeat a label on the same variable. """
     label::Symbol
-    """ The label of the `Blobstore` in which the `Blob` is stored.  Default is `:default`."""
-    blobstore::Symbol = :default
+    """ The label of the `Blobstore` in which the `Blob` is stored.  Default is `:primary`."""
+    storelabel::Symbol = :primary
     """ Machine friendly and unique within a `Blobstore` identifier of the 'Blob'."""
     blobid::UUID = uuid4() # was blobId
     """ (Optional) crc32c hash value to ensure data consistency which must correspond to the stored hash upon retrieval."""
@@ -52,21 +52,21 @@ version(::Type{Blobentry}) = v"0.1.0"
 
 function Blobentry(
     label::Symbol,
-    blobstore = :default;
+    storelabel = :primary;
     metadata::Union{JSONText, AbstractDict, NamedTuple} = JSONText("{}"),
     kwargs...,
 )
     if !(metadata isa JSONText)
         metadata = JSONText(JSON.json(metadata))
     end
-    return Blobentry(; label, blobstore, metadata, kwargs...)
+    return Blobentry(; label, storelabel, metadata, kwargs...)
 end
 # construction helper from existing Blobentry for user overriding via kwargs
 function Blobentry(
     entry::Blobentry;
     blobid::UUID = entry.blobid,
     label::Symbol = entry.label,
-    blobstore::Symbol = entry.blobstore,
+    storelabel::Symbol = entry.storelabel,
     crchash = entry.crchash,
     shahash = entry.shahash,
     size::Int64 = entry.size,
@@ -76,10 +76,15 @@ function Blobentry(
     metadata::JSONText = entry.metadata,
     timestamp::TimeDateZone = entry.timestamp,
     version = entry.version,
+    blobstore = nothing, # TODO note deprecated in v0.29
 )
+    !isnothing(blobstore) && Base.depwarn(
+        "The `blobstore` keyword argument has been renamed to `storelabel`",
+        :Blobentry,
+    )
     return Blobentry(;
         label,
-        blobstore,
+        storelabel,
         blobid,
         crchash,
         shahash,

src/DataBlobs/entities/BlobStores.jl

@@ -1,7 +1,7 @@
 """
     AbstractBlobstore{T}
 
-Abstract supertype for all blobstore implementations.
+Abstract supertype for all Blobstore implementations.
 
 # Usage
 
@@ -16,9 +16,9 @@ The parameter `T` represents the type of blobs stored (e.g., `Vector{UInt8}` or
 See concrete implementations for details.
 
 Design Notes
-- `blobid` is not considered unique across blobstores with different labels only within a single blobstore.
-- We cannot guarantee that `blobid` is unique across different blobstores with the same label and this is up to the end user.
-- Within a single blobstore `addBlob!` will fail if there is a UUID collision.
+- `blobid` is not considered unique across Blobstores with different labels only within a single Blobstore.
+- We cannot guarantee that `blobid` is unique across different Blobstores with the same label and this is up to the end user.
+- Within a single Blobstore `addBlob!` will fail if there is a UUID collision.
 - TODO: We should consider using uuid7 for `blobid`s (requires jl v1.12).
 - `Blobstrores`are identified by a `label::Symbol`, which allows for multiple blobstores to coexist in the same system.
 

src/DataBlobs/services/BlobEntry.jl

@@ -111,7 +111,7 @@ function Base.show(io::IO, ::MIME"text/plain", entry::Blobentry)
     println(io, "Blobentry {")
     println(io, "  blobid:        ", entry.blobid)
     println(io, "  label:         ", entry.label)
-    println(io, "  blobstore:     ", entry.blobstore)
+    println(io, "  storelabel:     ", entry.storelabel)
     println(io, "  origin:        ", entry.origin)
     println(io, "  description:   ", entry.description)
     println(io, "  mimetype:      ", entry.mimetype)
@@ -255,8 +255,12 @@ function addFactorBlobentries!(dfg::AbstractDFG, fLbl::Symbol, entries::Vector{B
     return entries
 end
 
-function addAgentBlobentries!(dfg::AbstractDFG, entries::Vector{Blobentry})
-    addAgentBlobentry!.(dfg, entries)
+function addAgentBlobentries!(
+    dfg::AbstractDFG,
+    agentlabel::Symbol,
+    entries::Vector{Blobentry},
+)
+    addAgentBlobentry!.(dfg, agentlabel, entries)
     return entries
 end
 
@@ -277,8 +281,12 @@ function mergeFactorBlobentries!(dfg::AbstractDFG, fLbl::Symbol, entries::Vector
     mergeFactorBlobentry!.(dfg, fLbl, entries)
     return length(entries)
 end
-function mergeAgentBlobentries!(dfg::AbstractDFG, entries::Vector{Blobentry})
-    mergeAgentBlobentry!.(dfg, entries)
+function mergeAgentBlobentries!(
+    dfg::AbstractDFG,
+    agentlabel::Symbol,
+    entries::Vector{Blobentry},
+)
+    mergeAgentBlobentry!.(dfg, agentlabel, entries)
     return length(entries)
 end
 function mergeGraphBlobentries!(dfg::AbstractDFG, entries::Vector{Blobentry})
@@ -308,9 +316,13 @@ function deleteFactorBlobentries!(
     return sum(cnts)
 end
 
-function deleteAgentBlobentries!(dfg::AbstractDFG, labels::Vector{Symbol})
+function deleteAgentBlobentries!(
+    dfg::AbstractDFG,
+    agentlabel::Symbol,
+    labels::Vector{Symbol},
+)
     cnts = map(labels) do label
-        return deleteAgentBlobentry!(dfg, label)
+        return deleteAgentBlobentry!(dfg, agentlabel, label)
     end
     return sum(cnts)
 end

src/DataBlobs/services/BlobStores.jl

@@ -3,7 +3,7 @@
 ##==============================================================================
 
 """
-Get the data blob for the specified blobstore or dfg.
+Get the data blob for the specified Blobstore or DFG.
 
 Related
 [`getBlobentry`](@ref)
@@ -15,7 +15,7 @@ $(METHODLIST)
 function getBlob end
 
 """
-Adds a blob to the blob store or dfg with the blobid.
+Adds a blob to the Blobstore with the blobid.
 
 Related
 [`addBlobentry!`](@ref)
@@ -53,9 +53,9 @@ function hasBlob end
 ##==============================================================================
 ## AbstractBlobstore derived CRUD for Blob 
 ##==============================================================================
-#TODO maybe we should generalize and move the cached blobstore to DFG.
+#TODO maybe we should generalize and move the cached Blobstore to DFG.
 function getBlob(dfg::AbstractDFG, entry::Blobentry)
-    storeLabel = entry.blobstore
+    storeLabel = entry.storelabel
     store = getBlobstore(dfg, storeLabel)
     return getBlob(store, entry.blobid)
 end
@@ -66,7 +66,7 @@ end
 
 #add 
 function addBlob!(dfg::AbstractDFG, entry::Blobentry, data)
-    return addBlob!(getBlobstore(dfg, entry.blobstore), entry, data)
+    return addBlob!(getBlobstore(dfg, entry.storelabel), entry, data)
 end
 
 function addBlob!(store::AbstractBlobstore{T}, entry::Blobentry, data::T) where {T}
@@ -78,7 +78,7 @@ addBlob!(store::AbstractBlobstore, data) = addBlob!(store, uuid4(), data)
 
 #delete
 function deleteBlob!(dfg::AbstractDFG, entry::Blobentry)
-    return deleteBlob!(getBlobstore(dfg, entry.blobstore), entry)
+    return deleteBlob!(getBlobstore(dfg, entry.storelabel), entry)
 end
 
 function deleteBlob!(store::AbstractBlobstore, entry::Blobentry)
@@ -90,7 +90,7 @@ function hasBlob(store::AbstractBlobstore, entry::Blobentry)
     return hasBlob(store, entry.blobid)
 end
 function hasBlob(dfg::AbstractDFG, entry::Blobentry)
-    return hasBlob(getBlobstore(dfg, entry.blobstore), entry.blobid)
+    return hasBlob(getBlobstore(dfg, entry.storelabel), entry.blobid)
 end
 
 #TODO
@@ -123,7 +123,7 @@ end
 
 FolderStore(label::Symbol, folder::String) = FolderStore{Vector{UInt8}}(label, folder)
 
-function FolderStore(foldername::String; label::Symbol = :default, createfolder = true)
+function FolderStore(foldername::String; label::Symbol = :primary, createfolder = true)
     storepath = expanduser(joinpath(foldername, string(label)))
     if createfolder && !isdir(storepath)
         @info "Folder '$storepath' doesn't exist - creating."
@@ -212,7 +212,7 @@ end
 function InMemoryBlobstore{T}(storeKey::Symbol) where {T}
     return InMemoryBlobstore{T}(storeKey, Dict{UUID, T}())
 end
-function InMemoryBlobstore(storeKey::Symbol = :default_inmemory_store)
+function InMemoryBlobstore(storeKey::Symbol = :primary)
     return InMemoryBlobstore{Vector{UInt8}}(storeKey)
 end
 

src/DataBlobs/services/BlobWrappers.jl

@@ -113,10 +113,10 @@ function saveBlob_Variable!(
     variable_label::Symbol,
     blob::Vector{UInt8},
     entry_label::Symbol,
-    blobstore::Symbol = :default;
+    storelabel::Symbol = :primary;
     blobentry_kwargs...,
 )
-    entry = Blobentry(entry_label, blobstore; blobentry_kwargs...)
+    entry = Blobentry(entry_label, storelabel; blobentry_kwargs...)
     return saveBlob_Variable!(dfg, variable_label, blob, entry)
 end
 
@@ -143,10 +143,10 @@ function saveBlob_Graph!(
     dfg::AbstractDFG,
     blob::Vector{UInt8},
     entry_label::Symbol,
-    blobstore::Symbol = :default;
+    storelabel::Symbol = :primary;
     blobentry_kwargs...,
 )
-    entry = Blobentry(entry_label, blobstore; blobentry_kwargs...)
+    entry = Blobentry(entry_label, storelabel; blobentry_kwargs...)
     return saveBlob_Graph!(dfg, blob, entry)
 end
 
@@ -157,32 +157,38 @@ function deleteBlob_Graph!(dfg::AbstractDFG, entry_label::Symbol)
     return 2
 end
 
-function loadBlob_Agent(dfg::AbstractDFG, entry_label::Symbol;)
-    entry = getAgentBlobentry(dfg, entry_label)
+function loadBlob_Agent(dfg::AbstractDFG, agentlabel::Symbol, entry_label::Symbol;)
+    entry = getAgentBlobentry(dfg, agentlabel, entry_label)
     blob = getBlob(dfg, entry)
     return entry, blob
 end
 
-function saveBlob_Agent!(dfg::AbstractDFG, blob::Vector{UInt8}, entry::Blobentry)
-    addAgentBlobentry!(dfg, entry)
+function saveBlob_Agent!(
+    dfg::AbstractDFG,
+    agentlabel::Symbol,
+    blob::Vector{UInt8},
+    entry::Blobentry,
+)
+    addAgentBlobentry!(dfg, agentlabel, entry)
     addBlob!(dfg, entry, blob)
     return entry
 end
 
 function saveBlob_Agent!(
     dfg::AbstractDFG,
+    agentlabel::Symbol,
     blob::Vector{UInt8},
     entry_label::Symbol,
-    blobstore::Symbol = :default;
+    storelabel::Symbol = :primary;
     blobentry_kwargs...,
 )
-    entry = Blobentry(entry_label, blobstore; blobentry_kwargs...)
-    return saveBlob_Agent!(dfg, blob, entry)
+    entry = Blobentry(entry_label, storelabel; blobentry_kwargs...)
+    return saveBlob_Agent!(dfg, agentlabel, blob, entry)
 end
 
-function deleteBlob_Agent!(dfg::AbstractDFG, entry_label::Symbol)
-    entry = getAgentBlobentry(dfg, entry_label)
-    deleteAgentBlobentry!(dfg, entry_label)
+function deleteBlob_Agent!(dfg::AbstractDFG, agentlabel::Symbol, entry_label::Symbol)
+    entry = getAgentBlobentry(dfg, agentlabel, entry_label)
+    deleteAgentBlobentry!(dfg, agentlabel, entry_label)
     deleteBlob!(dfg, entry)
     return 2
 end
@@ -209,10 +215,10 @@ function saveBlob_Factor!(
     factor_label::Symbol,
     blob::Vector{UInt8},
     entry_label::Symbol,
-    blobstore::Symbol = :default;
+    storelabel::Symbol = :primary;
     blobentry_kwargs...,
 )
-    entry = Blobentry(entry_label, blobstore; blobentry_kwargs...)
+    entry = Blobentry(entry_label, storelabel; blobentry_kwargs...)
     return saveBlob_Factor!(dfg, factor_label, blob, entry)
 end
 
@@ -228,7 +234,7 @@ function saveImage_Variable!(
     variable_label::Symbol,
     img::AbstractMatrix,
     entry_label::Symbol,
-    blobstore::Symbol = :default;
+    storelabel::Symbol = :primary;
     entry_kwargs...,
 )
     mimetype = get(entry_kwargs, :mimeType, MIME("image/png"))
@@ -239,7 +245,7 @@ function saveImage_Variable!(
 
     entry = Blobentry(
         entry_label,
-        blobstore;
+        storelabel;
         blobid = uuid4(),
         entry_kwargs...,
         size = length(blob),

src/Deprecated.jl

@@ -117,7 +117,7 @@ function _getDuplicatedEmptyDFG(
 ) where {P <: AbstractDFGParams, V <: AbstractGraphVariable, F <: AbstractGraphFactor}
     Base.depwarn("_getDuplicatedEmptyDFG is deprecated.", :_getDuplicatedEmptyDFG)
     newDfg = GraphsDFG{P, V, F}(;
-        agentLabel = getAgentLabel(dfg),
+        agents = deepcopy(dfg.agents),
         graphLabel = getGraphLabel(dfg),
         solverParams = deepcopy(dfg.solverParams),
     )
@@ -184,7 +184,9 @@ end
 # Function to generate source string - agentLabel|graphLabel|varLabel
 # """
 function buildSourceString(dfg::AbstractDFG, label::Symbol)
-    return "$(getAgentLabel(dfg))|$(getGraphLabel(dfg))|$label"
+    return error(
+        "buildSourceString is deprecated. Use agents with `listAgents(dfg)` and `getAgent(dfg, agentlabel)` instead.",
+    )
 end
 
 getAgentMetadata(args...) = error("getAgentMetadata is obsolete, use Bloblets instead.")
@@ -572,7 +574,7 @@ function deepcopyGraph(
     destDFG = T(;
         solverParams = getSolverParams(sourceDFG),
         graph = sourceDFG.graph,
-        agent = sourceDFG.agent,
+        agents = deepcopy(sourceDFG.agents),
         graphLabel,
     )
     copyGraph!(

src/DistributedFactorGraphs.jl

@@ -392,11 +392,16 @@ export loadDFG
 ## Other utility functions
 ##------------------------------------------------------------------------------
 
+## Agent CRUD (now in AbstractDFG services)
+export addAgent!
+export deleteAgent!
+export listAgents
+export getAgent
+export hasAgent
+# export mergeAgent!
+# public refAgents
+
 ## TODO maybe move to DFG from SDK
-# addAgent!
-# deleteAgent!
-# listAgents
-# getAgents
 # addGraph!
 # deleteGraph!
 # listGraphs
@@ -445,7 +450,6 @@ const unstable_functions::Vector{Symbol} = [
     :findVariablesNearTimestamp,
     :findShortestPathDijkstra,
     :findFactorsBetweenNaive, # TODO not really used
-    :getAgentLabel, #TODO check and mark as public
     :getGraphLabel, #TODO check and mark as public
     :getDescription,
     :getSolverParams,