Standardise to Blobstore label (#1113)

Co-authored-by: Johannes Terblanche <Affie@users.noreply.github.com>

Author: Affie

src/DataBlobs/services/BlobEntry.jl

@@ -79,11 +79,7 @@ end
 
 function getBlobEntry(var::VariableDFG, key::Symbol)
     if !hasBlobEntry(var, key)
-        throw(
-            KeyError(
-                "No dataEntry label $(key) found in variable $(getLabel(var)). Available keys: $(keys(var.dataDict))",
-            ),
-        )
+        throw(KeyError(key))
     end
     return var.blobEntries[findfirst(x -> x.label == key, var.blobEntries)]
 end

src/DataBlobs/services/BlobStores.jl

@@ -97,7 +97,7 @@ function getBlob(dfg::AbstractDFG, entry::BlobEntry)
     end
     throw(
         KeyError(
-            "could not find $(entry.label), uuid $(entry.blobId) in any of the listed blobstores:\n $([s->getKey(s) for (s,v) in stores]))",
+            "could not find $(entry.label), uuid $(entry.blobId) in any of the listed blobstores:\n $([s->getLabel(s) for (s,v) in stores]))",
         ),
     )
 end
@@ -177,13 +177,10 @@ end
 ## FolderStore
 ##==============================================================================
 struct FolderStore{T} <: AbstractBlobStore{T}
-    key::Symbol
+    label::Symbol
     folder::String
 end
 
-#TODO rename key to label for consistency
-getLabel(store::FolderStore) = store.key
-
 function FolderStore(foldername::String; label = :default_folder_store, createfolder = true)
     if createfolder && !isdir(foldername)
         @info "Folder '$foldername' doesn't exist - creating."
@@ -247,12 +244,13 @@ end
 
 hasBlob(store::FolderStore, entry::BlobEntry) = hasBlob(store, entry.originId)
 
+listBlobs(store::FolderStore) = readdir(store.folder)
 ##==============================================================================
 ## InMemoryBlobStore
 ##==============================================================================
 
 struct InMemoryBlobStore{T} <: AbstractBlobStore{T}
-    key::Symbol
+    label::Symbol
     blobs::Dict{UUID, T}
 end
 
@@ -295,22 +293,22 @@ listBlobs(store::InMemoryBlobStore) = collect(keys(store.blobs))
 ##==============================================================================
 
 struct LinkStore <: AbstractBlobStore{String}
-    key::Symbol
+    label::Symbol
     csvfile::String
     cache::Dict{UUID, String}
 
-    function LinkStore(key, csvfile)
+    function LinkStore(label, csvfile)
         if !isfile(csvfile)
             @info "File '$csvfile' doesn't exist - creating."
             # create new folder
             open(csvfile, "w") do io
                 return println(io, "blobid,path")
             end
-            return new(key, csvfile, Dict{UUID, String}())
+            return new(label, csvfile, Dict{UUID, String}())
         else
             file = CSV.File(csvfile)
             cache = Dict(UUID.(file.blobid) .=> file.path)
-            return new(key, csvfile, cache)
+            return new(label, csvfile, cache)
         end
     end
 end
@@ -373,7 +371,7 @@ end
 ## RowBlobStore
 
 struct RowBlobStore{T} <: AbstractBlobStore{T}
-    key::Symbol
+    label::Symbol
     blobs::OrderedDict{UUID, RowBlob{T}}
 end
 

src/DataBlobs/services/HelpersDataWrapEntryBlob.jl

@@ -191,7 +191,7 @@ function addData!(
         blobId,
         originId,
         label = bLbl,
-        blobstore = blobstore.key,
+        blobstore = getLabel(blobstore),
         hash = string(bytes2hex(hashfunction(blob))),
         origin = buildSourceString(dfg, vLbl),
         description,
@@ -224,7 +224,7 @@ function addData!(
         blobId,
         originId = blobId,
         label = blobLabel,
-        blobstore = blobstore.key,
+        blobstore = getLabel(blobstore),
         # hash = string(bytes2hex(hashfunction(blob))),
         hash = "",
         origin,
@@ -264,7 +264,7 @@ function updateData!(
     # order of operations with unknown new blobId not tested
     newEntry = BlobEntry(
         entry; # and kwargs to override new values
-        blobstore = blobstore.key,
+        blobstore = getLabel(blobstore),
         hash = string(bytes2hex(hashfunction(blob))),
         origin = buildSourceString(dfg, label),
         _version = string(_getDFGVersion()),

src/Deprecated.jl

@@ -39,6 +39,8 @@ export DFGSummary
 DFGSummary(args) = error("DFGSummary is deprecated")
 @deprecate getSummary(dfg::AbstractDFG) getSummaryGraph(dfg)
 
+@deprecate getKey(store::AbstractBlobStore) getLabel(store)
+
 ##------------------------------------------------------------------------------
 ## smallData
 ##------------------------------------------------------------------------------

src/GraphsDFG/services/GraphsDFGSerialization.jl

@@ -53,9 +53,9 @@ function packDFGMetadata(fg::GraphsDFG)
     blobStores = Dict{Symbol, FolderStore{Vector{UInt8}}}()
     foreach(values(fg.blobStores)) do store
         if store isa FolderStore{Vector{UInt8}}
-            blobStores[store.key] = store
+            blobStores[getLabel(store)] = store
         else
-            @warn "BlobStore $(store.key) of type $(typeof(store)) is not supported yet and will not be saved"
+            @warn "BlobStore $(getLabel(store)) of type $(typeof(store)) is not supported yet and will not be saved"
         end
     end
 

src/services/AbstractDFG.jl

@@ -253,16 +253,15 @@ function listModelBlobEntries end
 ##==============================================================================
 ## AbstractBlobStore  CRUD
 ##==============================================================================
-# AbstractBlobStore should have key or overwrite getKey
-getKey(store::AbstractBlobStore) = store.key
+# AbstractBlobStore should have label or overwrite getLabel
 
 getBlobStores(dfg::AbstractDFG) = dfg.blobStores
 getBlobStore(dfg::AbstractDFG, key::Symbol) = dfg.blobStores[key]
 function addBlobStore!(dfg::AbstractDFG, bs::AbstractBlobStore)
-    return push!(dfg.blobStores, getKey(bs) => bs)
+    return push!(dfg.blobStores, getLabel(bs) => bs)
 end
 function updateBlobStore!(dfg::AbstractDFG, bs::AbstractBlobStore)
-    return push!(dfg.blobStores, getKey(bs) => bs)
+    return push!(dfg.blobStores, getLabel(bs) => bs)
 end
 deleteBlobStore!(dfg::AbstractDFG, key::Symbol) = pop!(dfg.blobStores, key)
 emptyBlobStore!(dfg::AbstractDFG) = empty!(dfg.blobStores)

src/services/DFGFactor.jl

@@ -30,6 +30,7 @@ Notes
 """
 getFactorType(data::GenericFunctionNodeData) = data.fnc.usrfnc!
 getFactorType(fct::FactorCompute) = getFactorType(getSolverData(fct))
+getFactorType(f::FactorDFG) = getTypeFromSerializationModule(f.fnctype)() # TODO find a better way to do this that does not rely on empty constructor
 getFactorType(dfg::AbstractDFG, lbl::Symbol) = getFactorType(getFactor(dfg, lbl))
 
 """
@@ -150,7 +151,7 @@ getVariableOrder(dfg::AbstractDFG, fct::Symbol) = getVariableOrder(getFactor(dfg
 
 Retrieve solver data structure stored in a factor.
 """
-function getSolverData(f::F) where {F <: FactorCompute}
+function getSolverData(f::FactorCompute)
     return f.solverData
 end
 

test/consol_DataEntryBlobTests.jl

@@ -122,7 +122,7 @@ deleteData!(dfg, :x2, :random)
 #test default folder store
 dfs = FolderStore("/tmp/defaultfolderstore")
 @test dfs.folder == "/tmp/defaultfolderstore"
-@test dfs.key == :default_folder_store
+@test getLabel(dfs) == :default_folder_store
 @test dfs isa FolderStore{Vector{UInt8}}
 
 ##==============================================================================

test/testBlocks.jl

@@ -1115,14 +1115,14 @@ function blobsStoresTestBlock!(fg)
     # Adding
     addBlobStore!(fg, fs)
     # Listing
-    @test listBlobStores(fg) == [fs.key]
+    @test listBlobStores(fg) == [fs.label]
     # Getting
-    @test getBlobStore(fg, fs.key) == fs
+    @test getBlobStore(fg, fs.label) == fs
     # Deleting
-    @test deleteBlobStore!(fg, fs.key) == fs
+    @test deleteBlobStore!(fg, fs.label) == fs
     # Updating
     updateBlobStore!(fg, fs)
-    @test listBlobStores(fg) == [fs.key]
+    @test listBlobStores(fg) == [fs.label]
     # Emptying
     emptyBlobStore!(fg)
     @test listBlobStores(fg) == []
@@ -1132,7 +1132,7 @@ function blobsStoresTestBlock!(fg)
     # Data functions
     testData = rand(UInt8, 50)
     # Adding 
-    newData = addData!(fg, fs.key, :a, :testing, testData) # convenience wrapper over addBlob!
+    newData = addData!(fg, fs.label, :a, :testing, testData) # convenience wrapper over addBlob!
     # Listing
     @test :testing in listBlobEntries(fg, :a)
     # Getting
@@ -1154,7 +1154,7 @@ function blobsStoresTestBlock!(fg)
     @test updateData[1].hash != data[1].hash
     @test updateData[2] != data[2]
     @show bllb = DistributedFactorGraphs.incrDataLabelSuffix(fg, :a, :testing)
-    newData2 = addData!(fg, fs.key, :a, bllb, testData) # convenience wrapper over addBlob!
+    newData2 = addData!(fg, fs.label, :a, bllb, testData) # convenience wrapper over addBlob!
     nbe = listBlobEntries(fg, :a)
     filter!(s -> occursin(r"testing", string(s)), nbe)
     @test 2 == length(nbe)