Merge branch 'develop' into compathelper/new_version/2025-10-03-01-06-04-647-02452626131

Author: Affie

Project.toml

@@ -1,6 +1,6 @@
 name = "DistributedFactorGraphs"
 uuid = "b5cc3c7e-6572-11e9-2517-99fb8daf2f04"
-version = "0.27.1"
+version = "0.28.0"
 
 [deps]
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
@@ -13,7 +13,7 @@ DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
 FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
 Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6"
 InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
-JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
+JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 ManifoldsBase = "3362f125-f0bb-47a3-aa74-596ffd7ef2fb"
 OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
@@ -24,7 +24,6 @@ RecursiveArrayTools = "731186ca-8d62-57ce-b412-fbd966d074cd"
 SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
 StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
-StructTypes = "856f2bd8-1eba-4b0a-8007-ebc267875bd4"
 Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
 Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
 TensorCast = "02d47bb6-7ce6-556a-be16-bb1710789e2b"
@@ -46,19 +45,18 @@ Base64 = "1.10"
 CSV = "0.10"
 CodecZlib = "0.7"
 Colors = "0.10, 0.11, 0.12, 0.13"
-DataStructures = "0.18"
 Dates = "1.10"
 Distributions = "0.23, 0.24, 0.25"
 DocStringExtensions = "0.8, 0.9"
 FileIO = "1"
 GraphMakie = "0.5.14, 0.6"
 Graphs = "1.4"
 InteractiveUtils = "1.10"
-JSON3 = "1"
+JSON = "1.0.0"
 LieGroups = "0.1"
 LinearAlgebra = "1.10"
-Manifolds = "0.9, 0.10"
-ManifoldsBase = "0.14, 0.15, 1, 2"
+Manifolds = "0.10, 0.11"
+ManifoldsBase = "1, 2"
 OrderedCollections = "1.4"
 Pkg = "1.4, 1.5"
 ProgressMeter = "1"
@@ -68,7 +66,6 @@ SHA = "0.7, 1"
 SparseArrays = "1.10"
 StaticArrays = "1"
 Statistics = "1.10"
-StructTypes = "1"
 Tables = "v1.11.1"
 Tar = "1.9"
 TensorCast = "0.3.3, 0.4"
@@ -79,7 +76,6 @@ julia = "1.10"
 
 [extras]
 Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
-DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
 GraphMakie = "1ecd5474-83a3-4783-bb4f-06765db800d2"
 LieGroups = "6774de46-80ba-43f8-ba42-e41071ccfc5f"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
@@ -89,4 +85,4 @@ Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Aqua", "Test", "DataStructures", "GraphMakie", "LinearAlgebra", "Manifolds", "Pkg", "Statistics", "LieGroups"]
+test = ["Aqua", "Test", "GraphMakie", "LinearAlgebra", "Manifolds", "Pkg", "Statistics", "LieGroups"]

docs/src/blob_ref.md

@@ -26,7 +26,7 @@ Pages = [
     "DataBlobs/services/BlobEntry.jl",
     "DataBlobs/services/BlobStores.jl",
     "DataBlobs/services/BlobPacking.jl",
-    "DataBlobs/services/HelpersDataWrapEntryBlob.jl",
+    "DataBlobs/services/BlobWrappers.jl",
 ]
 
 ```

src/DataBlobs/entities/BlobEntry.jl

@@ -2,7 +2,10 @@
 ##==============================================================================
 ## Blobentry
 ##==============================================================================
-
+#TODO think origin and buildSourceString should be deprecated, description can be used instead
+#TODO hash - maybe use both crc32c for fast error check and sha256 for strong integrity check
+#            stored seperately as crc and sha or as a tuple `hash::Tuple{Symbol, String}` where Symbol is :crc32c or :sha256
+#            or an enum with suppored hash types
 """
     $(TYPEDEF)
 
@@ -16,9 +19,7 @@ Base.@kwdef struct Blobentry
     """ Remotely assigned and globally unique identifier for the `Blobentry` itself (not the `.blobId`). """
     id::Union{UUID, Nothing} = nothing
     """ Machine friendly and globally unique identifier of the 'Blob', usually assigned from a common point in the system.  This can be used to guarantee unique retrieval of the large data blob. """
-    blobId::Union{UUID, Nothing} = uuid4() #Deprecated in v0.25 TODO  remove union, blobId is mandatory
-    """ TBD if Deprecated. Machine friendly and locally assigned identifier of the 'Blob'."""
-    originId::Union{UUID, Nothing} = blobId #Deprecated in v0.25 TODO remove union or remove originId completely
+    blobId::UUID = uuid4()
     """ Human friendly label of the `Blob` and also used as unique identifier per node on which a `Blobentry` is added.  E.g. do "LEFTCAM_1", "LEFTCAM_2", ... of you need to repeat a label on the same variable. """
     label::Symbol
     """ A hint about where the `Blob` itself might be stored.  Remember that a Blob may be duplicated over multiple blobstores. """
@@ -28,7 +29,7 @@ Base.@kwdef struct Blobentry
     """ Context from which a Blobentry=>Blob was first created. E.g. agent|graph|varlabel. """
     origin::String = ""
     """ number of bytes in blob as a string"""
-    size::Union{String, Nothing} = "-1" #TODO remove union -> size::String = "-1"
+    size::String = "-1"
     """ Additional information that can help a different user of the Blob. """
     description::String = ""
     """ MIME description describing the format of binary data in the `Blob`, e.g. 'image/png' or 'application/json; _type=CameraModel'. """
@@ -45,6 +46,41 @@ Base.@kwdef struct Blobentry
     _version::VersionNumber = _getDFGVersion()
 end
 
-StructTypes.StructType(::Type{Blobentry}) = StructTypes.UnorderedStruct()
-StructTypes.idproperty(::Type{Blobentry}) = :id
-StructTypes.omitempties(::Type{Blobentry}) = (:id,)
+function Blobentry(label::Symbol, blobstore = :default; kwargs...)
+    return Blobentry(; label, blobstore, kwargs...)
+end
+# construction helper from existing Blobentry for user overriding via kwargs
+function Blobentry(
+    entry::Blobentry;
+    id::Union{UUID, Nothing} = entry.id,
+    blobId::UUID = entry.blobId,
+    label::Symbol = entry.label,
+    blobstore::Symbol = entry.blobstore,
+    hash::String = entry.hash,
+    size::Union{String, Int, Nothing} = entry.size,
+    origin::String = entry.origin,
+    description::String = entry.description,
+    mimeType::String = entry.mimeType,
+    metadata::String = entry.metadata,
+    timestamp::ZonedDateTime = entry.timestamp,
+    createdTimestamp = entry.createdTimestamp,
+    lastUpdatedTimestamp = entry.lastUpdatedTimestamp,
+    _version = entry._version,
+)
+    return Blobentry(;
+        id,
+        blobId,
+        label,
+        blobstore,
+        hash,
+        origin,
+        size = string(size),
+        description,
+        mimeType,
+        metadata,
+        timestamp,
+        createdTimestamp,
+        lastUpdatedTimestamp,
+        _version,
+    )
+end

src/DataBlobs/services/BlobEntry.jl

@@ -1,7 +1,7 @@
 ##==============================================================================
 ## Blobentry - common
 ##==============================================================================
-
+#TODO think origin and buildSourceString should be deprecated, description can be used instead
 """
     $(SIGNATURES)
 Function to generate source string - agentLabel|graphLabel|varLabel
@@ -329,15 +329,16 @@ If the blob label `datalabel_1` already exists, then this function will return t
 """
 function incrDataLabelSuffix(
     dfg::AbstractDFG,
-    vla::Union{Symbol, <:AbstractString},
-    bllb::S;
+    vla::Symbol,
+    bllb::Union{Symbol, <:AbstractString};
     datalabel = Ref(""),
-) where {S <: Union{Symbol, <:AbstractString}}
+)
     count = 1
     hasund = false
     len = 0
     try
-        de, _ = getData(dfg, Symbol(vla), bllb)
+        de = getfirstBlobentry(dfg, vla; labelFilter = contains(string(bllb)))
+        isnothing(de) && return Symbol(bllb) # no match, return as is
         bllb = string(bllb)
         # bllb *= bllb[end] != '_' ? "_" : ""
         datalabel[] = string(de.label)
@@ -361,5 +362,5 @@ function incrDataLabelSuffix(
     end
     bllb *= string(count)
 
-    return S(bllb)
+    return Symbol(bllb)
 end

src/DataBlobs/services/BlobStores.jl

@@ -123,9 +123,6 @@ struct FolderStore{T} <: AbstractBlobstore{T}
     folder::String
 end
 
-#TODO added in v0.25 to avoid a breaking change in deserialization old DFGs, remove.
-StructTypes.StructType(::Type{<:FolderStore}) = StructTypes.OrderedStruct()
-
 function FolderStore(foldername::String; label::Symbol = :default, createfolder = true)
     storepath = joinpath(foldername, string(label))
     if createfolder && !isdir(storepath)
@@ -367,13 +364,6 @@ function addBlob!(store::RowBlobstore{T}, blobId::UUID, blob::T) where {T}
     return blobId
 end
 
-function updateBlob!(store::RowBlobstore{T}, blobId::UUID, blob::T) where {T}
-    if haskey(store.blobs, blobId)
-        @warn "Key '$blobId' doesn't exist."
-    end
-    return store.blobs[blobId] = RowBlob(blobId, blob)
-end
-
 function deleteBlob!(store::RowBlobstore, blobId::UUID)
     if !haskey(store.blobs, blobId)
         throw(IdNotFoundError("Blob", blobId))