Standard filters for listVariables and getVariables

Author: Affie

src/Common.jl

@@ -69,6 +69,63 @@ function sortDFG(vars::Vector{<:AbstractGraphNode}; by = getTimestamp, kwargs...
 end
 sortDFG(vars::Vector{Symbol}; lt = natural_lt, kwargs...) = sort(vars; lt = lt, kwargs...)
 
+##==============================================================================
+## Filtering
+##==============================================================================
+
+# std_numeric_predicates = [==, <, <=, >, >=, in]
+# std_string_predicates = [==, in, contains, startswith, endswith]
+
+# # full list
+# tags_includes(tag::Symbol) = Base.Fix1(in, tag)
+# solvable_eq(x::Int) = ==(x)
+# solvable_in(x::Vector{Int}) = in(x)
+# solvable_lt(x::Int) = <(x)
+# solvable_lte(x::Int) = <=(x)
+# solvable_gt(x::Int) = >(x)
+# solvable_gte(x::Int) = >=(x)
+# label_in(x::Vector{Symbol}) = in(x)
+# label_contains(x::String) = contains(x)
+# label_startswith(x::String) = startswith(x)
+# label_endswith(x::String) = endswith(x)
+# type_eq(x::AbstractVariableStateType) = ==(x)
+# type_in(x::Vector{<:AbstractVariableStateType}) = in(x)
+# type_contains(x::String) = contains(x)
+# type_startswith(x::String) = startswith(x)
+# type_endswith(x::String) = endswith(x)
+
+# Set predicates
+# collection_includes(item) = Base.Fix1(in, item) # collection includes item (item in collection)
+
+# not supported helper
+# collection_overlap(collection) = !isdisjoint(collection) # collection overlaps with another collection
+
+"""
+    $SIGNATURES
+Filter nodes in a DFG based on a predicate function.
+This function modifies the input `nodes` vector in place, removing nodes that do not satisfy the predicate.
+
+- **For cross-backend compatibility:**  
+  Use only the standard predicates (`==`, `<`, `<=`, `>`, `>=`, `in`, `contains`, `startswith`, `endswith`) 
+  when you need your code to work with both in-memory and database-backed DFGs. 
+  These are likely to be supported by database query languages and are defined in `std_numeric_predicates` and `std_string_predicates`.
+
+- **For in-memory only operations:**  
+  You can use any Julia predicate, since you have full access to the data and Julia's capabilities.
+  This is more flexible but will not work if you later switch to a database backend or another programming language.
+
+Standard predicates
+- Numeric predicates: `==`, `<`, `<=`, `>`, `>=`, `in`
+- String predicates: `==`, `contains`, `startswith`, `endswith`, `in`
+"""
+function filterDFG! end
+
+filterDFG!(nodes, predicate::Nothing, by::Function = identity) = nodes
+# function filterDFG!(nodes, predicate::Base.Fix2, by=identity)
+function filterDFG!(nodes, predicate::Function, by::Function = identity)
+    return filter!(predicate ∘ by, nodes)
+end
+
 ##==============================================================================
 ## Validation of session, robot, and user labels.
 ##==============================================================================
@@ -83,10 +140,8 @@ $(SIGNATURES)
 Returns true if the label is valid for node.
 """
 function isValidLabel(id::Union{Symbol, String})
-    if typeof(id) == Symbol
-        id = String(id)
-    end
-    return !in(uppercase(id), _invalidIds) && !isnothing(match(_validLabelRegex, id))
+    _id = string(id)
+    return occursin(_validLabelRegex, _id) && !in(uppercase(_id), _invalidIds)
 end
 
 """

src/DistributedFactorGraphs.jl

@@ -337,7 +337,7 @@ export pack, unpack, packDistribution, unpackDistribution
 export natural_lt, sortDFG
 
 # Validation
-export isValidLabel
+# export isValidLabel
 
 ## List
 export ls, lsf, ls2
@@ -431,6 +431,9 @@ include("DataBlobs/services/BlobStores.jl")
 include("DataBlobs/services/BlobPacking.jl")
 include("DataBlobs/services/HelpersDataWrapEntryBlob.jl")
 
+# To be moved as necessary.
+include("Common.jl")
+
 # In Memory Types
 include("GraphsDFG/GraphsDFG.jl")
 using .GraphsDFGs
@@ -453,9 +456,6 @@ include("FileDFG/FileDFG.jl")
 # Custom show and printing for variable factor etc.
 include("services/CustomPrinting.jl")
 
-# To be moved as necessary.
-include("Common.jl")
-
 include("weakdeps_prototypes.jl")
 
 #TODO start off as just an alias before deprecating

src/GraphsDFG/GraphsDFG.jl

@@ -15,7 +15,8 @@ using ...DistributedFactorGraphs:
     FactorgraphRoot,
     AbstractGraphVariable,
     AbstractGraphFactor,
-    NoSolverParams
+    NoSolverParams,
+    filterDFG!
 
 # import DFG functions to extend
 import ...DistributedFactorGraphs:

src/GraphsDFG/entities/GraphsDFG.jl

@@ -30,7 +30,9 @@ function DFG.setMetadata!(dfg::GraphsDFG, metadata::Dict{Symbol, SmallDataTypes}
     return merge!(dfg.graph.metadata, metadata)
 end
 
-DFG.setDescription!(dfg::GraphsDFG, description::String) = dfg.graph.description = description
+function DFG.setDescription!(dfg::GraphsDFG, description::String)
+    return dfg.graph.description = description
+end
 
 """
     $(SIGNATURES)
@@ -75,8 +77,8 @@ function GraphsDFG{T, V, F}(
 ) where {T <: AbstractDFGParams, V <: AbstractGraphVariable, F <: AbstractGraphFactor}
 
     # Validate the graphLabel and agentLabel
-    !isValidLabel(graphLabel) && error("'$graphLabel' is not a valid label")
-    !isValidLabel(agentLabel) && error("'$agentLabel' is not a valid label")
+    !DFG.isValidLabel(graphLabel) && error("'$graphLabel' is not a valid label")
+    !DFG.isValidLabel(agentLabel) && error("'$agentLabel' is not a valid label")
 
     return GraphsDFG{T, V, F}(
         g,

src/GraphsDFG/services/GraphsDFG.jl

@@ -128,25 +128,56 @@ function deleteFactor!(dfg::GraphsDFG, label::Symbol; suppressGetFactor::Bool =
     return 1
 end
 
+# """
+# tagsFilter = ⊇([:x1])
+# tagsFilter([:x1, :x2])
+# true
+# tagsFilter = Base.Fix1(in, :x1)
+# tagsFilter([:x1, :x2])
+# true
+# """
+
 function getVariables(
     dfg::GraphsDFG,
-    regexFilter::Union{Nothing, Regex} = nothing;
+    regex::Union{Nothing, Regex} = nothing;
     tags::Vector{Symbol} = Symbol[],
-    solvable::Int = 0,
-    solvableFilter::Union{Nothing, Base.Fix2} = nothing,
+    solvable::Union{Nothing, Int} = nothing,
+    solvableFilter::Union{Nothing, Function} = nothing,
+    labelFilter::Union{Nothing, Function} = nothing,
+    tagsFilter::Union{Nothing, Function} = nothing,
+    typeFilter::Union{Nothing, Function} = nothing,
 )
-
-    # variables = map(v -> v.dfgNode, filter(n -> n.dfgNode isa VariableCompute, vertices(dfg.g)))
     variables = collect(values(dfg.g.variables))
 
-    !isnothing(regexFilter) &&
-        filter!(v -> occursin(regexFilter, String(v.label)), variables)
-
-    solvable != 0 && filter!(v -> _isSolvable(dfg, v.label, solvable), variables)
-
-    !isempty(tags) && filter!(v -> !isempty(intersect(v.tags, tags)), variables)
+    if !isnothing(regex)
+        # NOTE that contains(regex::Regex) is not supported by the NvaDFG.
+        Base.depwarn(
+            "The regex filter argument is deprecated, use kwarg `labelFilter=contains(regex)` instead", #v0.28
+            :getVariable,
+        )
+        filterDFG!(variables, contains(regex), (String ∘ getLabel))
+    end
+    if !isempty(tags)
+        # NOTE that !isdisjoint is not supported by NvaDFG.
+        Base.depwarn(
+            "tags kwarg is deprecated, use kwarg `tagsFilter = !isdisjoint(tags)`` instead", #v0.28
+            :getVariable,
+        )
+        filterDFG!(variables, !isdisjoint(tags), getTags)
+    end
+    if !isnothing(solvable)
+        #TODO review. just one solvableFilter or keep solvable as well.
+        Base.depwarn(
+            "solvable kwarg is deprecated, use kwarg `solvableFilter = (>=solvable)`` instead", #v0.28
+            :getVariable,
+        )
+        filterDFG!(variables, >=(solvable), getSolvable)
+    end
 
-    !isnothing(solvableFilter) && filter!(v -> solvableFilter(getSolvable(v)), variables)
+    filterDFG!(variables, labelFilter, (String ∘ getLabel))
+    filterDFG!(variables, solvableFilter, getSolvable)
+    filterDFG!(variables, tagsFilter, getTags)
+    filterDFG!(variables, typeFilter, getVariableType)
 
     return variables
 end
@@ -155,23 +186,36 @@ function listVariables(
     dfg::GraphsDFG,
     regexFilter::Union{Nothing, Regex} = nothing;
     tags::Vector{Symbol} = Symbol[],
-    solvable::Int = 0,
-    solvableFilter::Union{Nothing, Base.Fix2} = nothing,
+    solvable::Union{Nothing, Int} = nothing,
+    solvableFilter::Union{Nothing, Function} = nothing,
+    tagsFilter::Union{Nothing, Function} = nothing,
+    typeFilter::Union{Nothing, Function} = nothing,
+    labelFilter::Union{Nothing, Function} = nothing,
 )
-
-    # variables = map(v -> v.dfgNode, filter(n -> n.dfgNode isa VariableCompute, vertices(dfg.g)))
-    if length(tags) > 0
+    if !isnothing(solvableFilter) ||
+       !isnothing(tagsFilter) ||
+       !isnothing(typeFilter) ||
+       !isnothing(regexFilter) ||  #TODO deprecated
+       !isempty(tags) ||           #TODO deprecated
+       !isnothing(solvable)        #TODO Maybe deprecated?
         return map(
-            v -> v.label,
-            getVariables(dfg, regexFilter; tags = tags, solvable = solvable),
-        )::Vector{Symbol}
+            getLabel,
+            getVariables(
+                dfg,
+                regexFilter;
+                tags,
+                solvable,
+                solvableFilter,
+                tagsFilter,
+                typeFilter,
+                labelFilter,
+            ),
+        )
     else
-        variables = copy(dfg.g.variables.keys)
-        !isnothing(regexFilter) && filter!(v -> occursin(regexFilter, String(v)), variables)
-        solvable != 0 && filter!(vId -> _isSolvable(dfg, vId, solvable), variables)
-        !isnothing(solvableFilter) &&
-            filter!(v -> solvableFilter(getSolvable(dfg, v)), variables)
-        return variables::Vector{Symbol}
+        # Is it ok to continue using the internal keys property? collect(keys(dfg.g.variables)) allowcates a lot.
+        labels = copy(dfg.g.variables.keys)
+        filterDFG!(labels, labelFilter, string)
+        return labels
     end
 end
 

src/services/AbstractDFG.jl

@@ -393,9 +393,22 @@ function deleteFactor! end
 
 """
     $(SIGNATURES)
-List the DFGVariables in the DFG.
-Optionally specify a label regular expression to retrieves a subset of the variables.
-Tags is a list of any tags that a node must have (at least one match).
+Get the variables in the DFG as a Vector, supporting various filters.
+
+Arguments
+- `regexFilt`: Optional Regex to filter variable labels (deprecated, use `labelFilter` instead).
+Keyword arguments
+- `tags`: Vector of tags; only variables with at least one matching tag are returned.
+- `solvable`: Optional Int; only variables with `solvable >= solvable` are returned.
+- `solvableFilter`: Optional function to filter on the `solvable` property, eg `>=(1)`.
+- `labelFilter`: Optional function to filter on label e.g., `contains(r"x1")`.
+- `tagsFilter`: Optional function to filter on tags, eg. `⊇([:x1])`.
+- `typeFilter`: Optional function to filter on the variable type.
+
+Returns
+- `Vector{<:AbstractGraphVariable}` matching the filters.
+
+See also: [`listVariables`](@ref), [`ls`](@ref)
 """
 function getVariables end
 
@@ -665,11 +678,11 @@ Notes:
 - Returns `Vector{Symbol}`
 """
 function ls(
-    dfg::G,
+    dfg::AbstractDFG,
     regexFilter::Union{Nothing, Regex} = nothing;
     tags::Vector{Symbol} = Symbol[],
     solvable::Int = 0,
-) where {G <: AbstractDFG}
+)
     return listVariables(dfg, regexFilter; tags = tags, solvable = solvable)
 end
 

src/services/CompareUtils.jl

@@ -544,7 +544,7 @@ function compareFactorGraphs(
         :solverParams,
         :factorOperationalMemoryType,
         :agent,
-        :graph
+        :graph,
     ]
     skiplist = union(skiplist, skip)
     @warn "compareFactorGraphs will skip comparisons on: $skiplist"