Merge pull request #1691 from vsg-dev/ScavangeResources

Improved DatabasePager automatic load balancing

Author: robertosfield

include/vsg/app/CompileManager.h

@@ -39,6 +39,18 @@ namespace vsg
         bool requiresViewerUpdate() const;
     };
 
+    /// ResourceScavenger provides a mechanism for releasing and reusing unused resources when allocation of required GPU memory fails.
+    class VSG_DECLSPEC ResourceScavenger : public Inherit<Object, ResourceScavenger>
+    {
+    public:
+        explicit ResourceScavenger(ref_ptr<DatabasePager> in_databasePager);
+
+        virtual bool scavenge(ResourceRequirements& resourceRequirements);
+
+        uint64_t sleepDuration = 16 * 5; /// milliseconds sleep to make after adjusting load targets to allow other threads to free up space, default to 5 frames at 60fps
+        observer_ptr<DatabasePager> databasePager;
+    };
+
     /// CompileManager is a helper class that compiles subgraphs for the windows/framebuffers associated with the CompileManager.
     class VSG_DECLSPEC CompileManager : public Inherit<Object, CompileManager>
     {
@@ -71,7 +83,15 @@ namespace vsg
         /// compile all the command graphs in a task
         CompileResult compileTask(ref_ptr<RecordAndSubmitTask> task, const ResourceRequirements& resourceRequirements = {});
 
+        /// mechanism for releasing and reusing used resources
+        ref_ptr<ResourceScavenger> resourceScavenger;
+
+        std::atomic_uint successfulCompileCount{0};
+        std::atomic_uint failedCompileCount{0};
+
     protected:
+        ~CompileManager();
+
         using CompileTraversals = ThreadSafeQueue<ref_ptr<CompileTraversal>>;
         size_t numCompileTraversals = 0;
         ref_ptr<CompileTraversals> compileTraversals;

include/vsg/core/observer_ptr.h

@@ -185,6 +185,9 @@ namespace vsg
                 return {};
         }
 
+        /// We strongly recommend access via ref_ptr<> rather than get(). Only use get() to get access to the raw C pointer when it's know that the object pointed to will remain in memory through use of that raw C pointer.
+        T* get() const noexcept { return _ptr; }
+
     protected:
         template<class R>
         friend class observer_ptr;

include/vsg/core/type_name.h

@@ -49,7 +49,7 @@ namespace vsg
         template<> constexpr const char* vsg::type_name<T>() noexcept { return #T; } \
         template<> constexpr const char* vsg::type_name<const T>() noexcept { return "const "#T; }
 
-    /// convinience function for adding a space in front of the type_name string.
+    /// convenience function for adding a space in front of the type_name string.
     template<typename T>
     std::string space_type_name(const T&) { return std::string(" ") + type_name<T>(); }
 

include/vsg/io/DatabasePager.h

@@ -104,7 +104,10 @@ namespace vsg
 
         void add(ref_ptr<PagedLOD> plod, const CompileResult& cr);
 
-        ref_ptr<PagedLOD> take_when_available();
+        /// prune entries older than specified frameCount
+        uint32_t prune(uint64_t frameCount);
+
+        ref_ptr<PagedLOD> take_when_available(uint64_t frameCount);
 
         Nodes take_all(CompileResult& result);
 
@@ -138,13 +141,16 @@ namespace vsg
         ref_ptr<CompileManager> compileManager;
 
         std::atomic_uint numActiveRequests{0};
-        std::atomic_uint64_t frameCount;
+        std::atomic_uint64_t frameCount{0};
 
         ref_ptr<CulledPagedLODs> culledPagedLODs;
 
         /// for systems with smaller GPU memory limits you may need to reduce the targetMaxNumPagedLODWithHighResSubgraphs to keep memory usage within available limits.
         uint32_t targetMaxNumPagedLODWithHighResSubgraphs = 1500;
 
+        /// number of frames before a PagedLOD with a failed load/compile is attempted to be loaded/compiled again.
+        uint64_t delayBeforeNextLoadAttempt = 60;
+
         std::mutex pendingPagedLODMutex;
 
         ref_ptr<PagedLODContainer> pagedLODContainer;
@@ -158,16 +164,16 @@ namespace vsg
         /// read and delete threads created by start()
         std::list<std::thread> threads;
 
+        ref_ptr<ActivityStatus> status;
+        ref_ptr<DeleteQueue> deleteQueue;
+
     protected:
         virtual ~DatabasePager();
 
         void requestDiscarded(PagedLOD* plod);
 
-        ref_ptr<ActivityStatus> _status;
-
         ref_ptr<DatabaseQueue> _requestQueue;
         ref_ptr<DatabaseQueue> _toMergeQueue;
-        ref_ptr<DeleteQueue> _deleteQueue;
     };
     VSG_type_name(vsg::DatabasePager);
 

include/vsg/nodes/InstanceNode.h

@@ -21,10 +21,10 @@ namespace vsg
 {
 
     /// InstanceNode provides a mechanism for specifying the translations, rotations and scales (transform arrays) of subgraph
-    /// that contains InstanceDraw leaf node(s) that utlize the InstanceNode's per instance transform arrays combined with the
+    /// that contains InstanceDraw leaf node(s) that utilize the InstanceNode's per instance transform arrays combined with the
     /// InstanceDraw nodes per vertex arrays.
     ///
-    /// InstanceNode only work correctly when the child subgraphs that obey these contraints:
+    /// InstanceNode only work correctly when the child subgraphs that obey these constraints:
     /// 1. Do not contain any Transform nodes
     /// 2. Do not contain any Culling nodes
     /// 3. Do not contain any InstanceNode nodes

include/vsg/nodes/PagedLOD.h

@@ -89,6 +89,9 @@ namespace vsg
         mutable std::atomic_uint64_t frameHighResLastUsed{0};
         mutable std::atomic_uint requestCount{0};
 
+        mutable std::atomic_uint64_t frameNextLoadAttempt{0};
+        mutable std::atomic_uint64_t loadAttempts{0};
+
         enum RequestStatus : unsigned int
         {
             NoRequest = 0,

include/vsg/threading/DeleteQueue.h

@@ -37,6 +37,7 @@ namespace vsg
 
         std::atomic_uint64_t frameCount = 0;
         uint64_t retainForFrameCount = 3;
+        std::atomic_uint64_t deletedCount = 0;
 
         ActivityStatus* getStatus() { return _status; }
         const ActivityStatus* getStatus() const { return _status; }

include/vsg/vk/Device.h

@@ -85,8 +85,8 @@ namespace vsg
         /// return true if Device was created with specified extension
         bool supportsDeviceExtension(const char* extensionName) const;
 
-        /// return the amount of memory available in deviceMemoryBufferPools and allocatable on device
-        VkDeviceSize availableMemory(bool includeMemoryPools = true) const;
+        /// return the amount of remaining memory, compatible with specified flags, available that can be allocated.
+        VkDeviceSize availableMemory(VkMemoryPropertyFlags memoryPropertiesFlags, double allocatedMemoryLimit = 1.0) const;
 
         // provide observer_ptr to memory buffer, descriptor pools and transferTask so that these can be accessed when required
         observer_ptr<MemoryBufferPools> deviceMemoryBufferPools;

include/vsg/vk/DeviceMemory.h

@@ -44,6 +44,7 @@ namespace vsg
         const VkMemoryPropertyFlags& getMemoryPropertyFlags() const { return _properties; }
 
         MemorySlots::OptionalOffset reserve(VkDeviceSize size);
+        MemorySlots::OptionalOffset reserve(VkDeviceSize size, VkDeviceSize alignment);
         void release(VkDeviceSize offset, VkDeviceSize size);
 
         bool full() const;

src/vsg/app/CompileManager.cpp

@@ -21,6 +21,10 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 
 using namespace vsg;
 
+////////////////////////////////////////////////////////////////////////////////////////////////////
+//
+// CompileResult
+//
 void CompileResult::reset()
 {
     result = VK_INCOMPLETE;
@@ -66,6 +70,51 @@ bool CompileResult::requiresViewerUpdate() const
     return false;
 }
 
+////////////////////////////////////////////////////////////////////////////////////////////////////
+//
+// CompileManager
+//
+
+ResourceScavenger::ResourceScavenger(ref_ptr<DatabasePager> in_databasePager) :
+    databasePager(in_databasePager)
+{
+}
+
+bool ResourceScavenger::scavenge(ResourceRequirements& /*resourceRequirements*/)
+{
+    bool scavenged = false;
+
+    // get raw C pointer to avoid a database pager thread invoking scavenger and keeping the database pager alive and pausing destruction
+    if (auto ref_databasePager = databasePager.get())
+    {
+        if (!ref_databasePager->status->active()) return false;
+
+        uint32_t targetPagedLOD = ref_databasePager->pagedLODContainer->activeList.count;
+        if (ref_databasePager->pagedLODContainer->inactiveList.count > ref_databasePager->numActiveRequests) targetPagedLOD += ref_databasePager->pagedLODContainer->inactiveList.count - ref_databasePager->numActiveRequests;
+
+        if (targetPagedLOD < ref_databasePager->targetMaxNumPagedLODWithHighResSubgraphs)
+        {
+            debug("ResourceScavenger::scavenge(..) resetting databasePager->targetMaxNumPagedLODWithHighResSubgraphs to ", targetPagedLOD);
+
+            ref_databasePager->targetMaxNumPagedLODWithHighResSubgraphs = targetPagedLOD;
+        }
+
+        auto before_deletedCount = ref_databasePager->deleteQueue->deletedCount.load();
+
+        if (sleepDuration > 0) std::this_thread::sleep_for(std::chrono::milliseconds(sleepDuration));
+
+        auto after_deletedCount = ref_databasePager->deleteQueue->deletedCount.load();
+
+        scavenged = (after_deletedCount > before_deletedCount);
+    }
+
+    return scavenged;
+}
+
+////////////////////////////////////////////////////////////////////////////////////////////////////
+//
+// CompileManager
+//
 CompileManager::CompileManager(Viewer& viewer, ref_ptr<ResourceHints> hints)
 {
     compileTraversals = CompileTraversals::create(viewer.status);
@@ -84,6 +133,11 @@ CompileManager::CompileManager(Viewer& viewer, ref_ptr<ResourceHints> hints)
 #endif
 }
 
+CompileManager::~CompileManager()
+{
+    vsg::info("CompileManager::~CompileManager() successfulCompileCount= ", successfulCompileCount, ", failedCompileCount = ", failedCompileCount);
+}
+
 CompileManager::CompileTraversals::container_type CompileManager::takeCompileTraversals(size_t count)
 {
     CompileTraversals::container_type cts;
@@ -167,6 +221,7 @@ CompileResult CompileManager::compile(ref_ptr<Object> object, ContextSelectionFu
     auto& requirements = collectRequirements.requirements;
     auto& viewDetailsStack = requirements.viewDetailsStack;
 
+    VkResult reserve_result = VK_INCOMPLETE;
     CompileResult result;
     result.maxSlots = requirements.maxSlots;
     result.containsPagedLOD = requirements.containsPagedLOD;
@@ -199,9 +254,24 @@ CompileResult CompileManager::compile(ref_ptr<Object> object, ContextSelectionFu
                         }
                     }
                 }
+            }
 
-                auto reserveResult = context->reserve(requirements);
-                if (reserveResult != VK_SUCCESS) throw vsg::Exception{"Context::reserve() failed", reserveResult};
+            for (auto& context : compileTraversal->contexts)
+            {
+                reserve_result = context->reserve(requirements);
+
+                // vsg::info("  done reserve context->reserve() ",  reserve_result);
+                if (reserve_result != VK_SUCCESS && resourceScavenger && resourceScavenger->scavenge(requirements))
+                {
+                    reserve_result = context->reserve(requirements);
+                }
+
+                if (reserve_result != VK_SUCCESS)
+                {
+                    result.message = vsg::make_string("Context::reserve() failed", reserve_result);
+                    result.result = reserve_result;
+                    return;
+                }
             }
 
             object->accept(*compileTraversal);
@@ -251,6 +321,15 @@ CompileResult CompileManager::compile(ref_ptr<Object> object, ContextSelectionFu
 
     compileTraversals->add(compileTraversal);
 
+    if (result.result == VK_SUCCESS)
+    {
+        ++successfulCompileCount;
+    }
+    else
+    {
+        ++failedCompileCount;
+    }
+
     return result;
 }