GPUUploadManager: split uploads into normal and large streams

Author: TheMostDiligent

Graphics/GraphicsTools/include/GPUUploadManagerImpl.hpp

@@ -38,6 +38,7 @@
 
 #include <memory>
 #include <vector>
+#include <array>
 #include <mutex>
 #include <map>
 #include <unordered_map>
@@ -70,7 +71,7 @@ class GPUUploadManagerImpl final : public ObjectBase<IGPUUploadManager>
     public:
         explicit Page(Uint32 Size, bool PersistentMapped = false) noexcept;
 
-        Page(IRenderDevice* pDevice, Uint32 Size);
+        Page(size_t StreamIndex, IRenderDevice* pDevice, Uint32 Size);
         ~Page();
 
         enum class WritingStatus
@@ -142,6 +143,8 @@ class GPUUploadManagerImpl final : public ObjectBase<IGPUUploadManager>
         // Returns true if the page was not previously enqueued, false otherwise.
         bool TryEnqueue();
 
+        size_t GetStreamIndex() const { return m_StreamIdx; }
+
         Uint64 GetFenceValue() const { return m_FenceValue; }
         Uint32 GetSize() const { return m_Size; }
 
@@ -179,6 +182,7 @@ class GPUUploadManagerImpl final : public ObjectBase<IGPUUploadManager>
     private:
         const Uint32 m_Size             = 0;
         const bool   m_PersistentMapped = false;
+        const size_t m_StreamIdx        = 0;
 
         RefCntAutoPtr<IBuffer> m_pStagingBuffer;
 
@@ -238,26 +242,14 @@ class GPUUploadManagerImpl final : public ObjectBase<IGPUUploadManager>
     };
 
 private:
-    void ScheduleUpdate(IDeviceContext* pContext,
-                        Uint32          UpdateSize,
-                        const void*     pUpdateInfo,
-                        bool            ScheduleUpdate(Page::Writer& Writer, const void* pUpdateInfo));
-
-    void  ReclaimCompletedPages(IDeviceContext* pContext);
-    bool  SealAndSwapCurrentPage(IDeviceContext* pContext);
-    void  AddFreePages(IDeviceContext* pContext);
-    void  ProcessPendingPages(IDeviceContext* pContext);
-    bool  TryRotatePage(IDeviceContext* pContext, Page* ExpectedCurrent, Uint32 RequiredSize);
-    bool  TryEnqueuePage(Page* P);
-    Page* AcquireFreePage(IDeviceContext* pContext, Uint32 RequiredSize = 0);
-    Page* CreatePage(IDeviceContext* pContext, Uint32 RequiredSize = 0);
-    void  ProcessPagesToRelease(IDeviceContext* pContext);
-    void  UpdateBucketInfo();
+    class UploadStream;
 
-private:
-    const Uint32 m_PageSize;
-    const Uint32 m_MaxPageCount;
+    void ReclaimCompletedPages(IDeviceContext* pContext);
+    void ProcessPendingPages(IDeviceContext* pContext);
+
+    UploadStream& GetStreamForUpdateSize(Uint32 UpdateSize);
 
+private:
     RefCntAutoPtr<IRenderDevice>  m_pDevice;
     RefCntAutoPtr<IDeviceContext> m_pContext;
 
@@ -281,7 +273,6 @@ class GPUUploadManagerImpl final : public ObjectBase<IGPUUploadManager>
         std::map<Uint32, std::vector<Page*>> m_PagesBySize;
         std::atomic<size_t>                  m_Size{0};
     };
-    FreePages m_FreePages;
 
     // Pages that have been submitted for execution and are being processed by the GPU.
     std::vector<Page*> m_InFlightPages;
@@ -290,24 +281,75 @@ class GPUUploadManagerImpl final : public ObjectBase<IGPUUploadManager>
     RefCntAutoPtr<IFence> m_pFence;
     Uint64                m_NextFenceValue = 1;
 
-    std::atomic<Page*>    m_pCurrentPage{nullptr};
-    Threading::TickSignal m_PageRotatedSignal;
+    class UploadStream
+    {
+    public:
+        UploadStream(GPUUploadManagerImpl& Mgr,
+                     size_t                StreamIdx,
+                     Uint32                PageSize,
+                     Uint32                MaxPageCount,
+                     Uint32                InitialPageCount) noexcept;
+
+        Page* CreatePage(IDeviceContext* pContext, Uint32 RequiredSize = 0);
+        Page* AcquireFreePage(IDeviceContext* pContext, Uint32 RequiredSize = 0);
+        bool  SealAndSwapCurrentPage(IDeviceContext* pContext);
+        bool  TryRotatePage(IDeviceContext* pContext, Page* ExpectedCurrent, Uint32 RequiredSize);
+        bool  TryEnqueuePage(Page* P);
+        void  ProcessPagesToRelease(IDeviceContext* pContext);
+        void  AddFreePages(IDeviceContext* pContext);
+        void  AddFreePage(Page* pPage) { m_FreePages.Push(pPage); }
+
+        void ScheduleUpdate(IDeviceContext* pContext,
+                            Uint32          UpdateSize,
+                            const void*     pUpdateInfo,
+                            bool            ScheduleUpdate(Page::Writer& Writer, const void* pUpdateInfo));
+        void ReleaseStagingBuffers();
+        void SignalPageRotated() { m_PageRotatedSignal.Tick(); }
+        void SignalStop();
+
+        Uint32 GetPageSize() const { return m_PageSize; }
+
+        void GetStats(GPUUploadManagerStreamStats& Stats) const;
+
+    private:
+        GPUUploadManagerImpl& m_Mgr;
+        const size_t          m_StreamIdx;
+        const Uint32          m_PageSize;
+        const Uint32          m_MaxPageCount;
+
+        std::atomic<Page*> m_pCurrentPage{nullptr};
 
-    std::unordered_map<Page*, std::unique_ptr<Page>> m_Pages;
-    std::map<Uint32, Uint32>                         m_PageSizeToCount;
-    std::vector<GPUUploadManagerBucketInfo>          m_BucketInfo;
+        Threading::TickSignal m_PageRotatedSignal;
+
+        std::unordered_map<Page*, std::unique_ptr<Page>> m_Pages;
+        std::map<Uint32, Uint32>                         m_PageSizeToCount;
+        mutable std::vector<GPUUploadManagerBucketInfo>  m_BucketInfo;
+
+        std::atomic<Uint32> m_NumRunningUpdates{0};
+        std::atomic<Uint32> m_MaxPendingUpdateSize{0};
+        std::atomic<Uint32> m_TotalPendingUpdateSize{0};
+
+        FreePages m_FreePages;
+
+        std::atomic<Uint32> m_PeakUpdateSize{0};
+        Uint32              m_PeakTotalPendingUpdateSize = 0;
+        Uint32              m_PeakPageCount              = 0;
+    };
+
+    enum class UploadStreamType : Uint32
+    {
+        Normal = 0,
+        Large  = 1,
+        Count
+    };
+    std::array<UploadStream, static_cast<size_t>(UploadStreamType::Count)> m_Streams;
 
     // The number of running ScheduleBufferUpdate operations.
     std::atomic<Uint32> m_NumRunningUpdates{0};
     std::atomic<bool>   m_Stopping{false};
     Threading::Signal   m_LastRunningThreadFinishedSignal;
 
-    std::atomic<Uint32> m_MaxPendingUpdateSize{0};
-    std::atomic<Uint32> m_TotalPendingUpdateSize{0};
-
-    std::atomic<Uint32> m_PeakUpdateSize{0};
-    Uint32              m_PeakTotalPendingUpdateSize = 0;
-    Uint32              m_PeakPageCount              = 0;
+    mutable std::array<GPUUploadManagerStreamStats, static_cast<size_t>(UploadStreamType::Count)> m_StreamStats;
 };
 
 } // namespace Diligent

Graphics/GraphicsTools/interface/GPUUploadManager.h

@@ -42,19 +42,30 @@ struct GPUUploadManagerCreateInfo
     /// Pointer to the device context. Must not be null.
     IDeviceContext* pContext DEFAULT_INITIALIZER(nullptr);
 
-    /// Size of the staging buffer page.
+    /// Size of the upload page.
     Uint32 PageSize DEFAULT_INITIALIZER(4 * 1024 * 1024);
 
+    /// Size of the large upload page.
+    Uint32 LargePageSize DEFAULT_INITIALIZER(16 * 1024 * 1024);
+
     /// Initial number of upload pages. If the manager runs out of pages to write to,
     /// it will create new ones as needed. This parameter controls how many pages are created at startup.
     Uint32 InitialPageCount DEFAULT_INITIALIZER(1);
 
-    /// Maximum number of pages that the manager should maintain.
+    /// Initial number of large upload pages.
+    Uint32 InitialLargePageCount DEFAULT_INITIALIZER(1);
+
+    /// Maximum number of pages that the manager should maintain. If 0, there is no limit to the number
+    /// of pages that can be created.
     ///
     /// Note the manager may temporarily exceed this limit in certain scenarios (for example, if large
     /// update is scheduled while the maximum is already reached), but it will reduce the number of
     /// pages to the maximum as soon as possible.
     Uint32 MaxPageCount DEFAULT_INITIALIZER(64);
+
+    /// Maximum number of large pages that the manager should maintain. If 0, there is no limit to the number
+    /// of large pages that can be created.
+    Uint32 MaxLargePageCount DEFAULT_INITIALIZER(16);
 };
 typedef struct GPUUploadManagerCreateInfo GPUUploadManagerCreateInfo;
 
@@ -408,18 +419,18 @@ struct GPUUploadManagerBucketInfo
 typedef struct GPUUploadManagerBucketInfo GPUUploadManagerBucketInfo;
 
 
-/// GPU upload manager statistics.
-struct GPUUploadManagerStats
+/// GPU upload manager stream statistics.
+struct GPUUploadManagerStreamStats
 {
+    /// Page size in bytes for this stream.
+    Uint32 PageSize DEFAULT_INITIALIZER(0);
+
     /// The number of pages in the manager.
     Uint32 NumPages DEFAULT_INITIALIZER(0);
 
     /// The number of free pages that are ready to be written to.
     Uint32 NumFreePages DEFAULT_INITIALIZER(0);
 
-    /// The number of pages that are currently being used by the GPU for copy operations.
-    Uint32 NumInFlightPages DEFAULT_INITIALIZER(0);
-
     /// The peak number of pages that were created by the manager. This value can exceed the maximum page count,
     /// but only temporarily when the manager needs to create new pages to accommodate large updates.
     Uint32 PeakNumPages DEFAULT_INITIALIZER(0);
@@ -435,11 +446,23 @@ struct GPUUploadManagerStats
     Uint32 NumBuckets DEFAULT_INITIALIZER(0);
 
     /// Information about each bucket. The array contains NumBuckets valid entries.
-    /// The pointer is valid only until the next call to RenderThreadUpdate() or
-    /// ScheduleBufferUpdate() with a non-null device context, which may change the number
-    /// of buckets.
+    /// The pointer is valid only until the next call to GetStats().
     const GPUUploadManagerBucketInfo* pBucketInfo DEFAULT_INITIALIZER(nullptr);
 };
+typedef struct GPUUploadManagerStreamStats GPUUploadManagerStreamStats;
+
+/// GPU upload manager statistics.
+struct GPUUploadManagerStats
+{
+    /// An array of NumStreams stream statistics.
+    const GPUUploadManagerStreamStats* pStreamStats DEFAULT_INITIALIZER(nullptr);
+
+    /// The number of streams in the pStreamStats array.
+    Uint32 NumStreams DEFAULT_INITIALIZER(0);
+
+    /// The number of pages that are currently being used by the GPU for copy operations.
+    Uint32 NumInFlightPages DEFAULT_INITIALIZER(0);
+};
 typedef struct GPUUploadManagerStats GPUUploadManagerStats;
 
 
@@ -518,6 +541,10 @@ DILIGENT_END_INTERFACE
 
 // clang-format on
 
+#else
+
+std::string GetManagerStatsString(const GPUUploadManagerStats& Stats);
+
 #endif
 
 /// Creates an instance of the GPU upload manager.