Improve cmd buffer sync

- Use semaphores to sync cmd buffers in the same frame

Author: xCollateral

src/main/java/net/vulkanmod/render/texture/ImageUploadHelper.java

@@ -21,8 +21,8 @@ public void submitCommands() {
             return;
         }
 
-        long fence = queue.submitCommands(this.currentCmdBuffer);
-        Synchronization.INSTANCE.addCommandBuffer(this.currentCmdBuffer);
+        queue.submitCommands(this.currentCmdBuffer, true);
+        Synchronization.INSTANCE.addCommandBuffer(this.currentCmdBuffer, true);
 
         this.currentCmdBuffer = null;
     }

src/main/java/net/vulkanmod/vulkan/Renderer.java

@@ -338,9 +338,20 @@ private void submitFrame() {
             VkSubmitInfo submitInfo = VkSubmitInfo.calloc(stack);
             submitInfo.sType(VK_STRUCTURE_TYPE_SUBMIT_INFO);
 
-            submitInfo.waitSemaphoreCount(1);
-            submitInfo.pWaitSemaphores(stack.longs(imageAvailableSemaphores.get(currentFrame)));
-            submitInfo.pWaitDstStageMask(stack.ints(VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT));
+            Synchronization.INSTANCE.addWaitSemaphore(imageAvailableSemaphores.get(currentFrame));
+            var waitSemaphores = Synchronization.INSTANCE.getWaitSemaphores(stack);
+            int waitSemaphoreCount = waitSemaphores.limit();
+            IntBuffer waitDstStageMask = stack.mallocInt(waitSemaphoreCount);
+
+            for (int i = 0; i < waitSemaphoreCount - 1; i++) {
+                waitDstStageMask.put(i, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT);
+            }
+            // Image available semaphore mask
+            waitDstStageMask.put(waitSemaphoreCount - 1, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT);
+
+            submitInfo.pWaitSemaphores(waitSemaphores);
+            submitInfo.waitSemaphoreCount(waitSemaphores.limit());
+            submitInfo.pWaitDstStageMask(waitDstStageMask);
             submitInfo.pSignalSemaphores(stack.longs(renderFinishedSemaphores.get(currentFrame)));
             submitInfo.pCommandBuffers(stack.pointers(currentCmdBuffer));
 
@@ -370,6 +381,9 @@ private void submitFrame() {
                 throw new RuntimeException("Failed to present rendered frame: %s".formatted(VkResult.decode(vkResult)));
             }
 
+            // Semaphore waited command buffers will be reset right after waiting this command buffer's fence
+            Synchronization.INSTANCE.scheduleCbReset();
+
             currentFrame = (currentFrame + 1) % framesNum;
         }
     }
@@ -414,10 +428,10 @@ public void submitUploads() {
         if (transferCb.isRecording()) {
             final var transferQueue = DeviceManager.getTransferQueue();
             try (MemoryStack stack = MemoryStack.stackPush()) {
-                transferCb.submitCommands(stack, transferQueue.vkQueue(), false);
+                transferCb.submitCommands(stack, transferQueue.vkQueue(), true);
             }
 
-            Synchronization.INSTANCE.addCommandBuffer(transferCb);
+            Synchronization.INSTANCE.addCommandBuffer(transferCb, true);
 
             transferCbs.set(currentFrame, transferQueue.getCommandPool().getCommandBuffer());
         }

src/main/java/net/vulkanmod/vulkan/Synchronization.java

@@ -1,15 +1,21 @@
 package net.vulkanmod.vulkan;
 
+import it.unimi.dsi.fastutil.longs.LongArrayList;
 import it.unimi.dsi.fastutil.objects.ObjectArrayList;
+import net.vulkanmod.vulkan.memory.MemoryManager;
 import net.vulkanmod.vulkan.queue.CommandPool;
 import net.vulkanmod.vulkan.util.VUtil;
+import org.lwjgl.system.MemoryStack;
 import org.lwjgl.system.MemoryUtil;
 import org.lwjgl.vulkan.VkDevice;
 
 import java.nio.LongBuffer;
 
 import static org.lwjgl.vulkan.VK10.*;
 
+/***
+ * Synchronization utility to sync in frame ops that need to be completed before executing main cmd buffer.
+ */
 public class Synchronization {
     private static final int ALLOCATION_SIZE = 50;
 
@@ -18,15 +24,28 @@ public class Synchronization {
     private final LongBuffer fences;
     private int idx = 0;
 
-    private ObjectArrayList<CommandPool.CommandBuffer> commandBuffers = new ObjectArrayList<>();
+    private final ObjectArrayList<CommandPool.CommandBuffer> fenceCbs = new ObjectArrayList<>();
+
+    private final LongArrayList semaphores = new LongArrayList();
+    private final ObjectArrayList<CommandPool.CommandBuffer> semaphoreCbs = new ObjectArrayList<>();
 
     Synchronization(int allocSize) {
         this.fences = MemoryUtil.memAllocLong(allocSize);
     }
 
-    public synchronized void addCommandBuffer(CommandPool.CommandBuffer commandBuffer) {
-        this.addFence(commandBuffer.getFence());
-        this.commandBuffers.add(commandBuffer);
+    public void addCommandBuffer(CommandPool.CommandBuffer commandBuffer) {
+        addCommandBuffer(commandBuffer, false);
+    }
+
+    public synchronized void addCommandBuffer(CommandPool.CommandBuffer commandBuffer, boolean useSemaphore) {
+        if (!useSemaphore) {
+            this.addFence(commandBuffer.getFence());
+            this.fenceCbs.add(commandBuffer);
+        }
+        else {
+            this.semaphores.add(commandBuffer.getSemaphore());
+            this.semaphoreCbs.add(commandBuffer);
+        }
     }
 
     public synchronized void addFence(long fence) {
@@ -47,13 +66,37 @@ public synchronized void waitFences() {
 
         vkWaitForFences(device, fences, true, VUtil.UINT64_MAX);
 
-        this.commandBuffers.forEach(CommandPool.CommandBuffer::reset);
-        this.commandBuffers.clear();
+        this.fenceCbs.forEach(CommandPool.CommandBuffer::reset);
+        this.fenceCbs.clear();
 
         fences.limit(ALLOCATION_SIZE);
         idx = 0;
     }
 
+    public synchronized void addWaitSemaphore(long semaphore) {
+        this.semaphores.add(semaphore);
+    }
+
+    public LongBuffer getWaitSemaphores(MemoryStack stack) {
+        var buffer = stack.mallocLong(this.semaphores.size())
+                          .put(this.semaphores.elements(), 0, this.semaphores.size());
+        buffer.flip();
+
+        this.semaphores.clear();
+        return buffer;
+    }
+
+    public void scheduleCbReset() {
+        final var frameSemaphoreCbs = this.semaphoreCbs.clone();
+        MemoryManager.getInstance().addFrameOp(
+                () -> {
+                    frameSemaphoreCbs.forEach(CommandPool.CommandBuffer::reset);
+                }
+        );
+
+        this.semaphoreCbs.clear();
+    }
+
     public static void waitFence(long fence) {
         VkDevice device = Vulkan.getVkDevice();
 

src/main/java/net/vulkanmod/vulkan/queue/CommandPool.java

@@ -130,6 +130,10 @@ public long getFence() {
             return fence;
         }
 
+        public long getSemaphore() {
+            return semaphore;
+        }
+
         public boolean isSubmitted() {
             return submitted;
         }

src/main/java/net/vulkanmod/vulkan/queue/Queue.java

@@ -47,9 +47,13 @@ public synchronized CommandPool.CommandBuffer beginCommands() {
             this.commandPool = new CommandPool(familyIndex);
     }
 
-    public synchronized long submitCommands(CommandPool.CommandBuffer commandBuffer) {
+    public long submitCommands(CommandPool.CommandBuffer commandBuffer) {
+        return submitCommands(commandBuffer, false);
+    }
+
+    public synchronized long submitCommands(CommandPool.CommandBuffer commandBuffer, boolean useSemaphore) {
         try (MemoryStack stack = stackPush()) {
-            return commandBuffer.submitCommands(stack, vkQueue, false);
+            return commandBuffer.submitCommands(stack, vkQueue, useSemaphore);
         }
     }