Enable VK_EXT_swapchain_colorspace and fix Display P3 color space for macOS

Author: TNTpower0v0

src/main/java/net/vulkanmod/mixin/window/WindowMixin.java

@@ -61,6 +61,15 @@ private void redirect(int hint, int value) { }
     private void vulkanHint(WindowEventHandler windowEventHandler, ScreenManager screenManager, DisplayData displayData, String string, String string2, CallbackInfo ci) {
         GLFW.glfwWindowHint(GLFW_CLIENT_API, GLFW_NO_API);
 
+        // Enable 32bit P3 wide color gamut on macOS
+        if (Platform.isMacOS()) {
+            GLFW.glfwWindowHint(GLFW_COCOA_RETINA_FRAMEBUFFER, GLFW_TRUE);
+            GLFW.glfwWindowHint(GLFW_RED_BITS, 10);
+            GLFW.glfwWindowHint(GLFW_GREEN_BITS, 10);
+            GLFW.glfwWindowHint(GLFW_BLUE_BITS, 10);
+            GLFW.glfwWindowHint(GLFW_ALPHA_BITS, 2);
+        }
+
         //Fix Gnome Client-Side Decorators
         boolean b = (Platform.isGnome() | Platform.isWeston() | Platform.isGeneric()) && Platform.isWayLand();
         GLFW.glfwWindowHint(GLFW_DECORATED, (b ? GLFW_FALSE : GLFW_TRUE));

src/main/java/net/vulkanmod/vulkan/Vulkan.java

@@ -13,6 +13,7 @@
 import net.vulkanmod.vulkan.util.VkResult;
 import org.lwjgl.PointerBuffer;
 import org.lwjgl.system.MemoryStack;
+import static org.lwjgl.vulkan.EXTSwapchainColorspace.VK_EXT_SWAPCHAIN_COLOR_SPACE_EXTENSION_NAME;
 import org.lwjgl.util.vma.VmaAllocatorCreateInfo;
 import org.lwjgl.util.vma.VmaVulkanFunctions;
 import org.lwjgl.vulkan.*;
@@ -361,23 +362,21 @@ private static void createCommandPool() {
     }
 
     private static PointerBuffer getRequiredInstanceExtensions() {
-
         PointerBuffer glfwExtensions = glfwGetRequiredInstanceExtensions();
+        MemoryStack stack = stackGet();
 
         if (ENABLE_VALIDATION_LAYERS) {
-
-            MemoryStack stack = stackGet();
-
-            PointerBuffer extensions = stack.mallocPointer(glfwExtensions.capacity() + 1);
-
+            PointerBuffer extensions = stack.mallocPointer(glfwExtensions.capacity() + 2);
             extensions.put(glfwExtensions);
             extensions.put(stack.UTF8(VK_EXT_DEBUG_UTILS_EXTENSION_NAME));
-
-            // Rewind the buffer before returning it to reset its position back to 0
+            extensions.put(stack.UTF8(VK_EXT_SWAPCHAIN_COLOR_SPACE_EXTENSION_NAME));
             return extensions.rewind();
         }
 
-        return glfwExtensions;
+        PointerBuffer extensions = stack.mallocPointer(glfwExtensions.capacity() + 1);
+        extensions.put(glfwExtensions);
+        extensions.put(stack.UTF8(VK_EXT_SWAPCHAIN_COLOR_SPACE_EXTENSION_NAME));
+        return extensions.rewind();
     }
 
     public static void checkResult(int result, String errorMessage) {

src/main/java/net/vulkanmod/vulkan/framebuffer/SwapChain.java

@@ -9,6 +9,7 @@
 import net.vulkanmod.vulkan.queue.Queue;
 import net.vulkanmod.vulkan.texture.SamplerManager;
 import net.vulkanmod.vulkan.texture.VulkanImage;
+import net.vulkanmod.config.Platform;
 import org.lwjgl.system.MemoryStack;
 import org.lwjgl.vulkan.*;
 
@@ -24,6 +25,7 @@
 import static org.lwjgl.glfw.GLFW.glfwGetFramebufferSize;
 import static org.lwjgl.system.MemoryStack.stackGet;
 import static org.lwjgl.system.MemoryStack.stackPush;
+import static org.lwjgl.vulkan.EXTSwapchainColorspace.*;
 import static org.lwjgl.vulkan.KHRSurface.*;
 import static org.lwjgl.vulkan.KHRSwapchain.*;
 import static org.lwjgl.vulkan.VK10.*;
@@ -243,19 +245,33 @@ public long getImageView(int i) {
     private VkSurfaceFormatKHR getFormat(VkSurfaceFormatKHR.Buffer availableFormats) {
         List<VkSurfaceFormatKHR> list = availableFormats.stream().toList();
 
-        VkSurfaceFormatKHR format = list.get(0);
+        // Extended sRGB (P3 gamut with sRGB gamma) on macOS
+        if (Platform.isMacOS()) {
+            for (VkSurfaceFormatKHR f : list) {
+                if (f.format() == VK_FORMAT_B8G8R8A8_UNORM && f.colorSpace() == VK_COLOR_SPACE_DISPLAY_P3_NONLINEAR_EXT) {
+                    isBGRAformat = true;
+                    return f;
+                }
+            }
+        }
 
+        // If fail try RGBA format with standard sRGB
         for (VkSurfaceFormatKHR availableFormat : list) {
             if (availableFormat.format() == VK_FORMAT_R8G8B8A8_UNORM && availableFormat.colorSpace() == VK_COLOR_SPACE_SRGB_NONLINEAR_KHR)
                 return availableFormat;
+        }
 
+        // Standard BGRA sRGB
+        VkSurfaceFormatKHR format = list.get(0);
+        for (VkSurfaceFormatKHR availableFormat : list) {
             if (availableFormat.format() == VK_FORMAT_B8G8R8A8_UNORM && availableFormat.colorSpace() == VK_COLOR_SPACE_SRGB_NONLINEAR_KHR) {
                 format = availableFormat;
             }
         }
 
         if (format.format() == VK_FORMAT_B8G8R8A8_UNORM)
             isBGRAformat = true;
+
         return format;
     }