Add the ability to change clip space Y axis orientation

Author: fincs

Primer.md

@@ -199,10 +199,12 @@ Field      | Default   | Description
 
 `DkDeviceFlags_*`    | Default | Description
 ---------------------|---------|--------------------------------------------------
-`DepthZeroToOne`     | ✓       | Clip space Z is [0, 1] like Vulkan
+`DepthZeroToOne`     | ✓       | Clip space Z is [0, 1] like Vulkan/Direct3D/Metal
 `DepthMinusOneToOne` |         | Clip space Z is [-1, 1] like OpenGL
 `OriginUpperLeft`    | ✓       | Image rows are stored sequentially from top to bottom, with 0.0 corresponding to the top edge of the image and 1.0 (or the image height if non-normalized) corresponding to the bottom
 `OriginLowerLeft`    |         | Image rows are stored sequentially from bottom to top, with 0.0 corresponding to the bottom edge of the image and 1.0 (or the image height if non-normalized) corresponding to the top
+`YAxisPointsUp`      | ✓       | Clip space Y axis points up like OpenGL/Direct3D/Metal
+`YAxisPointsDown`    |         | Clip space Y axis points down like Vulkan
 
 The debugging version of the library can use an optional callback (`cbDebug`). There are two situations in which deko3d makes usage of the debug callback:
 
@@ -218,7 +220,7 @@ In the release version of deko3d, warnings and parameter/state validation don't
 By default if memory allocation callbacks are not provided, deko3d uses the standard heap (i.e. malloc/free) for dynamic memory allocations.
 
 `gl_FragCoord` in fragment shaders obeys the device origin mode when it comes to the Y axis and has pixel centers at half-integers, *with GLSL layout qualifiers having absolutely no effect*.
-Please note that regardless of the Origin setting, the clip space Y axis points *up* like in OpenGL. Clip space X and Y are both in the range [-1, 1] as well.
+Please also note that the origin mode and clip space Y axis orientation can be configured independently of each other. Clip space X and Y are both in the range [-1, 1] as well.
 
 The current GPU tick can be queried without queuing a command buffer with `dkDeviceGetCurrentTimestamp`, and converted back and forth to nanoseconds using `dkNsToTimestamp`/`dkTimestampToNs`. See also [counters](#counters).
 

include/deko3d.h

@@ -117,6 +117,8 @@ enum
 	DkDeviceFlags_DepthMinusOneToOne = 1U << 8,
 	DkDeviceFlags_OriginUpperLeft    = 0U << 9,
 	DkDeviceFlags_OriginLowerLeft    = 1U << 9,
+	DkDeviceFlags_YAxisPointsUp      = 0U << 10,
+	DkDeviceFlags_YAxisPointsDown    = 1U << 10,
 };
 
 typedef struct DkDeviceMaker

source/dk_device.h

@@ -90,7 +90,16 @@ class Device
 	constexpr GpuInfo const& getGpuInfo() const noexcept { return m_gpuInfo; }
 
 	bool isDepthModeOpenGL() const noexcept { return (m_maker.flags & DkDeviceFlags_DepthMinusOneToOne) != 0; }
-	bool isOriginModeOpenGL() const noexcept { return (m_maker.flags & DkDeviceFlags_OriginLowerLeft) != 0; }
+	bool isOriginLowerLeft() const noexcept { return (m_maker.flags & DkDeviceFlags_OriginLowerLeft) != 0; }
+	bool isYAxisPointsDown() const noexcept { return (m_maker.flags & DkDeviceFlags_YAxisPointsDown) != 0; }
+
+	// Rasterizer uses framebuffer coordinates (with +Y = down) to calculate winding direction.
+	// In LowerLeft mode images are stored upside down, so triangle winding must be flipped.
+	bool windingFlip() const noexcept { return isOriginLowerLeft(); }
+
+	// Rasterizer expects clip space +Y to point down. In UpperLeft mode with Y pointing up
+	// (or LowerLeft mode with Y pointing down) we need to flip the incoming Y coordinate.
+	bool viewportFlipY() const noexcept { return !isYAxisPointsDown() ^ !isOriginLowerLeft(); }
 
 	DkResult initialize() noexcept;
 	~Device();

source/dk_swapchain.cpp

@@ -98,7 +98,7 @@ DkResult Swapchain::initialize(void* nativeWindow, DkImage const* const images[]
 	// If the origin mode is lower_left (i.e. OpenGL style), all rendered images are upside down
 	// (as in the BMP format). The compositor, of course, expects images to be stored the normal
 	// way (with Y pointing down), so we must instruct it to vertically flip whatever we feed it.
-	if (getDevice()->isOriginModeOpenGL())
+	if (getDevice()->isOriginLowerLeft())
 		nwindowSetTransform(m_nwin, HAL_TRANSFORM_FLIP_V);
 
 	return DkResult_Success;

source/maxwell/engine_3d.def

@@ -419,12 +419,15 @@ engine _3D 0xB197;
 0x4E6 StencilFrontFuncMask;
 0x4E7 StencilFrontMask;
 
-0x4EB SetWindowOriginMode bits (
-	0 NegateY; // wtf
-	4 Mode enum ( // nouveau calls this TriangleRastFlip
-		0 UpperLeft; // as in Direct3D/Vulkan
-		1 LowerLeft; // as in OpenGL
+0x4EB SetWindowOrigin bits (
+	// Value of special register 0x12 (SR_Y_DIRECTION)
+	0 Mode enum (
+		0 UpperLeft; // SR_Y_DIRECTION = +1.0
+		1 LowerLeft; // SR_Y_DIRECTION = -1.0
 	);
+
+	// Framebuffer orientation for the purposes of calculating polygon winding (CW/CCW)
+	4 FlipY bool; // 0=normal, 1=upside down
 );
 0x4EC LineWidthSmooth;
 0x4ED LineWidthAliased;

source/maxwell/gpu_3d_base.cpp

@@ -147,14 +147,9 @@ void Queue::setup3DEngine()
 	w << MacroSetRegisterInArray<E::ViewportTransform>(E::ViewportTransform::TranslateZ{}, isDepthOpenGL ? 0.5f : 0.0f);
 
 	// Configure viewport transform XY to convert [-1,1] into [0,1]: newXY = 0.5*oldXY + 0.5
-	// Additionally, for UpperLeft origin mode, the Y value needs to be reversed since the incoming Y coordinate points up,
-	// and that needs to be fixed to point down.
-	// Also, SetWindowOriginMode seems to affect how the hardware treats polygons as front-facing or back-facing,
-	// but it doesn't actually flip rendering.
-	bool isOriginOpenGL = getDevice()->isOriginModeOpenGL();
-	w << CmdInline(3D, SetWindowOriginMode{}, isOriginOpenGL ? E::SetWindowOriginMode::Mode::LowerLeft : E::SetWindowOriginMode::Mode::UpperLeft);
+	w << CmdInline(3D, SetWindowOrigin{}, E::SetWindowOrigin::FlipY{getDevice()->windingFlip()});
 	w << MacroSetRegisterInArray<E::ViewportTransform>(E::ViewportTransform::ScaleX{}, +0.5f);
-	w << MacroSetRegisterInArray<E::ViewportTransform>(E::ViewportTransform::ScaleY{}, isOriginOpenGL ? +0.5f : -0.5f);
+	w << MacroSetRegisterInArray<E::ViewportTransform>(E::ViewportTransform::ScaleY{}, getDevice()->viewportFlipY() ? -0.5f : +0.5f);
 	w << MacroSetRegisterInArray<E::ViewportTransform>(E::ViewportTransform::TranslateX{}, +0.5f);
 	w << MacroSetRegisterInArray<E::ViewportTransform>(E::ViewportTransform::TranslateY{}, +0.5f);
 	w << MacroSetRegisterInArray<E::Viewport>(E::Viewport::Horizontal{}, 0U | (1U << 16)); // x=0 w=1
@@ -323,8 +318,8 @@ void dkCmdBufSetViewports(DkCmdBuf obj, uint32_t firstId, DkViewport const viewp
 	DK_DEBUG_BAD_INPUT(firstId > DK_NUM_VIEWPORTS || numViewports > DK_NUM_VIEWPORTS || (firstId+numViewports) > DK_NUM_VIEWPORTS, "viewport range out of bounds");
 	DK_DEBUG_NON_NULL_ARRAY(viewports, numViewports);
 
-	bool isOriginOpenGL = obj->getDevice()->isOriginModeOpenGL();
-	bool isDepthOpenGL  = obj->getDevice()->isDepthModeOpenGL();
+	bool viewportFlipY = obj->getDevice()->viewportFlipY();
+	bool isDepthOpenGL = obj->getDevice()->isDepthModeOpenGL();
 
 	CmdBufWriter w{obj};
 	w.reserve(12*numViewports);
@@ -339,7 +334,7 @@ void dkCmdBufSetViewports(DkCmdBuf obj, uint32_t firstId, DkViewport const viewp
 		float depthRange = v.far - v.near;
 
 		float scaleX = halfWidth;
-		float scaleY = isOriginOpenGL ? halfHeight : (-halfHeight);
+		float scaleY = viewportFlipY ? (-halfHeight) : halfHeight;
 		float scaleZ = isDepthOpenGL ? 0.5f*depthRange : depthRange;
 
 		float offsetX = v.x + halfWidth;
@@ -431,7 +426,7 @@ void Queue::decompressSurface(DkImage const* image)
 	w << ColorTargetBindCmds(rt, 0);
 	w << CmdInline(3D, MultisampleMode{}, DkMsMode_1x);
 	w << Cmd(3D, ScreenScissorHorizontal{}, rt.m_width<<16, rt.m_height<<16);
-	w << CmdInline(3D, SetWindowOriginMode{}, E::SetWindowOriginMode::Mode::LowerLeft); // ??
+	w << CmdInline(3D, SetWindowOrigin{}, E::SetWindowOrigin::FlipY{}); // ??
 	w << CmdInline(3D, SetMultisampleRasterEnable{}, 0);
 
 	w << CmdInline(3D, SurfaceDecompress{}, 0);
@@ -440,7 +435,7 @@ void Queue::decompressSurface(DkImage const* image)
 	w << ColorTargetBindCmds(rt, 0);
 	w << CmdInline(3D, MultisampleMode{}, DkMsMode_1x);
 	w << Cmd(3D, ScreenScissorHorizontal{}, rt.m_width<<16, rt.m_height<<16);
-	w << CmdInline(3D, SetWindowOriginMode{}, E::SetWindowOriginMode::Mode::LowerLeft); // ??
+	w << CmdInline(3D, SetWindowOrigin{}, E::SetWindowOrigin::FlipY{}); // ??
 	w << CmdInline(3D, SetMultisampleRasterEnable{}, 0);
 
 	w << SetShadowRamControl(SRC::MethodTrackWithFilter);