feat: ensure accurate texture coords in ao

While the arithmetics I've used are correct, small differences in
operations have caused texture coordinates to occasionally differ
ever-so-slightly from the original (we're talking one bit difference).

Replacing it with something that more closely resembles calculations
that the game is performing during asset loading seems to resolve it.

Author: Krzyhau

Core/Conversion/Formats/ArtObjectConverter.cs

@@ -83,7 +83,7 @@ private static ArtObject LoadFromTransmissionFormat(Stream modelStream)
             var entry = entries.First();
             var artObject = ConfiguredJsonSerializer.DeserializeFromNode<ArtObject>(entry.Extras) ?? new ArtObject();
             artObject.Geometry = entry.Geometry.WithReversedWindingIndices();
-            FezGeometryUtil.RecalculateCubemapTexCoords(artObject.Geometry, artObject.Size);
+            FezGeometryUtil.RecalculateCubemapTexCoords(artObject.Geometry, artObject.Size, true);
 
             (Stream? albedo, Stream? emission) = GltfUtil.ExtractCubemapStreams(modelRoot);
             LoadCubemap(ref artObject, albedo, emission);
@@ -96,7 +96,7 @@ private static void AppendGeometryStream(ref ArtObject data, Stream geometryStre
             var geometries = WavefrontObjUtil.FromWavefrontObjStream<Matrix>(geometryStream);
             if (geometries.Count < 1) return;
             data.Geometry = geometries.First().Value.WithReversedWindingIndices();
-            FezGeometryUtil.RecalculateCubemapTexCoords(data.Geometry, data.Size);
+            FezGeometryUtil.RecalculateCubemapTexCoords(data.Geometry, data.Size, true);
         }
 
         private static void LoadCubemap(ref ArtObject data, Stream? albedoStream, Stream? emissionStream)

Core/Helpers/FezGeometryUtil.cs

@@ -8,26 +8,33 @@ internal static class FezGeometryUtil
     {
         /// Recalculates texture coordinates to match what game does with them during loading process.
         public static void RecalculateCubemapTexCoords<VertexType>(
-            IndexedPrimitives<VertexInstance, VertexType> geometry, Vector3 geometryBounds
+            IndexedPrimitives<VertexInstance, VertexType> geometry, Vector3 geometryBounds, bool widen
         ) {
+            // keeping arithmetics vaguely similar to the one in game to prevent number mismatch after conversion
             foreach (var vertex in geometry.Vertices)
             {
-                (int textureOffset, Vector3 xAxis, Vector3 yAxis) = vertex.NormalByte switch
+                (float textureOffset, Vector3 xAxis, Vector3 yAxis) = vertex.NormalByte switch
                 {
-                    5 => (0, new Vector3(1, 0, 0), new Vector3(0, -1, 0)), // front
-                    3 => (1, new Vector3(0, 0, -1), new Vector3(0, -1, 0)), // right
-                    2 => (2, new Vector3(-1, 0, 0), new Vector3(0, -1, 0)), // back
-                    0 => (3, new Vector3(0, 0, 1), new Vector3(0, -1, 0)), // left
-                    4 => (4, new Vector3(1, 0, 0), new Vector3(0, 0, 1)), // top
-                    1 => (5, new Vector3(1, 0, 0), new Vector3(0, 0, -1)), // bottom
+                    5 => (0, new Vector3(1, 0, 0), new Vector3(0, 1, 0)), // front
+                    3 => (0.25f, new Vector3(0, 0, -1), new Vector3(0, 1, 0)), // right
+                    2 => (0.375f, new Vector3(-1, 0, 0), new Vector3(0, 1, 0)), // back
+                    0 => (0.375f, new Vector3(0, 0, 1), new Vector3(0, 1, 0)), // left
+                    4 => (0.5f, new Vector3(1, 0, 0), new Vector3(0, 0, 1)), // top
+                    1 => (0.625f, new Vector3(1, 0, 0), new Vector3(0, 0, 1)), // bottom
                     _ => (0, new Vector3(), new Vector3())
                 };
 
-                var texturePlanePosition = vertex.Position / geometryBounds;
-                vertex.TextureCoordinate = new Vector2(
-                    (Vector3.Dot(texturePlanePosition, xAxis) + 0.5f + textureOffset) / 6.0f,
-                    Vector3.Dot(texturePlanePosition, yAxis) + 0.5f
-                );
+                var texturePlanePosition = (Vector3.One - vertex.Normal) * (vertex.Position / geometryBounds) * 2f + vertex.Normal;
+                var texturePlanePositionNormalized = texturePlanePosition / 2f + new Vector3(0.5f, 0.5f, 0.5f);
+                float uCoordinate = Vector3.Dot(xAxis, texturePlanePositionNormalized);
+                float vCoordinate = Vector3.Dot(yAxis, texturePlanePositionNormalized);
+                if (vertex.NormalByte != 4)
+                {
+                    vCoordinate = 1.0f - vCoordinate;
+                }
+
+                float finalUCoordinate = (textureOffset + uCoordinate / 8f) * (widen ? 1.3333334f : 1.0f);
+                vertex.TextureCoordinate = new Vector2(finalUCoordinate, vCoordinate);
             }
         }