Fixed infinite recursion when colliding a TriangleShape vs a TriangleShape. (jrouwe#1621)

Fixes jrouwe#1620

Author: jrouwe

Docs/ReleaseNotes.md

@@ -11,6 +11,7 @@ For breaking API changes see [this document](https://github.com/jrouwe/JoltPhysi
 
 ### Bug Fixes
 
+* Fixed infinite recursion when colliding a `TriangleShape` vs a `TriangleShape`.
 * 32-bit MinGW g++ doesn't call the correct overload for the new operator when a type is 16 bytes aligned. This could cause unaligned read access violations.
 * Fixed compiling in double precision and fixed issues with floating point contraction that caused unit test failures on LoongArch architecture.
 * Added an epsilon to the `CastRay` / `CastShape` early out condition to avoid dividing by a very small number and overflowing to INF. This can cause a float overflow exception.

Jolt/Physics/Collision/Shape/TriangleShape.cpp

@@ -414,8 +414,12 @@ void TriangleShape::sRegister()
 		CollisionDispatch::sRegisterCollideShape(s, EShapeSubType::Triangle, sCollideConvexVsTriangle);
 		CollisionDispatch::sRegisterCastShape(s, EShapeSubType::Triangle, sCastConvexVsTriangle);
 
-		CollisionDispatch::sRegisterCollideShape(EShapeSubType::Triangle, s, CollisionDispatch::sReversedCollideShape);
-		CollisionDispatch::sRegisterCastShape(EShapeSubType::Triangle, s, CollisionDispatch::sReversedCastShape);
+		// Avoid registering triangle vs triangle as a reversed test to prevent infinite recursion
+		if (s != EShapeSubType::Triangle)
+		{
+			CollisionDispatch::sRegisterCollideShape(EShapeSubType::Triangle, s, CollisionDispatch::sReversedCollideShape);
+			CollisionDispatch::sRegisterCastShape(EShapeSubType::Triangle, s, CollisionDispatch::sReversedCastShape);
+		}
 	}
 
 	// Specialized collision functions

UnitTests/Physics/CollideShapeTests.cpp

@@ -536,4 +536,25 @@ TEST_SUITE("CollideShapeTests")
 				}
 			}
 	}
+
+	TEST_CASE("TestCollideTriangleVsTriangle")
+	{
+		constexpr float cPenetration = 0.01f;
+
+		// A triangle centered around the origin in the XZ plane
+		RefConst<Shape> t1 = new TriangleShape(Vec3(-1, 0, 1), Vec3(1, 0, 1), Vec3(0, 0, -1));
+
+		// A triangle in the XY plane with its tip just pointing in the origin
+		RefConst<Shape> t2 = new TriangleShape(Vec3(-1, 1, 0), Vec3(1, 1, 0), Vec3(0, -cPenetration, 0));
+
+		CollideShapeSettings collide_settings;
+		ClosestHitCollisionCollector<CollideShapeCollector> collector;
+		CollisionDispatch::sCollideShapeVsShape(t1, t2, Vec3::sOne(), Vec3::sOne(), Mat44::sIdentity(), Mat44::sIdentity(), SubShapeIDCreator(), SubShapeIDCreator(), collide_settings, collector);
+
+		CHECK(collector.HadHit());
+		CHECK_APPROX_EQUAL(collector.mHit.mContactPointOn1, Vec3::sZero());
+		CHECK_APPROX_EQUAL(collector.mHit.mContactPointOn2, Vec3(0, -cPenetration, 0));
+		CHECK_APPROX_EQUAL(collector.mHit.mPenetrationDepth, cPenetration);
+		CHECK_APPROX_EQUAL(collector.mHit.mPenetrationAxis.Normalized(), Vec3(0, 1, 0));
+	}
 }