[upstream] Threading cleanup (erincatto/box2d#1049)

- Added internal scheduler so users don't need one.
- Simplified b2EnqueueTaskCallback. Users are no longer responsible for
providing parallel-for.
- Simplified b2TaskCallback. Users no longer need to provide a
thread/worker index.
- Removed usage of enkiTS from the samples app. It was very useful, but
I needed more control over threading. Users can now refer to scheduler.h
for example task scheduler hook up.
- Cleaned up and documented the solver tasking system
- Added b2World_SetWorkerCount and b2World_GetWorkerCount so the worker
count can be modified on an existing world
- Added `B2_MAX_TASKS` so users can pre-allocate task arrays with stable
pointers.

Author: ikpil

src/Box2D.NET.Samples/Primitives/SampleTask.cs

@@ -19,6 +19,7 @@
 using Silk.NET.OpenGL.Extensions.ImGui;
 using Silk.NET.Windowing;
 using static Box2D.NET.B2Cores;
+using static Box2D.NET.B2Constants;
 using static Box2D.NET.B2Diagnostics;
 using static Box2D.NET.B2Buffers;
 using static Box2D.NET.B2MathFunction;
@@ -758,7 +759,7 @@ private unsafe void ScrollCallback(WindowHandle* window, double dx, double dy)
 
     private void UpdateUI()
     {
-        int maxWorkers = (int)(Environment.ProcessorCount * 1.5f);
+        int maxWorkers = B2_MAX_WORKERS;
 
         float fontSize = ImGui.GetFontSize();
         float menuWidth = 13.0f * fontSize;

src/Box2D.NET.Samples/Primitives/TaskScheduler.cs

@@ -27,8 +27,7 @@ namespace Box2D.NET.Samples.Samples;
 
 public class Sample : IDisposable
 {
-    public const int k_maxContactPoints = 12 * 2048;
-    public const int m_maxTasks = 64;
+    public const int m_maxTasks = 512;
     public const int m_maxThreads = 64;
     public const int m_profileCapacity = 512;
 
@@ -42,11 +41,6 @@ public class Sample : IDisposable
     protected Camera m_camera;
     protected Draw m_draw;
 
-    private TaskScheduler m_scheduler;
-    private SampleTask[] m_tasks;
-    private int m_taskCount;
-    protected int m_threadCount;
-
     private B2BodyId m_mouseBodyId;
 
     //
@@ -79,19 +73,6 @@ public Sample(SampleContext context)
         m_camera = context.camera;
         m_draw = context.draw;
 
-        m_scheduler = new TaskScheduler();
-        m_scheduler.Initialize(m_context.workerCount);
-
-        m_tasks = new SampleTask[m_maxTasks];
-        for (int i = 0; i < m_maxTasks; ++i)
-        {
-            m_tasks[i] = new SampleTask();
-        }
-
-        m_taskCount = 0;
-
-        m_threadCount = 1 + m_context.workerCount;
-
         m_worldId = b2_nullWorldId;
 
         m_textIncrement = 26;
@@ -125,9 +106,7 @@ public virtual void Dispose()
     {
         // By deleting the world, we delete the bomb, mouse joint, etc.
         b2DestroyWorld(m_worldId);
-
-        // delete m_scheduler;
-        // delete[] m_tasks;
+        
     }
 
     public void CreateWorld()
@@ -140,8 +119,8 @@ public void CreateWorld()
 
         B2WorldDef worldDef = b2DefaultWorldDef();
         worldDef.workerCount = m_context.workerCount;
-        worldDef.enqueueTask = EnqueueTask;
-        worldDef.finishTask = FinishTask;
+        // worldDef.enqueueTask = EnqueueTask;
+        // worldDef.finishTask = FinishTask;
         worldDef.userTaskContext = this;
         worldDef.enableSleep = m_context.enableSleep;
 
@@ -340,41 +319,6 @@ private void DrawProfileSeries(ImDrawListPtr drawList, Vector2 origin, Vector2 s
             previous = current;
         }
     }
-
-
-    private static object EnqueueTask(b2TaskCallback task, int itemCount, int minRange, object taskContext, object userContext)
-    {
-        Sample sample = userContext as Sample;
-        if (sample.m_taskCount < m_maxTasks)
-        {
-            SampleTask sampleTask = sample.m_tasks[sample.m_taskCount];
-            sampleTask.m_SetSize = itemCount;
-            sampleTask.m_MinRange = minRange;
-            sampleTask.m_task = task;
-            sampleTask.m_taskContext = taskContext;
-            sample.m_scheduler.AddTaskSetToPipe(sampleTask);
-            ++sample.m_taskCount;
-            return sampleTask;
-        }
-        else
-        {
-            // This is not fatal but the maxTasks should be increased
-            B2_ASSERT(false);
-            task(0, itemCount, 0, taskContext);
-            return null;
-        }
-    }
-
-    private static void FinishTask(object taskPtr, object userContext)
-    {
-        if (taskPtr != null)
-        {
-            SampleTask sampleTask = taskPtr as SampleTask;
-            Sample sample = userContext as Sample;
-            sample.m_scheduler.WaitforTask(sampleTask);
-        }
-    }
-
     public void ResetText()
     {
         m_textLine = m_textIncrement;
@@ -568,7 +512,7 @@ public virtual void Step()
         for (int i = 0; i < 1; ++i)
         {
             b2World_Step(m_worldId, timeStep, m_context.subStepCount);
-            m_taskCount = 0;
+            // m_taskCount = 0;
         }
 
         if (timeStep > 0.0f)

src/Box2D.NET.Samples/SampleApp.cs

@@ -3,11 +3,9 @@
 // SPDX-License-Identifier: MIT
 
 using System;
-using System.Runtime.InteropServices;
 using System.Runtime.CompilerServices;
 using static Box2D.NET.B2Constants;
 using static Box2D.NET.B2Buffers;
-using static Box2D.NET.B2Diagnostics;
 
 namespace Box2D.NET
 {
@@ -205,5 +203,12 @@ public static void b2Array_Destroy<T>(ref B2Array<T> a)
             a.count = 0;
             a.capacity = 0;
         }
+
+        public static void b2Array_ResizeAndSetZero<T>(ref B2Array<T> a, int n) where T : new()
+        {
+            b2Array_Reserve(ref a, n);
+            // memset(0, ...)
+            a.count = n;
+        }
     }
-}
+}

src/Box2D.NET.Samples/Samples/Sample.cs

@@ -17,6 +17,7 @@
 using static Box2D.NET.B2ArenaAllocators;
 using static Box2D.NET.B2MathFunction;
 using static Box2D.NET.B2Shapes;
+using static Box2D.NET.B2ParallelFors;
 
 namespace Box2D.NET
 {
@@ -341,12 +342,11 @@ public static bool b2PairQueryCallback(int proxyId, ulong userData, ref B2QueryP
         }
 
 
-        public static void b2FindPairsTask(int startIndex, int endIndex, uint threadIndex, object context)
+        public static void b2FindPairsTask(int startIndex, int endIndex, int workerIndex, object context)
         {
-            b2TracyCZoneNC(B2TracyCZone.pair_task, "Pair", B2HexColor.b2_colorMediumSlateBlue, true);
-
-            B2_UNUSED(threadIndex);
+            B2_UNUSED(workerIndex);
 
+            b2TracyCZoneNC(B2TracyCZone.pair_task, "Pair", B2HexColor.b2_colorMediumSlateBlue, true);
             B2World world = context as B2World;
             B2BroadPhase bp = world.broadPhase;
 
@@ -406,12 +406,8 @@ public static void b2FindPairsTask(int startIndex, int endIndex, uint threadInde
             b2TracyCZoneEnd(B2TracyCZone.pair_task);
         }
 
-        public static void b2UpdateTreesTask(int startIndex, int endIndex, uint threadIndex, object context)
+        public static void b2UpdateTreesTask(object context)
         {
-            B2_UNUSED(startIndex);
-            B2_UNUSED(endIndex);
-            B2_UNUSED(threadIndex);
-
             b2TracyCZoneNC(B2TracyCZone.tree_task, "Rebuild BVH", B2HexColor.b2_colorFireBrick, true);
 
             B2World world = (B2World)context;
@@ -440,7 +436,7 @@ public static void b2UpdateBroadPhasePairs(B2World world)
             bp.moveResults = b2AllocateArenaItem<B2MoveResult>(alloc, moveCount, "move results");
 
             // This capacity can be exceeded if there are many overlapping pairs (e.g. all shapes at the origin)
-            bp.movePairCapacity = 8 * moveCount;
+            bp.movePairCapacity = 32 * moveCount;
 
             bp.movePairs = b2AllocateArenaItem<B2MovePair>(alloc, bp.movePairCapacity, "move pairs");
             b2AtomicStoreInt(ref bp.movePairIndex, 0);
@@ -451,20 +447,23 @@ public static void b2UpdateBroadPhasePairs(B2World world)
 #endif
 
             int minRange = 64;
-            object userPairTask = world.enqueueTaskFcn(b2FindPairsTask, moveCount, minRange, world, world.userTaskContext);
-            if (userPairTask != null)
-            {
-                world.finishTaskFcn(userPairTask, world.userTaskContext);
-                world.taskCount += 1;
-            }
+            b2ParallelFor(world, b2FindPairsTask, moveCount, minRange, world);
+
+            b2TracyCZoneNC(B2TracyCZone.create_contacts, "Create Contacts", B2HexColor.b2_colorCoral, true);
 
             // Task that can be done in parallel with the narrow-phase
             // - rebuild the collision tree for dynamic and kinematic bodies to keep their query performance good
-            world.userTreeTask = world.enqueueTaskFcn(b2UpdateTreesTask, 1, 1, world, world.userTaskContext);
-            world.taskCount += 1;
-            world.activeTaskCount += world.userTreeTask == null ? 0 : 1;
-
-            b2TracyCZoneNC(B2TracyCZone.create_contacts, "Create Contacts", B2HexColor.b2_colorCoral, true);
+            if (world.taskCount < B2_MAX_TASKS)
+            {
+                world.userTreeTask = world.enqueueTaskFcn(b2UpdateTreesTask, world, world.userTaskContext);
+                world.taskCount += 1;
+                world.activeTaskCount += world.userTreeTask == null ? 0 : 1;
+            }
+            else
+            {
+                world.userTreeTask = null;
+                b2UpdateTreesTask(world);
+            }
 
             // Single-threaded work
             // - Clear move flags
@@ -579,4 +578,4 @@ internal static void b2ValidateNoEnlarged(B2BroadPhase bp)
 #endif
         }
     }
-}
+}

src/Box2D.NET/B2Arrays.cs

@@ -12,8 +12,14 @@ public static class B2Constants
         // problems, so 100km as a limit should be fine in all cases.
         public static float B2_HUGE => (100000.0f * b2GetLengthUnitsPerMeter());
 
-        // Maximum parallel workers. Used to size some static arrays.
-        public const int B2_MAX_WORKERS = 64;
+        // Maximum parallel workers. Used for some fixed size arrays.
+        public const int B2_MAX_WORKERS = 32;
+
+        // Maximum number of tasks queued per world step. b2EnqueueTaskCallback will never be called
+        // more than this per world step. This is related to B2_MAX_WORKERS. With 32 workers,
+        // the maximum observed task count is 130. This allows an external task system to use a fixed
+        // size array for Box2D task, which may help with creating stable user task pointers.
+        public const int B2_MAX_TASKS = 256;
 
         // Maximum number of colors in the constraint graph. Constraints that cannot
         // find a color are added to the overflow set which are solved single-threaded.
@@ -81,4 +87,4 @@ public static class B2Constants
         /// Simple djb2 hash function for determinism testing
         public const int B2_HASH_INIT = 5381;
     }
-}
+}