psweep: Attempt to remove lock contention during page dequeue

Author: luke-gruber

gc/default/default.c

@@ -502,10 +502,12 @@ typedef struct rb_heap_struct {
     rb_atomic_t background_sweep_steps; // only incremented/checked by sweep thread
     rb_nativethread_cond_t sweep_page_cond; // associated with global sweep lock
     rb_nativethread_lock_t swept_pages_lock;
+    rb_nativethread_lock_t sweeping_page_lock;
     size_t pre_swept_slots_deferred;
     bool is_finished_sweeping;
     bool done_background_sweep;
     bool skip_sweep_continue; // skip current sweep continue
+    bool dequeued_last_page;
 } rb_heap_t;
 
 enum {
@@ -1088,6 +1090,7 @@ typedef struct lock_stats {
 
 static lock_stats_t sweep_lock_stats = {"objspace->sweep_lock", {{0}}, 0};
 static lock_stats_t swept_pages_lock_stats = {"heap->swept_pages_lock", {{0}}, 0};
+static lock_stats_t sweeping_page_lock_stats = {"heap->sweeping_page_lock", {{0}}, 0};
 
 
 static lock_callsite_stats_t*
@@ -1139,9 +1142,9 @@ print_lock_stats(void)
     fprintf(stderr, "%-40s %-30s %12s %12s %10s\n", "Lock Name", "Callsite", "Uncontended", "Contended", "Ratio");
     fprintf(stderr, "%-40s %-30s %12s %12s %10s\n", "---------", "--------", "-----------", "---------", "-----");
 
-    lock_stats_t *all_stats[] = {&sweep_lock_stats, &swept_pages_lock_stats};
+    lock_stats_t *all_stats[] = {&sweep_lock_stats, &swept_pages_lock_stats, &sweeping_page_lock_stats};
 
-    for (int i = 0; i < 2; i++) {
+    for (int i = 0; i < 3; i++) {
         lock_stats_t *stats = all_stats[i];
 
         /* Sort callsites by total contentions (descending) */
@@ -4624,8 +4627,14 @@ gc_sweep_step_worker(rb_objspace_t *objspace, rb_heap_t *heap)
         return;
     }
     while (1) {
+#if PSWEEP_LOCK_STATS > 0
+        instrumented_lock_acquire(&heap->sweeping_page_lock, &sweeping_page_lock_stats);
+#else
+        rb_native_mutex_lock(&heap->sweeping_page_lock);
+#endif
         struct heap_page *sweep_page = heap->sweeping_page;
         if (!sweep_page) {
+            rb_native_mutex_unlock(&heap->sweeping_page_lock);
             GC_ASSERT(!heap->done_background_sweep);
             GC_ASSERT(objspace->heaps_done_background_sweep < HEAP_COUNT);
             heap->done_background_sweep = true;
@@ -4637,6 +4646,7 @@ gc_sweep_step_worker(rb_objspace_t *objspace, rb_heap_t *heap)
         struct heap_page *next = ccan_list_next(&heap->pages, sweep_page, page_node);
 
         if (!next) {
+            rb_native_mutex_unlock(&heap->sweeping_page_lock);
             GC_ASSERT(!heap->done_background_sweep);
             GC_ASSERT(objspace->heaps_done_background_sweep < HEAP_COUNT);
             heap->done_background_sweep = true;
@@ -4647,6 +4657,7 @@ gc_sweep_step_worker(rb_objspace_t *objspace, rb_heap_t *heap)
         }
 
         heap->sweeping_page = next;
+        rb_native_mutex_unlock(&heap->sweeping_page_lock);
         heap->pre_sweeping_page = sweep_page;
 
         sweep_lock_unlock(&objspace->sweep_lock);
@@ -4782,6 +4793,7 @@ gc_sweep_start_heap(rb_objspace_t *objspace, rb_heap_t *heap)
     heap->is_finished_sweeping = false;
     heap->done_background_sweep = false;
     heap->skip_sweep_continue = false;
+    heap->dequeued_last_page = false;
 
     struct heap_page *page = NULL;
 
@@ -5037,14 +5049,17 @@ gc_sweep_dequeue_page(rb_objspace_t *objspace, rb_heap_t *heap, bool free_in_use
             return cur;
         }
     }
+    else if (heap->dequeued_last_page) {
+        return NULL;
+    }
 
     struct heap_page *page = NULL;
 
     // Avoid taking the global sweep_lock if we can
-#if PSWEEP_LOCK_STATS > 0
+#if PSWEEP_LOCK_STATS > 3
     instrumented_lock_acquire(&heap->swept_pages_lock, &swept_pages_lock_stats);
 #else
-    rb_native_mutex_lock(&heap->swept_pages_lock);
+    rb_native_mutex_trylock(&heap->swept_pages_lock);
 #endif
     {
         if (heap->swept_pages) {
@@ -5056,6 +5071,22 @@ gc_sweep_dequeue_page(rb_objspace_t *objspace, rb_heap_t *heap, bool free_in_use
     rb_native_mutex_unlock(&heap->swept_pages_lock);
     if (page) return page;
 
+#if PSWEEP_LOCK_STATS > 0
+    instrumented_lock_acquire(&heap->sweeping_page_lock, &sweeping_page_lock_stats);
+#else
+    rb_native_mutex_lock(&heap->sweeping_page_lock);
+#endif
+    {
+        if (heap->sweeping_page) {
+            page = heap->sweeping_page; // this could be the last page
+            heap->sweeping_page = ccan_list_next(&heap->pages, page, page_node);
+            psweep_debug(0, "[gc] gc_sweep_dequeue_page: dequeued unswept page from heap(%p) (heap %ld)\n", heap, heap - heaps);
+            *dequeued_unswept_page = true;
+        }
+    }
+    rb_native_mutex_unlock(&heap->sweeping_page_lock);
+    if (page) return page;
+
     sweep_lock_lock(&objspace->sweep_lock);
     {
         GC_ASSERT(!objspace->background_sweep_mode);
@@ -5074,14 +5105,11 @@ gc_sweep_dequeue_page(rb_objspace_t *objspace, rb_heap_t *heap, bool free_in_use
             }
             psweep_debug(0, "[gc] gc_sweep_dequeue_page: got nil page from heap(%p) (heap %ld) end\n", heap, heap - heaps);
         }
-        else {
-            *dequeued_unswept_page = true;
-            page = heap->sweeping_page; // this could be the last page
-            heap->sweeping_page = ccan_list_next(&heap->pages, page, page_node);
-            psweep_debug(0, "[gc] gc_sweep_dequeue_page: dequeued unswept page from heap(%p) (heap %ld)\n", heap, heap - heaps);
-        }
         GC_ASSERT(!objspace->background_sweep_mode);
     }
+    if (!heap->sweeping_page && !heap->pre_sweeping_page) {
+        heap->dequeued_last_page = true;
+    }
     sweep_lock_unlock(&objspace->sweep_lock);
 
     return page;
@@ -11044,6 +11072,7 @@ rb_gc_impl_objspace_free(void *objspace_ptr)
     for (int i = 0; i < HEAP_COUNT; i++) {
         rb_heap_t *heap = &heaps[i];
         rb_native_mutex_destroy(&heap->swept_pages_lock);
+        rb_native_mutex_destroy(&heap->sweeping_page_lock);
         rb_native_cond_destroy(&heap->sweep_page_cond);
         heap->total_pages = 0;
         heap->total_slots = 0;
@@ -11123,6 +11152,7 @@ rb_gc_impl_after_fork(void *objspace_ptr, rb_pid_t pid)
             rb_heap_t *heap = &heaps[i];
 
             rb_native_mutex_initialize(&heap->swept_pages_lock);
+            rb_native_mutex_initialize(&heap->sweeping_page_lock);
             rb_native_cond_initialize(&heap->sweep_page_cond);
             heap->pre_sweeping_page = NULL;
             heap->background_sweep_steps = heap->foreground_sweep_steps;
@@ -11235,6 +11265,7 @@ rb_gc_impl_objspace_init(void *objspace_ptr)
 
         ccan_list_head_init(&heap->pages);
         rb_native_mutex_initialize(&heap->swept_pages_lock);
+        rb_native_mutex_initialize(&heap->sweeping_page_lock);
         rb_native_cond_initialize(&heap->sweep_page_cond);
     }