Candidate root GC. Mostly working, but slow.

Author: markshannon

Include/internal/pycore_gc.h

@@ -118,20 +118,6 @@ static inline void _PyObject_GC_SET_SHARED(PyObject *op) {
 /* Bit 1 is set when the object is in generation which is GCed currently. */
 #define _PyGC_PREV_MASK_COLLECTING ((uintptr_t)2)
 
-/* Bit 0 in _gc_next is the old space bit.
- * It is set as follows:
- * Young: gcstate->visited_space
- * old[0]: 0
- * old[1]: 1
- * permanent: 0
- *
- * During a collection all objects handled should have the bit set to
- * gcstate->visited_space, as objects are moved from the young gen
- * and the increment into old[gcstate->visited_space].
- * When object are moved from the pending space, old[gcstate->visited_space^1]
- * into the increment, the old space bit is flipped.
-*/
-#define _PyGC_NEXT_MASK_OLD_SPACE_1    1
 
 #define _PyGC_PREV_SHIFT           2
 #define _PyGC_PREV_MASK            (((uintptr_t) -1) << _PyGC_PREV_SHIFT)
@@ -239,13 +225,13 @@ static inline void _PyObject_GC_TRACK(
                           filename, lineno, __func__);
 
     PyInterpreterState *interp = _PyInterpreterState_GET();
-    PyGC_Head *generation0 = &interp->gc.young.head;
-    PyGC_Head *last = (PyGC_Head*)(generation0->_gc_prev);
+    PyGC_Head *live = &interp->gc.live.head;
+    PyGC_Head *last = (PyGC_Head*)(live->_gc_prev);
     _PyGCHead_SET_NEXT(last, gc);
     _PyGCHead_SET_PREV(gc, last);
-    uintptr_t not_visited = 1 ^ interp->gc.visited_space;
-    gc->_gc_next = ((uintptr_t)generation0) | not_visited;
-    generation0->_gc_prev = (uintptr_t)gc;
+    gc->_gc_next = ((uintptr_t)live);
+    live->_gc_prev = (uintptr_t)gc;
+    assert((gc->_gc_next & 1) == 0);
 #endif
 }
 

Include/internal/pycore_interp_structs.h

@@ -200,7 +200,7 @@ enum _GCPhase {
 
 /* If we change this, we need to change the default value in the
    signature of gc.collect. */
-#define NUM_GENERATIONS 3
+#define NUM_GENERATIONS 4
 
 struct _gc_runtime_state {
     /* List of objects that still need to be cleaned up, singly linked
@@ -213,8 +213,8 @@ struct _gc_runtime_state {
     int enabled;
     int debug;
     /* linked lists of container objects */
-    struct gc_generation young;
-    struct gc_generation old[2];
+    struct gc_generation live;
+    struct gc_generation candidates[NUM_GENERATIONS];
     /* a permanent generation which won't be collected */
     struct gc_generation permanent_generation;
     struct gc_generation_stats generation_stats[NUM_GENERATIONS];
@@ -227,8 +227,6 @@ struct _gc_runtime_state {
 
     Py_ssize_t heap_size;
     Py_ssize_t work_to_do;
-    /* Which of the old spaces is the visited space */
-    int visited_space;
     int phase;
 
 #ifdef Py_GIL_DISABLED

Include/internal/pycore_object.h

@@ -436,6 +436,8 @@ _Py_DECREF_CODE(PyCodeObject *co)
 #ifndef Py_GIL_DISABLED
 #ifdef Py_REF_DEBUG
 
+extern void _Py_CandidateCycleRoot(PyObject *op);
+
 static inline void Py_DECREF_MORTAL(const char *filename, int lineno, PyObject *op)
 {
     if (op->ob_refcnt <= 0) {
@@ -449,6 +451,9 @@ static inline void Py_DECREF_MORTAL(const char *filename, int lineno, PyObject *
     if (--op->ob_refcnt == 0) {
         _Py_Dealloc(op);
     }
+    else if (PyObject_GC_IsTracked(op)) {
+        _Py_CandidateCycleRoot(op);
+    }
 }
 #define Py_DECREF_MORTAL(op) Py_DECREF_MORTAL(__FILE__, __LINE__, _PyObject_CAST(op))
 

Include/internal/pycore_runtime_init.h

@@ -134,11 +134,7 @@ extern PyTypeObject _PyExc_MemoryError;
         }, \
         .gc = { \
             .enabled = 1, \
-            .young = { .threshold = 2000, }, \
-            .old = { \
-                { .threshold = 10, }, \
-                { .threshold = 0, }, \
-            }, \
+            .live = { .threshold = 2000, }, \
             .work_to_do = -5000, \
             .phase = GC_PHASE_MARK, \
         }, \
@@ -233,4 +229,4 @@ extern PyTypeObject _PyExc_MemoryError;
 #ifdef __cplusplus
 }
 #endif
-#endif /* !Py_INTERNAL_RUNTIME_INIT_H */
+#endif /* !Py_INTERNAL_RUNTIME_INIT_H */

Include/object.h

@@ -634,6 +634,11 @@ given type object has a specified feature.
 #define _Py_TYPE_REVEALED_FLAG (1 << 3)
 #endif
 
+// TO DO: move to private header
+#define _Py_GC_OBJECT (1 << 4)
+#define _Py_GC_TRACKED (1 << 5)
+
+
 #define Py_CONSTANT_NONE 0
 #define Py_CONSTANT_FALSE 1
 #define Py_CONSTANT_TRUE 2

Include/refcount.h

@@ -314,21 +314,16 @@ PyAPI_FUNC(void) _Py_DecRefSharedDebug(PyObject *, const char *, int);
 PyAPI_FUNC(void) _Py_MergeZeroLocalRefcount(PyObject *);
 #endif
 
-#if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG))
-// Stable ABI implements Py_DECREF() as a function call on limited C API
-// version 3.12 and newer, and on Python built in debug mode. _Py_DecRef() was
-// added to Python 3.10.0a7, use Py_DecRef() on older Python versions.
-// Py_DecRef() accepts NULL whereas _Py_DecRef() doesn't.
-static inline void Py_DECREF(PyObject *op) {
-#  if Py_LIMITED_API+0 >= 0x030a00A7
-    _Py_DecRef(op);
-#  else
-    Py_DecRef(op);
-#  endif
-}
-#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op))
+PyAPI_FUNC(void) Py_DecRefDebug(const char *filename, int lineno,PyObject *);
+
+#ifdef Py_DEBUG
+#define Py_DECREF(op) Py_DecRefDebug(__FILE__, __LINE__, _PyObject_CAST(op))
+#else
+#define Py_DECREF(op) Py_DecRef(_PyObject_CAST(op))
+#endif
 
-#elif defined(Py_GIL_DISABLED) && defined(Py_REF_DEBUG)
+#ifdef Py_GIL_DISABLED
+#  ifdef Py_REF_DEBUG
 static inline void Py_DECREF(const char *filename, int lineno, PyObject *op)
 {
     uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local);
@@ -353,8 +348,7 @@ static inline void Py_DECREF(const char *filename, int lineno, PyObject *op)
     }
 }
 #define Py_DECREF(op) Py_DECREF(__FILE__, __LINE__, _PyObject_CAST(op))
-
-#elif defined(Py_GIL_DISABLED)
+#  else
 static inline void Py_DECREF(PyObject *op)
 {
     uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local);
@@ -375,39 +369,9 @@ static inline void Py_DECREF(PyObject *op)
     }
 }
 #define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op))
-
-#elif defined(Py_REF_DEBUG)
-
-static inline void Py_DECREF(const char *filename, int lineno, PyObject *op)
-{
-#if SIZEOF_VOID_P > 4
-    /* If an object has been freed, it will have a negative full refcnt
-     * If it has not it been freed, will have a very large refcnt */
-    if (op->ob_refcnt_full <= 0 || op->ob_refcnt > (((PY_UINT32_T)-1) - (1<<20))) {
-#else
-    if (op->ob_refcnt <= 0) {
-#endif
-        _Py_NegativeRefcount(filename, lineno, op);
-    }
-    if (_Py_IsImmortal(op)) {
-        _Py_DECREF_IMMORTAL_STAT_INC();
-        return;
-    }
-    _Py_DECREF_STAT_INC();
-    _Py_DECREF_DecRefTotal();
-    if (--op->ob_refcnt == 0) {
-        _Py_Dealloc(op);
-    }
-}
-#define Py_DECREF(op) Py_DECREF(__FILE__, __LINE__, _PyObject_CAST(op))
-
-#else
-
-extern Py_NO_INLINE PyAPI_FUNC(void) Py_DECREF(PyObject *op);
-#define Py_DECREF(op) Py_DECREF(_PyObject_CAST(op))
+#  endif
 #endif
 
-
 /* Safely decref `op` and set `op` to NULL, especially useful in tp_clear
  * and tp_dealloc implementations.
  *

Modules/gcmodule.c

@@ -159,12 +159,12 @@ gc_set_threshold_impl(PyObject *module, int threshold0, int group_right_1,
 {
     GCState *gcstate = get_gc_state();
 
-    gcstate->young.threshold = threshold0;
+    gcstate->live.threshold = threshold0;
     if (group_right_1) {
-        gcstate->old[0].threshold = threshold1;
+        gcstate->candidates[0].threshold = threshold1;
     }
     if (group_right_2) {
-        gcstate->old[1].threshold = threshold2;
+        gcstate->candidates[1].threshold = threshold2;
     }
     Py_RETURN_NONE;
 }
@@ -181,8 +181,8 @@ gc_get_threshold_impl(PyObject *module)
 {
     GCState *gcstate = get_gc_state();
     return Py_BuildValue("(iii)",
-                         gcstate->young.threshold,
-                         gcstate->old[0].threshold,
+                         gcstate->live.threshold,
+                         gcstate->candidates[0].threshold,
                          0);
 }
 
@@ -203,14 +203,14 @@ gc_get_count_impl(PyObject *module)
     struct _gc_thread_state *gc = &tstate->gc;
 
     // Flush the local allocation count to the global count
-    _Py_atomic_add_int(&gcstate->young.count, (int)gc->alloc_count);
+    _Py_atomic_add_int(&gcstate->live.count, (int)gc->alloc_count);
     gc->alloc_count = 0;
 #endif
 
     return Py_BuildValue("(iii)",
-                         gcstate->young.count,
-                         gcstate->old[gcstate->visited_space].count,
-                         gcstate->old[gcstate->visited_space^1].count);
+                         gcstate->live.count,
+                         gcstate->candidates[0].count,
+                         gcstate->candidates[1].count);
 }
 
 /*[clinic input]

Objects/object.c

@@ -264,18 +264,7 @@ _Py_AddToAllObjects(PyObject *op)
 }
 #endif  /* Py_TRACE_REFS */
 
-#undef Py_DECREF_MORTAL
-Py_NO_INLINE void Py_DECREF_MORTAL(PyObject *op)
-{
-    assert(!_Py_IsStaticImmortal(op));
-    _Py_DECREF_STAT_INC();
-    if (--op->ob_refcnt == 0) {
-        _Py_Dealloc(op);
-    }
-}
-
-#undef Py_DECREF
-Py_NO_INLINE void Py_DECREF(PyObject *op)
+Py_NO_INLINE void Py_DecRef(PyObject *op)
 {
     // Non-limited C API and limited C API for Python 3.9 and older access
     // directly PyObject.ob_refcnt.
@@ -284,9 +273,21 @@ Py_NO_INLINE void Py_DECREF(PyObject *op)
         return;
     }
     _Py_DECREF_STAT_INC();
+#ifdef Py_REF_DEBUG
+    _Py_DECREF_DecRefTotal();
+#endif
     if (--op->ob_refcnt == 0) {
         _Py_Dealloc(op);
     }
+    else if (PyObject_GC_IsTracked(op)) {
+        _Py_CandidateCycleRoot(op);
+    }
+}
+
+void Py_DecRefDebug(const char *filename, int lineno, PyObject *op)
+{
+    // TO DO --implement properly
+     Py_DecRef(op);
 }
 
 #ifdef Py_REF_DEBUG
@@ -361,24 +362,12 @@ Py_IncRef(PyObject *o)
     Py_XINCREF(o);
 }
 
-void
-Py_DecRef(PyObject *o)
-{
-    Py_XDECREF(o);
-}
-
 void
 _Py_IncRef(PyObject *o)
 {
     Py_INCREF(o);
 }
 
-void
-_Py_DecRef(PyObject *o)
-{
-    Py_DECREF(o);
-}
-
 #ifdef Py_GIL_DISABLED
 # ifdef Py_REF_DEBUG
 static int