Use ob_flags for object's GC state

Author: markshannon

Include/internal/pycore_freelist.h

@@ -13,6 +13,7 @@ extern "C" {
 #include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_STORE_PTR_RELAXED()
 #include "pycore_pystate.h"             // _PyThreadState_GET
 #include "pycore_stats.h"               // OBJECT_STAT_INC
+#include "pycore_gc.h"                  // _Py_GC_OBJECT
 
 static inline struct _Py_freelists *
 _Py_freelists_GET(void)
@@ -38,8 +39,8 @@ _Py_freelists_GET(void)
     _PyFreeList_Push(&_Py_freelists_GET()->NAME, _PyObject_CAST(op), limit)
 
 // Pops a PyObject from the freelist, returns NULL if the freelist is empty.
-#define _Py_FREELIST_POP(TYPE, NAME) \
-    _Py_CAST(TYPE*, _PyFreeList_Pop(&_Py_freelists_GET()->NAME))
+#define _Py_FREELIST_POP(TYPE, NAME, IS_GC) \
+    _Py_CAST(TYPE*, _PyFreeList_Pop(&_Py_freelists_GET()->NAME, (IS_GC) ? _Py_GC_OBJECT : 0))
 
 // Pops a non-PyObject data structure from the freelist, returns NULL if the
 // freelist is empty.
@@ -82,13 +83,28 @@ _PyFreeList_PopNoStats(struct _Py_freelist *fl)
     return obj;
 }
 
+#ifndef NDEBUG
+static int
+consistent_flags(PyObject *op, int flags)
+{
+    if (Py_TYPE(op)->tp_flags & Py_TPFLAGS_HAVE_GC) {
+        return flags == _Py_GC_OBJECT;
+    }
+    else {
+        return flags == 0;
+    }
+}
+#endif
+
 static inline PyObject *
-_PyFreeList_Pop(struct _Py_freelist *fl)
+_PyFreeList_Pop(struct _Py_freelist *fl, int flags)
 {
     PyObject *op = _PyFreeList_PopNoStats(fl);
     if (op != NULL) {
         OBJECT_STAT_INC(from_freelist);
         _Py_NewReference(op);
+        assert(consistent_flags(op, flags));
+        op->ob_flags = flags;
     }
     return op;
 }

Include/internal/pycore_gc.h

@@ -69,13 +69,16 @@ _PyObject_CLEAR_GC_BITS(PyObject *op, uint8_t bits_to_clear)
 
 #endif
 
+#define _Py_GC_OBJECT (1 << 4)
+#define _Py_GC_TRACKED (1 << 5)
+
+
 /* True if the object is currently tracked by the GC. */
 static inline int _PyObject_GC_IS_TRACKED(PyObject *op) {
 #ifdef Py_GIL_DISABLED
     return _PyObject_HAS_GC_BITS(op, _PyGC_BITS_TRACKED);
 #else
-    PyGC_Head *gc = _Py_AS_GC(op);
-    return (gc->_gc_next != 0);
+    return (op->ob_flags & _Py_GC_TRACKED) != 0;
 #endif
 }
 #define _PyObject_GC_IS_TRACKED(op) _PyObject_GC_IS_TRACKED(_Py_CAST(PyObject*, op))
@@ -246,6 +249,8 @@ static inline void _PyObject_GC_TRACK(
     uintptr_t not_visited = 1 ^ interp->gc.visited_space;
     gc->_gc_next = ((uintptr_t)generation0) | not_visited;
     generation0->_gc_prev = (uintptr_t)gc;
+    assert(op->ob_flags & _Py_GC_OBJECT);
+    op->ob_flags |= _Py_GC_TRACKED;
 #endif
 }
 
@@ -280,6 +285,7 @@ static inline void _PyObject_GC_UNTRACK(
     _PyGCHead_SET_PREV(next, prev);
     gc->_gc_next = 0;
     gc->_gc_prev &= _PyGC_PREV_MASK_FINALIZED;
+    op->ob_flags &= ~_Py_GC_TRACKED;
 #endif
 }
 

Include/internal/pycore_object.h

@@ -853,9 +853,7 @@ _PyObject_GET_WEAKREFS_LISTPTR_FROM_OFFSET(PyObject *op)
 static inline int
 _PyObject_IS_GC(PyObject *obj)
 {
-    PyTypeObject *type = Py_TYPE(obj);
-    return (_PyType_IS_GC(type)
-            && (type->tp_is_gc == NULL || type->tp_is_gc(obj)));
+    return (obj->ob_flags & _Py_GC_OBJECT) != 0;
 }
 
 // Fast inlined version of PyObject_Hash()

Modules/_asynciomodule.c

@@ -2052,7 +2052,7 @@ future_new_iter(PyObject *fut)
     asyncio_state *state = get_asyncio_state_by_def((PyObject *)fut);
     ENSURE_FUTURE_ALIVE(state, fut)
 
-    it = _Py_FREELIST_POP(futureiterobject, futureiters);
+    it = _Py_FREELIST_POP(futureiterobject, futureiters, 1);
     if (it == NULL) {
         it = PyObject_GC_New(futureiterobject, state->FutureIterType);
         if (it == NULL) {

Objects/classobject.c

@@ -114,7 +114,7 @@ PyMethod_New(PyObject *func, PyObject *self)
         PyErr_BadInternalCall();
         return NULL;
     }
-    PyMethodObject *im = _Py_FREELIST_POP(PyMethodObject, pymethodobjects);
+    PyMethodObject *im = _Py_FREELIST_POP(PyMethodObject, pymethodobjects, 1);
     if (im == NULL) {
         im = PyObject_GC_New(PyMethodObject, &PyMethod_Type);
         if (im == NULL) {

Objects/complexobject.c

@@ -410,7 +410,7 @@ complex_subtype_from_c_complex(PyTypeObject *type, Py_complex cval)
 PyObject *
 PyComplex_FromCComplex(Py_complex cval)
 {
-    PyComplexObject *op = _Py_FREELIST_POP(PyComplexObject, complexes);
+    PyComplexObject *op = _Py_FREELIST_POP(PyComplexObject, complexes, 0);
 
     if (op == NULL) {
         /* Inline PyObject_New */

Objects/dictobject.c

@@ -872,7 +872,7 @@ new_dict(PyInterpreterState *interp,
          Py_ssize_t used, int free_values_on_failure)
 {
     assert(keys != NULL);
-    PyDictObject *mp = _Py_FREELIST_POP(PyDictObject, dicts);
+    PyDictObject *mp = _Py_FREELIST_POP(PyDictObject, dicts, 1);
     if (mp == NULL) {
         mp = PyObject_GC_New(PyDictObject, &PyDict_Type);
         if (mp == NULL) {

Objects/exceptions.c

@@ -3978,6 +3978,8 @@ get_memory_error(int allow_allocation, PyObject *args, PyObject *kwds)
         self->dict = NULL;
         self->args = (PyObject *)&_Py_SINGLETON(tuple_empty);
         _Py_NewReference((PyObject *)self);
+        assert(((PyObject *)self)->ob_flags == 0);
+        ((PyObject *)self)->ob_flags = _Py_GC_OBJECT;
         _PyObject_GC_TRACK(self);
     }
     MEMERRORS_UNLOCK(state);

Objects/floatobject.c

@@ -123,7 +123,7 @@ PyFloat_GetInfo(void)
 PyObject *
 PyFloat_FromDouble(double fval)
 {
-    PyFloatObject *op = _Py_FREELIST_POP(PyFloatObject, floats);
+    PyFloatObject *op = _Py_FREELIST_POP(PyFloatObject, floats, 0);
     if (op == NULL) {
         op = PyObject_Malloc(sizeof(PyFloatObject));
         if (!op) {

Objects/genobject.c

@@ -1972,7 +1972,7 @@ PyTypeObject _PyAsyncGenASend_Type = {
 static PyObject *
 async_gen_asend_new(PyAsyncGenObject *gen, PyObject *sendval)
 {
-    PyAsyncGenASend *ags = _Py_FREELIST_POP(PyAsyncGenASend, async_gen_asends);
+    PyAsyncGenASend *ags = _Py_FREELIST_POP(PyAsyncGenASend, async_gen_asends, 1);
     if (ags == NULL) {
         ags = PyObject_GC_New(PyAsyncGenASend, &_PyAsyncGenASend_Type);
         if (ags == NULL) {
@@ -2059,7 +2059,7 @@ _PyAsyncGenValueWrapperNew(PyThreadState *tstate, PyObject *val)
 {
     assert(val);
 
-    _PyAsyncGenWrappedValue *o = _Py_FREELIST_POP(_PyAsyncGenWrappedValue, async_gens);
+    _PyAsyncGenWrappedValue *o = _Py_FREELIST_POP(_PyAsyncGenWrappedValue, async_gens, 1);
     if (o == NULL) {
         o = PyObject_GC_New(_PyAsyncGenWrappedValue,
                             &_PyAsyncGenWrappedValue_Type);