Reduce number of branches in freelist push

Author: markshannon

Include/internal/pycore_freelist.h

@@ -31,11 +31,11 @@ _Py_freelists_GET(void)
 
 // Pushes `op` to the freelist, calls `freefunc` if the freelist is full
 #define _Py_FREELIST_FREE(NAME, op, freefunc) \
-    _PyFreeList_Free(&_Py_freelists_GET()->NAME, _PyObject_CAST(op), \
-                     Py_ ## NAME ## _MAXFREELIST, freefunc)
+    _PyFreeList_Free(&_Py_freelists_GET()->NAME, _PyObject_CAST(op), freefunc)
+
 // Pushes `op` to the freelist, returns 1 if successful, 0 if the freelist is full
-#define _Py_FREELIST_PUSH(NAME, op, limit) \
-    _PyFreeList_Push(&_Py_freelists_GET()->NAME, _PyObject_CAST(op), limit)
+#define _Py_FREELIST_PUSH(NAME, op) \
+   _PyFreeList_Push(&_Py_freelists_GET()->NAME, _PyObject_CAST(op))
 
 // Pops a PyObject from the freelist, returns NULL if the freelist is empty.
 #define _Py_FREELIST_POP(TYPE, NAME) \
@@ -46,26 +46,39 @@ _Py_freelists_GET(void)
 #define _Py_FREELIST_POP_MEM(NAME) \
     _PyFreeList_PopMem(&_Py_freelists_GET()->NAME)
 
-#define _Py_FREELIST_SIZE(NAME) (int)((_Py_freelists_GET()->NAME).size)
+static inline uint32_t
+_PyFreeList_Size(struct _Py_freelist *fl)
+{
+    return fl->capacity - fl->available;
+}
+
+#define _Py_FREELIST_SIZE(NAME) _PyFreeList_Size(&_Py_freelists_GET()->NAME)
+
+static inline void
+_PyFreeList_Init(struct _Py_freelist *fl, uint32_t capacity)
+{
+    fl->freelist = NULL;
+    fl->capacity = fl->available = capacity;
+}
 
 static inline int
-_PyFreeList_Push(struct _Py_freelist *fl, void *obj, Py_ssize_t maxsize)
+_PyFreeList_Push(struct _Py_freelist *fl, void *obj)
 {
-    if (fl->size < maxsize && fl->size >= 0) {
+    if (fl->available != 0) {
         FT_ATOMIC_STORE_PTR_RELAXED(*(void **)obj, fl->freelist);
         fl->freelist = obj;
-        fl->size++;
+        fl->available--;
         OBJECT_STAT_INC(to_freelist);
         return 1;
     }
     return 0;
 }
 
 static inline void
-_PyFreeList_Free(struct _Py_freelist *fl, void *obj, Py_ssize_t maxsize,
+_PyFreeList_Free(struct _Py_freelist *fl, void *obj,
                  freefunc dofree)
 {
-    if (!_PyFreeList_Push(fl, obj, maxsize)) {
+    if (!_PyFreeList_Push(fl, obj)) {
         dofree(obj);
     }
 }
@@ -75,9 +88,10 @@ _PyFreeList_PopNoStats(struct _Py_freelist *fl)
 {
     void *obj = fl->freelist;
     if (obj != NULL) {
-        assert(fl->size > 0);
+        assert(fl->capacity > 0);
         fl->freelist = *(void **)obj;
-        fl->size--;
+        fl->available++;
+        assert(fl->available <= fl->capacity);
     }
     return obj;
 }
@@ -97,9 +111,7 @@ static inline void *
 _PyFreeList_PopMem(struct _Py_freelist *fl)
 {
     void *op = _PyFreeList_PopNoStats(fl);
-    if (op != NULL) {
-        OBJECT_STAT_INC(from_freelist);
-    }
+    OBJECT_STAT_INC_COND(from_freelist, op != NULL);
     return op;
 }
 

Include/internal/pycore_freelist_state.h

@@ -36,9 +36,10 @@ struct _Py_freelist {
     // For PyObjects, this overlaps with the `ob_refcnt` field or the `ob_tid`
     // field.
     void *freelist;
-
-    // The number of items in the free list or -1 if the free list is disabled
-    Py_ssize_t size;
+    // The remaining space in this freelist;
+    uint32_t available;
+    // The maximum number of items this freelist is allowed to hold
+    uint32_t capacity;
 };
 
 struct _Py_freelists {

Modules/_asynciomodule.c

@@ -1826,7 +1826,7 @@ FutureIter_dealloc(PyObject *it)
     PyObject_GC_UnTrack(it);
     tp->tp_clear(it);
 
-    if (!_Py_FREELIST_PUSH(futureiters, it, Py_futureiters_MAXFREELIST)) {
+    if (!_Py_FREELIST_PUSH(futureiters, it)) {
         PyObject_GC_Del(it);
         Py_DECREF(tp);
     }

Objects/object.c

@@ -900,10 +900,11 @@ clear_freelist(struct _Py_freelist *freelist, int is_finalization,
     while ((ptr = _PyFreeList_PopNoStats(freelist)) != NULL) {
         dofree(ptr);
     }
-    assert(freelist->size == 0 || freelist->size == -1);
+    assert(_PyFreeList_Size(freelist) == 0);
     assert(freelist->freelist == NULL);
+
     if (is_finalization) {
-        freelist->size = -1;
+        _PyFreeList_Init(freelist, 0);
     }
 }
 

Objects/tupleobject.c

@@ -1166,7 +1166,7 @@ maybe_freelist_push(PyTupleObject *op)
     }
     Py_ssize_t index = Py_SIZE(op) - 1;
     if (index < PyTuple_MAXSAVESIZE) {
-        return _Py_FREELIST_PUSH(tuples[index], op, Py_tuple_MAXFREELIST);
+        return _Py_FREELIST_PUSH(tuples[index], op);
     }
     return 0;
 }

Python/pystate.c

@@ -590,6 +590,26 @@ free_interpreter(PyInterpreterState *interp)
         PyMem_RawFree(interp);
     }
 }
+
+static void
+init_freelists(struct _Py_freelists *freelists)
+{
+    _PyFreeList_Init(&freelists->floats, Py_floats_MAXFREELIST);
+    for (Py_ssize_t i = 0; i < PyTuple_MAXSAVESIZE; i++) {
+        _PyFreeList_Init(&freelists->tuples[i], Py_tuple_MAXFREELIST);
+    }
+    _PyFreeList_Init(&freelists->lists, Py_lists_MAXFREELIST);
+    _PyFreeList_Init(&freelists->list_iters, Py_list_iters_MAXFREELIST);
+    _PyFreeList_Init(&freelists->tuple_iters, Py_tuple_iters_MAXFREELIST);
+    _PyFreeList_Init(&freelists->dicts, Py_dicts_MAXFREELIST);
+    _PyFreeList_Init(&freelists->dictkeys, Py_dictkeys_MAXFREELIST);
+    _PyFreeList_Init(&freelists->slices, Py_slices_MAXFREELIST);
+    _PyFreeList_Init(&freelists->contexts, Py_contexts_MAXFREELIST);
+    _PyFreeList_Init(&freelists->async_gens, Py_async_gens_MAXFREELIST);
+    _PyFreeList_Init(&freelists->async_gen_asends, Py_async_gen_asends_MAXFREELIST);
+    _PyFreeList_Init(&freelists->futureiters, Py_futureiters_MAXFREELIST);
+}
+
 #ifndef NDEBUG
 static inline int check_interpreter_whence(long);
 #endif
@@ -699,6 +719,7 @@ init_interpreter(PyInterpreterState *interp,
     _Py_stackref_associate(interp, Py_True, PyStackRef_True);
 #endif
 
+    init_freelists(&interp->object_state.freelists);
     interp->_initialized = 1;
     return _PyStatus_OK();
 }