Add per-size freelists. Work in progress

Author: markshannon

Include/internal/pycore_ceval.h

@@ -342,7 +342,7 @@ _Py_eval_breaker_bit_is_set(PyThreadState *tstate, uintptr_t bit)
 void _Py_set_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit);
 void _Py_unset_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit);
 
-PyAPI_FUNC(_PyStackRef) _PyFloat_FromDouble_ConsumeInputs(_PyStackRef left, _PyStackRef right, double value);
+PyAPI_FUNC(_PyStackRef) _PyFloat_FromDouble_ConsumeInputs(PyThreadState *ts, _PyStackRef left, _PyStackRef right, double value);
 
 #ifdef __cplusplus
 }

Include/internal/pycore_floatobject.h

@@ -31,9 +31,7 @@ struct _Py_float_runtime_state {
 };
 
 
-
-
-PyAPI_FUNC(void) _PyFloat_ExactDealloc(PyObject *op);
+PyAPI_FUNC(void) _PyFloat_ExactDealloc(PyThreadState *ts, PyObject *op);
 
 
 extern void _PyFloat_DebugMallocStats(FILE* out);

Include/internal/pycore_freelist.h

@@ -30,11 +30,10 @@ _Py_freelists_GET(void)
 
 // Pushes `op` to the freelist, calls `freefunc` if the freelist is full
 #define _Py_FREELIST_FREE(NAME, op, freefunc) \
-    _PyFreeList_Free(&_Py_freelists_GET()->NAME, _PyObject_CAST(op), \
-                     Py_ ## NAME ## _MAXFREELIST, freefunc)
+    _PyFreeList_Free(&_Py_freelists_GET()->NAME, _PyObject_CAST(op), freefunc)
 // Pushes `op` to the freelist, returns 1 if successful, 0 if the freelist is full
-#define _Py_FREELIST_PUSH(NAME, op, limit) \
-    _PyFreeList_Push(&_Py_freelists_GET()->NAME, _PyObject_CAST(op), limit)
+#define _Py_FREELIST_PUSH(NAME, op) \
+    _PyFreeList_Push(&_Py_freelists_GET()->NAME, _PyObject_CAST(op))
 
 // Pops a PyObject from the freelist, returns NULL if the freelist is empty.
 #define _Py_FREELIST_POP(TYPE, NAME) \
@@ -45,26 +44,36 @@ _Py_freelists_GET(void)
 #define _Py_FREELIST_POP_MEM(NAME) \
     _PyFreeList_PopMem(&_Py_freelists_GET()->NAME)
 
-#define _Py_FREELIST_SIZE(NAME) (int)((_Py_freelists_GET()->NAME).size)
+static inline uint32_t
+_PyFreeList_Size(struct _Py_freelist *fl)
+{
+    return fl->capacity - fl->available;
+}
+
+static inline void
+_PyFreeList_Init(struct _Py_freelist *fl, uint32_t capacity)
+{
+    fl->freelist = NULL;
+    fl->capacity = fl->available = 0; // capacity;
+}
 
 static inline int
-_PyFreeList_Push(struct _Py_freelist *fl, void *obj, Py_ssize_t maxsize)
+_PyFreeList_Push(struct _Py_freelist *fl, void *obj)
 {
-    if (fl->size < maxsize && fl->size >= 0) {
+    if (fl->available != 0) {
         FT_ATOMIC_STORE_PTR_RELAXED(*(void **)obj, fl->freelist);
         fl->freelist = obj;
-        fl->size++;
+        fl->available--;
         OBJECT_STAT_INC(to_freelist);
         return 1;
     }
     return 0;
 }
 
 static inline void
-_PyFreeList_Free(struct _Py_freelist *fl, void *obj, Py_ssize_t maxsize,
-                 freefunc dofree)
+_PyFreeList_Free(struct _Py_freelist *fl, void *obj, freefunc dofree)
 {
-    if (!_PyFreeList_Push(fl, obj, maxsize)) {
+    if (!_PyFreeList_Push(fl, obj)) {
         dofree(obj);
     }
 }
@@ -74,9 +83,9 @@ _PyFreeList_PopNoStats(struct _Py_freelist *fl)
 {
     void *obj = fl->freelist;
     if (obj != NULL) {
-        assert(fl->size > 0);
+        assert(fl->capacity > 0);
         fl->freelist = *(void **)obj;
-        fl->size--;
+        fl->available++;
     }
     return obj;
 }

Include/internal/pycore_freelist_state.h

@@ -32,28 +32,10 @@ struct _Py_freelist {
     // For PyObjects, this overlaps with the `ob_refcnt` field or the `ob_tid`
     // field.
     void *freelist;
-
-    // The number of items in the free list or -1 if the free list is disabled
-    Py_ssize_t size;
-};
-
-struct _Py_freelists {
-    struct _Py_freelist floats;
-    struct _Py_freelist ints;
-    struct _Py_freelist tuples[PyTuple_MAXSAVESIZE];
-    struct _Py_freelist lists;
-    struct _Py_freelist list_iters;
-    struct _Py_freelist tuple_iters;
-    struct _Py_freelist dicts;
-    struct _Py_freelist dictkeys;
-    struct _Py_freelist slices;
-    struct _Py_freelist contexts;
-    struct _Py_freelist async_gens;
-    struct _Py_freelist async_gen_asends;
-    struct _Py_freelist futureiters;
-    struct _Py_freelist object_stack_chunks;
-    struct _Py_freelist unicode_writers;
-    struct _Py_freelist pymethodobjects;
+    // The remaining space in this freelist;
+    uint32_t available;
+    // The maximum number of items this freelist is allowed to hold
+    uint32_t capacity;
 };
 
 #ifdef __cplusplus

Include/internal/pycore_interp.h

@@ -89,6 +89,9 @@ typedef struct _rare_events {
 
 /* interpreter state */
 
+typedef PyObject *(*alloc_func)(PyTypeObject *tp, size_t presize, size_t size);
+typedef PyObject *(*free_func)(PyObject *);
+
 /* PyInterpreterState holds the global state for one of the runtime's
    interpreters.  Typically the initial (main) interpreter is the only one.
 
@@ -100,6 +103,8 @@ struct _is {
      * which is by far the hottest field in this struct
      * and should be placed at the beginning. */
     struct _ceval_state ceval;
+    alloc_func alloc;
+    free_func free;
 
     PyInterpreterState *next;
 

Include/internal/pycore_object_alloc.h

@@ -4,6 +4,7 @@
 #include "pycore_object.h"      // _PyType_HasFeature()
 #include "pycore_pystate.h"     // _PyThreadState_GET()
 #include "pycore_tstate.h"      // _PyThreadStateImpl
+#include "pycore_freelist.h"     // _PyFreeList_Pop()
 
 #ifdef __cplusplus
 extern "C" {
@@ -65,6 +66,58 @@ _PyObject_ReallocWithType(PyTypeObject *tp, void *ptr, size_t size)
     return mem;
 }
 
+static inline PyObject *
+_PyObject_NewTstate(PyThreadState *ts, PyTypeObject *tp, size_t presize, size_t size)
+{
+    size += presize;
+    assert(size > 0);
+    if (size <= SMALL_REQUEST_THRESHOLD) {
+        int size_cls = (size - 1) >> ALIGNMENT_SHIFT;
+        struct _Py_freelist *fl = &ts->interp->object_state.freelists.by_size[size_cls];
+        char *mem = _PyFreeList_Pop(fl);
+        PyObject *op = (PyObject *)
+        if (mem != NULL) {
+            PyObject *op = (PyObject *)(mem + presize);
+            Py_SET_TYPE(op, tp);
+            op->ob_refcnt = 1;
+            return op;
+        }
+    }
+    return ts->interp->alloc(tp, presize, size);
+    char *mem = PyObject_Malloc(size);
+    if (mem == NULL) {
+        return PyErr_NoMemory();
+    }
+    PyObject *op = (PyObject *)(PyObject *)(mem + presize);
+    _PyObject_Init(op, tp);
+    return op;
+}
+
+static inline void
+_PyMem_FreeTstate(PyThreadState *ts, PyObject *obj, size_t presize, size_t size)
+{
+    size += presize;
+    assert(size > 0);
+    char *mem = ((char *)obj) - presize;
+    if (size <= SMALL_REQUEST_THRESHOLD) {
+        int size_cls = (size - 1) >> ALIGNMENT_SHIFT;
+        struct _Py_freelist *fl = &ts->interp->object_state.freelists.by_size[size_cls];
+        if (_PyFreeList_Push(fl, mem)) {
+            return;
+        }
+    }
+    OBJECT_STAT_INC(frees);
+    ts->interp->free(obj, presize);
+    _PyRuntime.allocators.standard.obj.free(_PyRuntime.allocators.standard.obj.ctx, mem);
+}
+
+static inline void
+_PyObject_FreeTstate(PyThreadState *ts, PyObject *obj, size_t size)
+{
+    assert(!PyObject_IS_GC(obj));
+    _PyMem_FreeTstate(ts, obj, 0, size);
+}
+
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_object_state.h

@@ -10,6 +10,7 @@ extern "C" {
 
 #include "pycore_freelist_state.h"  // _Py_freelists
 #include "pycore_hashtable.h"       // _Py_hashtable_t
+#include "pycore_obmalloc.h"        // NB_SMALL_SIZE_CLASSES
 
 
 /* Reference tracer state */
@@ -26,6 +27,26 @@ struct _py_object_runtime_state {
     int _not_used;
 };
 
+struct _Py_freelists {
+    struct _Py_freelist by_size[NB_SMALL_SIZE_CLASSES];
+    struct _Py_freelist floats;
+    struct _Py_freelist ints;
+    struct _Py_freelist tuples[PyTuple_MAXSAVESIZE];
+    struct _Py_freelist lists;
+    struct _Py_freelist list_iters;
+    struct _Py_freelist tuple_iters;
+    struct _Py_freelist dicts;
+    struct _Py_freelist dictkeys;
+    struct _Py_freelist slices;
+    struct _Py_freelist contexts;
+    struct _Py_freelist async_gens;
+    struct _Py_freelist async_gen_asends;
+    struct _Py_freelist futureiters;
+    struct _Py_freelist object_stack_chunks;
+    struct _Py_freelist unicode_writers;
+    struct _Py_freelist pymethodobjects;
+};
+
 struct _py_object_state {
 #if !defined(Py_GIL_DISABLED)
     struct _Py_freelists freelists;

Include/internal/pycore_stackref.h

@@ -171,6 +171,7 @@ _PyStackRef_DUP(_PyStackRef ref, const char *filename, int linenumber)
 #define PyStackRef_DUP(REF) _PyStackRef_DUP(REF, __FILE__, __LINE__)
 
 extern void PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct);
+extern void PyStackRef_CLOSE_SPECIALIZED_TSTATE(PyThreadState *tstate, _PyStackRef ref, destructor_tstate destruct);
 
 static inline _PyStackRef
 PyStackRef_MakeHeapSafe(_PyStackRef ref)
@@ -316,6 +317,13 @@ PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct)
     PyStackRef_CLOSE(ref);
 }
 
+static inline void
+PyStackRef_CLOSE_SPECIALIZED_TSTATE(PyThreadState *ts, _PyStackRef ref, destructor_tstate destruct)
+{
+    (void)destruct;
+    PyStackRef_CLOSE(ref);
+}
+
 static inline _PyStackRef
 PyStackRef_DUP(_PyStackRef stackref)
 {
@@ -575,6 +583,15 @@ PyStackRef_CLOSE_SPECIALIZED(_PyStackRef ref, destructor destruct)
     }
 }
 
+static inline void
+PyStackRef_CLOSE_SPECIALIZED_TSTATE(PyThreadState *tstate, _PyStackRef ref, destructor_tstate destruct)
+{
+    assert(!PyStackRef_IsNull(ref));
+    if (PyStackRef_RefcountOnObject(ref)) {
+        Py_DECREF_MORTAL_SPECIALIZED_TSTATE(tstate, BITS_TO_PTR(ref), destruct);
+    }
+}
+
 #ifdef _WIN32
 #define PyStackRef_XCLOSE PyStackRef_CLOSE
 #else

Include/internal/pycore_tstate.h

@@ -9,7 +9,6 @@ extern "C" {
 #endif
 
 #include "pycore_brc.h"             // struct _brc_thread_state
-#include "pycore_freelist_state.h"  // struct _Py_freelists
 #include "pycore_mimalloc.h"        // struct _mimalloc_thread_state
 #include "pycore_qsbr.h"            // struct qsbr
 

Include/object.h

@@ -351,6 +351,7 @@ typedef int (*traverseproc)(PyObject *, visitproc, void *);
 
 typedef void (*freefunc)(void *);
 typedef void (*destructor)(PyObject *);
+typedef void (*destructor_tstate)(struct _ts *, PyObject *);
 typedef PyObject *(*getattrfunc)(PyObject *, char *);
 typedef PyObject *(*getattrofunc)(PyObject *, PyObject *);
 typedef int (*setattrfunc)(PyObject *, char *, PyObject *);