Make frames a union of interpreter frames and builtin frames

Author: markshannon

Include/cpython/pystate.h

@@ -124,7 +124,7 @@ struct _ts {
     int what_event; /* The event currently being monitored, if any. */
 
     /* Pointer to currently executing frame. */
-    struct _PyInterpreterFrame *current_frame;
+    union _PyVMFrame *current_frame;
 
     Py_tracefunc c_profilefunc;
     Py_tracefunc c_tracefunc;

Include/internal/pycore_frame.h

@@ -61,32 +61,44 @@ enum _frameowner {
     FRAME_OWNED_BY_CSTACK = 4,
 };
 
+struct _PyFrameCore {
+    _PyStackRef f_executable; /* Deferred or strong reference */
+    union _PyVMFrame *previous;
+    char owner;
+};
+
 typedef struct _PyInterpreterFrame {
     _PyStackRef f_executable; /* Deferred or strong reference (code object or None) */
-    struct _PyInterpreterFrame *previous;
-    _PyStackRef f_funcobj; /* Deferred or strong reference. Only valid if not on C stack */
-    PyObject *f_globals; /* Borrowed reference. Only valid if not on C stack */
-    PyObject *f_builtins; /* Borrowed reference. Only valid if not on C stack */
-    PyObject *f_locals; /* Strong reference, may be NULL. Only valid if not on C stack */
-    PyFrameObject *frame_obj; /* Strong reference, may be NULL. Only valid if not on C stack */
-    _Py_CODEUNIT *instr_ptr; /* Instruction currently executing (or about to begin) */
-    _PyStackRef *stackpointer;
-#ifdef Py_GIL_DISABLED
-    /* Index of thread-local bytecode containing instr_ptr. */
-    int32_t tlbc_index;
-#endif
-    uint16_t return_offset;  /* Only relevant during a function call */
+    union _PyVMFrame *previous;
     char owner;
 #ifdef Py_DEBUG
     uint8_t visited:1;
     uint8_t lltrace:7;
 #else
     uint8_t visited;
 #endif
+    uint16_t return_offset;  /* Only relevant during a function call */
+#ifdef Py_GIL_DISABLED
+    /* Index of thread-local bytecode containing instr_ptr. */
+    int32_t tlbc_index;
+#endif
+    _PyStackRef f_funcobj; /* Deferred or strong reference. Only valid if not on C stack */
+    PyObject *f_globals; /* Borrowed reference. Only valid if not on C stack */
+    PyObject *f_builtins; /* Borrowed reference. Only valid if not on C stack */
+    PyObject *f_locals; /* Strong reference, may be NULL. Only valid if not on C stack */
+    PyFrameObject *frame_obj; /* Strong reference, may be NULL. Only valid if not on C stack */
+    _Py_CODEUNIT *instr_ptr; /* Instruction currently executing (or about to begin) */
+    _PyStackRef *stackpointer;
     /* Locals and stack */
     _PyStackRef localsplus[1];
 } _PyInterpreterFrame;
 
+
+typedef union _PyVMFrame {
+    struct _PyFrameCore core;
+    _PyInterpreterFrame iframe;
+} _PyVMFrame;
+
 #define _PyInterpreterFrame_LASTI(IF) \
     ((int)((IF)->instr_ptr - _PyFrame_GetBytecode((IF))))
 
@@ -200,7 +212,7 @@ _PyFrame_Initialize(
     PyThreadState *tstate, _PyInterpreterFrame *frame, _PyStackRef func,
     PyObject *locals, PyCodeObject *code, int null_locals_from, _PyInterpreterFrame *previous)
 {
-    frame->previous = previous;
+    frame->previous = (_PyVMFrame *)previous;
     frame->f_funcobj = func;
     frame->f_executable = PyStackRef_FromPyObjectNew(code);
     PyFunctionObject *func_obj = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(func);
@@ -264,23 +276,32 @@ _PyFrame_SetStackPointer(_PyInterpreterFrame *frame, _PyStackRef *stack_pointer)
  * Frames owned by a generator are always complete.
  */
 static inline bool
-_PyFrame_IsIncomplete(_PyInterpreterFrame *frame)
+_PyVMFrame_IsIncomplete(_PyVMFrame *frame)
 {
-    if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) {
+    if (frame->core.owner >= FRAME_OWNED_BY_INTERPRETER) {
         return true;
     }
-    return frame->owner != FRAME_OWNED_BY_GENERATOR &&
-           frame->instr_ptr < _PyFrame_GetBytecode(frame) +
-                                  _PyFrame_GetCode(frame)->_co_firsttraceable;
+    if (frame->core.owner == FRAME_OWNED_BY_GENERATOR) {
+        return false;
+    }
+    _PyInterpreterFrame *iframe = &frame->iframe;
+    return iframe->instr_ptr < _PyFrame_GetBytecode(iframe) +
+                                  _PyFrame_GetCode(iframe)->_co_firsttraceable;
+}
+
+static inline bool
+_PyFrame_IsIncomplete(_PyInterpreterFrame *frame)
+{
+    return _PyVMFrame_IsIncomplete((_PyVMFrame *)frame);
 }
 
 static inline _PyInterpreterFrame *
-_PyFrame_GetFirstComplete(_PyInterpreterFrame *frame)
+_PyFrame_GetFirstComplete(_PyVMFrame *frame)
 {
-    while (frame && _PyFrame_IsIncomplete(frame)) {
-        frame = frame->previous;
+    while (frame && _PyVMFrame_IsIncomplete(frame)) {
+        frame = frame->core.previous;
     }
-    return frame;
+    return (_PyInterpreterFrame *)frame;
 }
 
 static inline _PyInterpreterFrame *
@@ -376,7 +397,7 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int
     _PyInterpreterFrame *frame = (_PyInterpreterFrame *)tstate->datastack_top;
     tstate->datastack_top += code->co_framesize;
     assert(tstate->datastack_top < tstate->datastack_limit);
-    frame->previous = previous;
+    frame->previous = (_PyVMFrame *)previous;
     frame->f_funcobj = PyStackRef_None;
     frame->f_executable = PyStackRef_FromPyObjectNew(code);
 #ifdef Py_DEBUG

Objects/frameobject.c

@@ -2221,7 +2221,7 @@ _PyFrame_IsEntryFrame(PyFrameObject *frame)
     assert(frame != NULL);
     _PyInterpreterFrame *f = frame->f_frame;
     assert(!_PyFrame_IsIncomplete(f));
-    return f->previous && f->previous->owner == FRAME_OWNED_BY_INTERPRETER;
+    return f->previous && f->previous->core.owner == FRAME_OWNED_BY_INTERPRETER;
 }
 
 PyCodeObject *
@@ -2242,10 +2242,10 @@ PyFrame_GetBack(PyFrameObject *frame)
     assert(!_PyFrame_IsIncomplete(frame->f_frame));
     PyFrameObject *back = frame->f_back;
     if (back == NULL) {
-        _PyInterpreterFrame *prev = frame->f_frame->previous;
-        prev = _PyFrame_GetFirstComplete(prev);
-        if (prev) {
-            back = _PyFrame_GetFrameObject(prev);
+        _PyVMFrame *prev = frame->f_frame->previous;
+        _PyInterpreterFrame *iframe = _PyFrame_GetFirstComplete(prev);
+        if (iframe) {
+            back = _PyFrame_GetFrameObject(iframe);
         }
     }
     return (PyFrameObject*)Py_XNewRef(back);

Objects/genobject.c

@@ -490,9 +490,9 @@ _gen_throw(PyGenObject *gen, int close_on_genexit,
             /* Link frame into the stack to enable complete backtraces. */
             /* XXX We should probably be updating the current frame somewhere in
                ceval.c. */
-            _PyInterpreterFrame *prev = tstate->current_frame;
+            _PyVMFrame *prev = tstate->current_frame;
             frame->previous = prev;
-            tstate->current_frame = frame;
+            tstate->current_frame = (_PyVMFrame *)frame;
             /* Close the generator that we are currently iterating with
                'yield from' or awaiting on with 'await'. */
             PyFrameState state = gen->gi_frame_state;
@@ -514,9 +514,9 @@ _gen_throw(PyGenObject *gen, int close_on_genexit,
                 goto throw_here;
             }
 
-            _PyInterpreterFrame *prev = tstate->current_frame;
+            _PyVMFrame *prev = tstate->current_frame;
             frame->previous = prev;
-            tstate->current_frame = frame;
+            tstate->current_frame = (_PyVMFrame *)frame;
             PyFrameState state = gen->gi_frame_state;
             gen->gi_frame_state = FRAME_EXECUTING;
             ret = PyObject_CallFunctionObjArgs(meth, typ, val, tb, NULL);

Objects/object.c

@@ -2991,7 +2991,8 @@ _Py_Dealloc(PyObject *op)
 #if !defined(Py_GIL_DISABLED) && !defined(Py_STACKREF_DEBUG)
     /* This assertion doesn't hold for the free-threading build, as
      * PyStackRef_CLOSE_SPECIALIZED is not implemented */
-    assert(tstate->current_frame == NULL || tstate->current_frame->stackpointer != NULL);
+    assert(_PyFrame_GetFirstComplete(tstate->current_frame) == NULL ||
+           _PyFrame_GetFirstComplete(tstate->current_frame)->stackpointer != NULL);
 #endif
     PyObject *old_exc = tstate != NULL ? tstate->current_exception : NULL;
     // Keep the old exception type alive to prevent undefined behavior

Python/bytecodes.c

@@ -1113,7 +1113,8 @@ dummy_func(
             _Py_LeaveRecursiveCallPy(tstate);
             // GH-99729: We need to unlink the frame *before* clearing it:
             _PyInterpreterFrame *dying = frame;
-            frame = tstate->current_frame = dying->previous;
+            frame = (_PyInterpreterFrame *)frame->previous;
+            tstate->current_frame = (_PyVMFrame *)frame;
             _PyEval_FrameClearAndPop(tstate, dying);
             RELOAD_STACK();
             LOAD_IP(frame->return_offset);
@@ -1218,7 +1219,7 @@ dummy_func(
                 assert(INSTRUCTION_SIZE + oparg <= UINT16_MAX);
                 frame->return_offset = (uint16_t)(INSTRUCTION_SIZE + oparg);
                 assert(gen_frame->previous == NULL);
-                gen_frame->previous = frame;
+                gen_frame->previous = (_PyVMFrame *)frame;
                 DISPATCH_INLINED(gen_frame);
             }
             if (PyStackRef_IsNone(v) && PyIter_Check(receiver_o)) {
@@ -1263,7 +1264,7 @@ dummy_func(
             tstate->exc_info = &gen->gi_exc_state;
             assert(INSTRUCTION_SIZE + oparg <= UINT16_MAX);
             frame->return_offset = (uint16_t)(INSTRUCTION_SIZE + oparg);
-            gen_frame->previous = frame;
+            gen_frame->previous = (_PyVMFrame *)frame;
         }
 
         macro(SEND_GEN) =
@@ -1289,7 +1290,8 @@ dummy_func(
             gen->gi_exc_state.previous_item = NULL;
             _Py_LeaveRecursiveCallPy(tstate);
             _PyInterpreterFrame *gen_frame = frame;
-            frame = tstate->current_frame = frame->previous;
+            frame = (_PyInterpreterFrame *)frame->previous;
+            tstate->current_frame = (_PyVMFrame *)frame;
             gen_frame->previous = NULL;
             /* We don't know which of these is relevant here, so keep them equal */
             assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER);
@@ -3335,7 +3337,7 @@ dummy_func(
             gen->gi_frame_state = FRAME_EXECUTING;
             gen->gi_exc_state.previous_item = tstate->exc_info;
             tstate->exc_info = &gen->gi_exc_state;
-            gen_frame->previous = frame;
+            gen_frame->previous = (_PyVMFrame *)frame;
             // oparg is the return offset from the next instruction.
             frame->return_offset = (uint16_t)(INSTRUCTION_SIZE + oparg);
         }
@@ -3853,9 +3855,10 @@ dummy_func(
             DEAD(new_frame);
             SYNC_SP();
             _PyFrame_SetStackPointer(frame, stack_pointer);
-            assert(new_frame->previous == frame || new_frame->previous->previous == frame);
+            assert(new_frame->previous == (_PyVMFrame *)frame || new_frame->previous->core.previous == (_PyVMFrame *)frame);
             CALL_STAT_INC(inlined_py_calls);
-            frame = tstate->current_frame = temp;
+            tstate->current_frame = (_PyVMFrame *)temp;
+            frame = temp;
             tstate->py_recursion_remaining--;
             LOAD_SP();
             LOAD_IP(0);
@@ -4794,9 +4797,10 @@ dummy_func(
             gen->gi_frame_state = FRAME_CREATED;
             gen_frame->owner = FRAME_OWNED_BY_GENERATOR;
             _Py_LeaveRecursiveCallPy(tstate);
-            _PyInterpreterFrame *prev = frame->previous;
+            _PyVMFrame *prev = frame->previous;
             _PyThreadState_PopFrame(tstate, frame);
-            frame = tstate->current_frame = prev;
+            frame = (_PyInterpreterFrame *)prev;
+            tstate->current_frame = prev;
             LOAD_IP(frame->return_offset);
             RELOAD_STACK();
             res = PyStackRef_FromPyObjectStealMortal((PyObject *)gen);
@@ -5296,7 +5300,8 @@ dummy_func(
             assert(frame->owner != FRAME_OWNED_BY_INTERPRETER);
             // GH-99729: We need to unlink the frame *before* clearing it:
             _PyInterpreterFrame *dying = frame;
-            frame = tstate->current_frame = dying->previous;
+            frame = (_PyInterpreterFrame *)frame->previous;
+            tstate->current_frame = (_PyVMFrame *)frame;
             _PyEval_FrameClearAndPop(tstate, dying);
             frame->return_offset = 0;
             if (frame->owner == FRAME_OWNED_BY_INTERPRETER) {

Python/ceval.c

@@ -967,7 +967,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
 #if !Py_TAIL_CALL_INTERP
     uint8_t opcode;    /* Current opcode */
     int oparg;         /* Current opcode argument, if any */
-    assert(tstate->current_frame == NULL || tstate->current_frame->stackpointer != NULL);
+    assert(_PyFrame_GetFirstComplete(tstate->current_frame) == NULL ||
+           _PyFrame_GetFirstComplete(tstate->current_frame)->stackpointer != NULL);
 #endif
     _PyInterpreterFrame entry_frame;
 
@@ -1001,8 +1002,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
 #endif
     /* Push frame */
     entry_frame.previous = tstate->current_frame;
-    frame->previous = &entry_frame;
-    tstate->current_frame = frame;
+    frame->previous = (_PyVMFrame *)&entry_frame;
+    tstate->current_frame = (_PyVMFrame *)frame;
 
     /* support for generator.throw() */
     if (throwflag) {
@@ -1156,12 +1157,13 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
     assert(frame->owner != FRAME_OWNED_BY_INTERPRETER);
     // GH-99729: We need to unlink the frame *before* clearing it:
     _PyInterpreterFrame *dying = frame;
-    frame = tstate->current_frame = dying->previous;
+    _PyVMFrame *prev = dying->previous;
+    tstate->current_frame = prev;
     _PyEval_FrameClearAndPop(tstate, dying);
-    frame->return_offset = 0;
-    assert(frame->owner == FRAME_OWNED_BY_INTERPRETER);
+    assert(prev->core.owner == FRAME_OWNED_BY_INTERPRETER);
+    prev->iframe.return_offset = 0;
     /* Restore previous frame and exit */
-    tstate->current_frame = frame->previous;
+    tstate->current_frame = prev->core.previous;
     return NULL;
 }
 
@@ -2711,11 +2713,11 @@ int
 PyEval_MergeCompilerFlags(PyCompilerFlags *cf)
 {
     PyThreadState *tstate = _PyThreadState_GET();
-    _PyInterpreterFrame *current_frame = tstate->current_frame;
+    _PyVMFrame *current_frame = tstate->current_frame;
     int result = cf->cf_flags != 0;
 
-    if (current_frame != NULL) {
-        const int codeflags = _PyFrame_GetCode(current_frame)->co_flags;
+    if (current_frame != NULL && !_PyVMFrame_IsIncomplete(current_frame)) {
+        const int codeflags = _PyFrame_GetCode((_PyInterpreterFrame *)current_frame)->co_flags;
         const int compilerflags = codeflags & PyCF_MASK;
         if (compilerflags) {
             result = 1;

Python/ceval_macros.h

@@ -157,10 +157,11 @@ do { \
     do {                                                \
         assert(tstate->interp->eval_frame == NULL);     \
         _PyFrame_SetStackPointer(frame, stack_pointer); \
-        assert((NEW_FRAME)->previous == frame);         \
-        frame = tstate->current_frame = (NEW_FRAME);     \
+        assert((NEW_FRAME)->previous == (_PyVMFrame *)frame);  \
+        frame = (NEW_FRAME);                            \
+        tstate->current_frame = (_PyVMFrame *)frame;    \
         CALL_STAT_INC(inlined_py_calls);                \
-        JUMP_TO_LABEL(start_frame);                      \
+        JUMP_TO_LABEL(start_frame);                     \
     } while (0)
 
 /* Tuple access macros */