Bad

Author: Fidget-Spinner

Include/internal/pycore_opcode_metadata.h

@@ -46,6 +46,7 @@
 #define USE_COMPUTED_GOTOS 0
 #include "ceval_macros.h"
 #include "../Include/internal/pycore_code.h"
+#include "../Include/internal/pycore_opcode_metadata.h"
 #include "../Include/internal/pycore_stackref.h"
 
 /* Flow control macros */
@@ -1221,7 +1222,7 @@ dummy_func(
             tstate->current_frame = frame->previous;
             assert(!_PyErr_Occurred(tstate));
             PyObject *result = PyStackRef_AsPyObjectSteal(retval);
-#if !_Py_TAIL_CALL_INTERP
+#if !_Py_TAIL_CALL_INTERP && !TRACING_JIT
             assert(frame == &entry.frame);
 #endif
 #ifdef _Py_TIER2
@@ -1233,7 +1234,13 @@ dummy_func(
             }
 #endif
             LLTRACE_RESUME_FRAME();
+#if defined(TRACING_JIT) && !_Py_TAIL_CALL_INTERP
+            // We should have exited the trace earlier once we hit
+            // the entry frame.
+            Py_UNREACHABLE();
+#else
             return result;
+#endif
         }
 
         // The stack effect here is a bit misleading.
@@ -1522,7 +1529,7 @@ dummy_func(
 
         tier1 inst(CLEANUP_THROW, (sub_iter, last_sent_val, exc_value_st -- none, value)) {
             PyObject *exc_value = PyStackRef_AsPyObjectBorrow(exc_value_st);
-            #if !_Py_TAIL_CALL_INTERP
+            #if !_Py_TAIL_CALL_INTERP && !TRACING_JIT
             assert(throwflag);
             #endif
             assert(exc_value && PyExceptionInstance_Check(exc_value));
@@ -2977,6 +2984,7 @@ dummy_func(
                         DISPATCH();
                     }
                 }
+            #ifndef TRACING_JIT
                 int _is_sys_tracing = (tstate->c_tracefunc != NULL) || (tstate->c_profilefunc != NULL);
                 if (!_is_sys_tracing) {
                     /* Back up over EXTENDED_ARGs so executor is inserted at the correct place */
@@ -2987,12 +2995,41 @@ dummy_func(
                     }
                     _PyJit_InitializeTracing(tstate, frame, insert_exec_at, next_instr, STACK_LEVEL(), 0, NULL);
                     ENTER_TRACING();
+                    SAVE_STACK();
+                    _PyJitTracerReturnValue retval = _PyEval_EvalFrameDefaultTracing(frame, tstate, next_instr, opcode, oparg);
+                    next_instr = retval.next_instr;
+                    RELOAD_STACK();
+                    _PyCeval_LabelIds status = retval.status;
+                    if (status == CEVAL_LABEL_error) {
+                        goto error;
+                    }
+                    else if (status == CEVAL_LABEL_exception_unwind) {
+                        goto exception_unwind;
+                    }
+                    else if (status == CEVAL_LABEL_exit_unwind) {
+                        goto exit_unwind;
+                    }
+                    else if (status == CEVAL_LABEL_pop_1_error) {
+                        goto pop_1_error;
+                    }
+                    else if (status == CEVAL_LABEL_pop_2_error) {
+                        goto pop_2_error;
+                    }
+                    else if (status == CEVAL_LABEL_start_frame) {
+                        goto start_frame;
+                    }
+
                 }
+            #endif
+            #if TRACING_JIT
                 int _jump_taken = false;
                 PyCodeObject *old_code = _PyFrame_GetCode(frame);
                 PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj);
                 int _old_stack_level = 0;
                 TRACING_DISPATCH();
+            #else
+                DISPATCH();
+            #endif
             }
             else {
                 ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter);
@@ -3059,9 +3096,10 @@ dummy_func(
             tstate->jit_exit = NULL;
             #if TRACING_JIT
             RECORD_TRACE_NO_DISPATCH();
-            #endif
-            TIER1_TO_TIER2(executor);
+            // We should have cut the executor here and ended the trace.
             #else
+            TIER1_TO_TIER2(executor);
+            #endif
             Py_FatalError("ENTER_EXECUTOR is not supported in this build");
             #endif /* _Py_TIER2 */
         }
@@ -5622,7 +5660,7 @@ dummy_func(
                 goto exit_unwind;
             }
             next_instr = frame->instr_ptr;
-        #ifdef Py_DEBUG
+        #if defined(Py_DEBUG) && !TRACING_JIT
             int lltrace = maybe_lltrace_resume_frame(frame, GLOBALS());
             if (lltrace < 0) {
                 JUMP_TO_LABEL(exit_unwind);

Python/bytecodes.c

@@ -1048,6 +1048,55 @@ typedef struct {
     _PyStackRef stack[1];
 } _PyEntryFrame;
 
+
+#if defined(_Py_TIER2) && !_Py_TAIL_CALL_INTERP
+
+typedef struct {
+    _PyCeval_LabelIds status;
+    _Py_CODEUNIT *next_instr;
+} _PyJitTracerReturnValue;
+
+static _PyJitTracerReturnValue
+record_trace(PyThreadState *tstate, _PyInterpreterFrame *frame, PyCodeObject *old_code, PyFunctionObject *old_func, int _old_stack_level, _Py_CODEUNIT *this_instr, _Py_CODEUNIT *next_instr, int opcode, int oparg, int _jump_taken)
+{
+    int full = add_to_code_trace(tstate, frame, old_code, old_func, _old_stack_level, this_instr, next_instr, opcode, oparg, _jump_taken);
+    if (full) {
+        int err = bail_tracing_and_jit(tstate, frame);
+        if (err < 0) {
+            return ((_PyJitTracerReturnValue){CEVAL_LABEL_error, next_instr});
+        }
+        return ((_PyJitTracerReturnValue){CEVAL_LABEL_JIT_DONE_TRACING, next_instr});
+    }
+    return ((_PyJitTracerReturnValue){CEVAL_LABEL_JIT_CONTINUE_TRACING, NULL});
+}
+
+_PyJitTracerReturnValue DONT_SLP_VECTORIZE
+_PyEval_EvalFrameDefaultTracing(_PyInterpreterFrame *frame,
+    PyThreadState *tstate, _Py_CODEUNIT *next_instr, int opcode, int oparg)
+{
+#if USE_COMPUTED_GOTOS
+#define USE_JIT_TARGETS
+    /* Import the static jump table */
+#include "opcode_targets.h"
+#undef USE_JIT_TARGETS
+    void **opcode_targets = opcode_tracing_targets_table;
+#endif
+    _Py_CODEUNIT *this_instr = frame->instr_ptr;
+    int _jump_taken = false;
+    PyCodeObject *old_code = _PyFrame_GetCode(frame);
+    PyFunctionObject *old_func = (PyFunctionObject *)PyStackRef_AsPyObjectBorrow(frame->f_funcobj);
+    int _old_stack_level = 0;
+    int is_tracing_flag = true;
+    _PyStackRef *stack_pointer = frame->stackpointer;
+    TRACING_DISPATCH();
+    {
+#include "generated_tracer_cases.c.h"
+    error:
+    return ((_PyJitTracerReturnValue){CEVAL_LABEL_error, next_instr});
+}
+
+#endif
+
 PyObject* _Py_HOT_FUNCTION DONT_SLP_VECTORIZE
 _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag)
 {
@@ -1068,6 +1117,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
     uint8_t opcode;    /* Current opcode */
     int oparg;         /* Current opcode argument, if any */
     assert(tstate->current_frame == NULL || tstate->current_frame->stackpointer != NULL);
+    const bool is_tracing_flag = false;
 #endif
     _PyEntryFrame entry;
 
@@ -1154,7 +1204,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int
 #else
     goto start_frame;
 #   include "generated_cases.c.h"
-    #include "generated_tracer_cases.c.h"
 #endif
 
 

Python/ceval.c

@@ -84,12 +84,6 @@
 #   define Py_PRESERVE_NONE_CC __attribute__((preserve_none))
     Py_PRESERVE_NONE_CC typedef PyObject* (*py_tail_call_funcptr)(TAIL_CALL_PARAMS);
 
-#   define DISPATCH_TABLE_VAR instruction_funcptr_table
-#   define DISPATCH_TABLE instruction_funcptr_handler_table
-#   define TRACING_DISPATCH_TABLE instruction_funcptr_tracing_table
-#   define TARGET(op) Py_PRESERVE_NONE_CC PyObject *_TAIL_CALL_##op(TAIL_CALL_PARAMS)
-#   define TRACING_TARGET(op) Py_PRESERVE_NONE_CC PyObject *_TAIL_CALL_TRACING_##op(TAIL_CALL_PARAMS)
-
 #   define DISPATCH_GOTO() \
         do { \
             Py_MUSTTAIL return (((py_tail_call_funcptr *)instruction_funcptr_table)[opcode])(TAIL_CALL_ARGS); \
@@ -111,8 +105,6 @@
 #   endif
 #    define LABEL(name) TARGET(name)
 #elif USE_COMPUTED_GOTOS
-#  define DISPATCH_TABLE_VAR opcode_targets
-#  define DISPATCH_TABLE opcode_targets_table
 #  define TRACING_DISPATCH_TABLE opcode_tracing_targets_table
 #  define TARGET(op) TARGET_##op:
 #  define TRACING_TARGET(op) TARGET_TRACING_##op:
@@ -129,21 +121,21 @@
 #  define LABEL(name) name:
 #endif
 
-#define TRACING_JUMP_TO_LABEL(label) \
-    RECORD_DYNAMIC_JUMP_TAKEN() \
-    RECORD_TRACE_NO_DISPATCH() \
-    assert(!IS_JIT_TRACING()); \
-    JUMP_TO_LABEL(label);
 
-#if _Py_TAIL_CALL_INTERP || USE_COMPUTED_GOTOS
+#if _Py_TIER2
+#if _Py_TAIL_CALL_INTERP
+#   define DISPATCH_TABLE_VAR instruction_funcptr_table
+#   define DISPATCH_TABLE instruction_funcptr_handler_table
+#   define TRACING_DISPATCH_TABLE instruction_funcptr_tracing_table
+#   define ENTER_TRACING() DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE
+#   define TARGET(op) Py_PRESERVE_NONE_CC PyObject *_TAIL_CALL_##op(TAIL_CALL_PARAMS)
+#   define TRACING_TARGET(op) Py_PRESERVE_NONE_CC PyObject *_TAIL_CALL_TRACING_##op(TAIL_CALL_PARAMS)
+
 #  define IS_JIT_TRACING() (DISPATCH_TABLE_VAR == TRACING_DISPATCH_TABLE)
 // tstate->interp->jit_state.last_specialized_instr != this_instr is required to not get stuck in infinite
 // specialization loops due to specialization failure.
 #  define IS_JIT_TRACING_MAKING_PROGRESS() (IS_JIT_TRACING() && tstate->interp->jit_state.last_specialized_instr != this_instr)
-#  define ENTER_TRACING() \
-    DISPATCH_TABLE_VAR = TRACING_DISPATCH_TABLE;
-#  define LEAVE_TRACING() \
-    DISPATCH_TABLE_VAR = DISPATCH_TABLE;
+#  define LEAVE_TRACING() DISPATCH_TABLE_VAR = DISPATCH_TABLE;
 #  define RECORD_TRACE_NO_DISPATCH() do { \
         int err = 0; \
         _PyFrame_SetStackPointer(frame, stack_pointer); \
@@ -158,8 +150,33 @@
         stack_pointer = _PyFrame_GetStackPointer(frame); \
         if (err < 0) { JUMP_TO_LABEL(error); }  \
     } while (0);
-#endif
+#define TRACING_JUMP_TO_LABEL(label) \
+    RECORD_DYNAMIC_JUMP_TAKEN() \
+    RECORD_TRACE_NO_DISPATCH() \
+    assert(!IS_JIT_TRACING()); \
+    JUMP_TO_LABEL(label);
 
+#else
+// Note: computed gotos interpreter slows down a lot with tracing.
+// So we can't use the same dispatch table technique the tail calling
+// interpreter uses.
+#  define ENTER_TRACING()
+#  define IS_JIT_TRACING() (is_tracing_flag)
+#  define IS_JIT_TRACING_MAKING_PROGRESS() (IS_JIT_TRACING() && tstate->interp->jit_state.last_specialized_instr != this_instr)
+#  define RECORD_TRACE_NO_ERR_CHECK() \
+        _PyFrame_SetStackPointer(frame, stack_pointer); \
+        _PyJitTracerReturnValue jit_status = record_trace(tstate, frame, old_code, old_func, _old_stack_level, this_instr, next_instr, opcode, oparg, _jump_taken); \
+        stack_pointer = _PyFrame_GetStackPointer(frame);
+#  define RECORD_TRACE_NO_DISPATCH() \
+        RECORD_TRACE_NO_ERR_CHECK(); \
+        if (jit_status.status != CEVAL_LABEL_JIT_CONTINUE_TRACING) { return jit_status; }
+#define TRACING_JUMP_TO_LABEL(label) \
+    RECORD_DYNAMIC_JUMP_TAKEN(); \
+    RECORD_TRACE_NO_ERR_CHECK(); \
+    (void)(jit_status); \
+    return ((_PyJitTracerReturnValue){CEVAL_LABEL_##label, next_instr});
+#endif
+#endif
 
 /* PRE_DISPATCH_GOTO() does lltrace if enabled. Normally a no-op */
 #ifdef Py_DEBUG
@@ -204,14 +221,6 @@ do { \
         DISPATCH_GOTO(); \
     }
 
-#define TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() \
-{ \
-    tstate->interp->jit_state.last_specialized_instr = this_instr; \
-    opcode = next_instr->op.code; \
-    PRE_DISPATCH_GOTO(); \
-    DISPATCH_GOTO(); \
-}
-
 #define DISPATCH_INLINED(NEW_FRAME)                     \
     do {                                                \
         assert(tstate->interp->eval_frame == NULL);     \
@@ -222,6 +231,14 @@ do { \
         JUMP_TO_LABEL(start_frame);                      \
     } while (0)
 
+#define TRACING_SPECIALIZE_DISPATCH_SAME_OPARG() \
+{ \
+    tstate->interp->jit_state.last_specialized_instr = this_instr; \
+    opcode = next_instr->op.code; \
+    PRE_DISPATCH_GOTO(); \
+    DISPATCH_GOTO(); \
+}
+
 #define TRACING_DISPATCH_INLINED(NEW_FRAME) \
     tstate->interp->jit_state.last_specialized_instr = this_instr; \
     RECORD_TRACE_NO_DISPATCH(); \