Skip to content

Commit d47c84c

Browse files
committed
WIP: rebasing pyutils#334: start from a clean slate
CHANGELOG.rst line_profiler/CMakeLists.txt setup.py Updated (cherry-picked from where we were before merging pyutils#347) - `CHANGELOG.rst`: Added entry - <Others>: Added handling for the below files line_profiler/Python_wrapper.h line_profiler/c_trace_callbacks.{c,h} Added new files to be used by `line_profiler/_line_profiler.pyx` (cherry-picked from where we were before merging pyutils#347) - `Python_wrapper.h`: New header file which wraps around `Python.h` and provides compatibility layer over CPython C APIs - `c_trace_callbacks.*`: New source/header files for code which handles the retrieval and use of C-level trace callbacks
1 parent 5817f3e commit d47c84c

6 files changed

Lines changed: 313 additions & 2 deletions

File tree

CHANGELOG.rst

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,10 @@ Changes
2525
like class methods and properties
2626
* ``LineProfiler`` can now be used as a class decorator
2727
* FIX: Fixed line tracing for Cython code; superseded use of the legacy tracing system with ``sys.monitoring``
28-
* ENH: Fixed edge case where :py:meth:`LineProfiler.get_stats()` neglects data from duplicate code objects (#348)
28+
* ENH: Fixed edge cases where:
29+
* ``LineProfiler.get_stats()`` neglects data from duplicate code objects (#348)
30+
* ``LineProfiler`` instances may stop receiving tracing events when multiple instances are used (#350)
31+
* FIX: ``LineProfiler`` now caches the existing ``sys`` or ``sys.monitoring`` trace callbacks in ``.enable()`` and restores them in ``.disable()``, instead of always discarding it on the way out; also added experimental support for calling (instead of suspending) said callbacks during profiling (#333)
2932

3033
4.2.0
3134
~~~~~

line_profiler/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ add_cython_target(${module_name} "${cython_source}" C OUTPUT_VAR sources)
1010
# Add any other non-cython dependencies to the sources
1111
list(APPEND sources
1212
"${CMAKE_CURRENT_SOURCE_DIR}/timers.c"
13+
"${CMAKE_CURRENT_SOURCE_DIR}/c_trace_callbacks.c"
1314
)
1415
message(STATUS "[OURS] sources = ${sources}")
1516

line_profiler/Python_wrapper.h

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
// Compatibility layer over `Python.h`.
2+
3+
#ifndef LINE_PROFILER_PYTHON_WRAPPER_H
4+
#define LINE_PROFILER_PYTHON_WRAPPER_H
5+
6+
#include "Python.h"
7+
8+
// CPython 3.11 broke some stuff by moving PyFrameObject :(
9+
#if PY_VERSION_HEX >= 0x030b00a6
10+
#ifndef Py_BUILD_CORE
11+
#define Py_BUILD_CORE 1
12+
#endif
13+
#include "internal/pycore_frame.h"
14+
#include "cpython/code.h"
15+
#include "pyframe.h"
16+
#endif
17+
18+
// Ensure PyFrameObject availability
19+
#if PY_VERSION_HEX < 0x030900b1 // 3.9.0b1
20+
#include "frameobject.h"
21+
#endif
22+
23+
#if PY_VERSION_HEX < 0x030900b1 // 3.9.0b1
24+
/*
25+
* Notes:
26+
* While 3.9.0a1 already has `PyFrame_GetCode()`, it doesn't
27+
* INCREF the code object until 0b1 (PR #19773), so override
28+
* that for consistency.
29+
*/
30+
#define PyFrame_GetCode(x) PyFrame_GetCode_backport(x)
31+
inline PyCodeObject *PyFrame_GetCode_backport(PyFrameObject *frame)
32+
{
33+
PyCodeObject *code;
34+
assert(frame != NULL);
35+
code = frame->f_code;
36+
assert(code != NULL);
37+
Py_INCREF(code);
38+
return code;
39+
}
40+
#endif
41+
42+
#if PY_VERSION_HEX < 0x030B00b1 // 3.11.0b1
43+
/*
44+
* Notes:
45+
* Since 3.11.0a7 (PR #31888) `co_code` has been made a
46+
* descriptor, so:
47+
* - This already creates a NewRef, so don't INCREF in that
48+
* case; and
49+
* - `code->co_code` will not work.
50+
*/
51+
inline PyObject *PyCode_GetCode(PyCodeObject *code)
52+
{
53+
PyObject *code_bytes;
54+
if (code == NULL) return NULL;
55+
#if PY_VERSION_HEX < 0x030B00a7 // 3.11.0a7
56+
code_bytes = code->co_code;
57+
Py_XINCREF(code_bytes);
58+
#else
59+
code_bytes = PyObject_GetAttrString(code, "co_code");
60+
#endif
61+
return code_bytes;
62+
}
63+
#endif
64+
65+
#if PY_VERSION_HEX < 0x030D00a1 // 3.13.0a1
66+
inline PyObject *PyImport_AddModuleRef(const char *name)
67+
{
68+
PyObject *mod = NULL, *name_str = NULL;
69+
name_str = PyUnicode_FromString(name);
70+
if (name_str == NULL) goto cleanup;
71+
mod = PyImport_AddModuleObject(name_str);
72+
Py_XINCREF(mod);
73+
cleanup:
74+
Py_XDECREF(name_str);
75+
return mod;
76+
}
77+
#endif
78+
79+
#endif // LINE_PROFILER_PYTHON_WRAPPER_H

line_profiler/c_trace_callbacks.c

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
#include "c_trace_callbacks.h"
2+
3+
#define CYTHON_MODULE "line_profiler._line_profiler"
4+
#define DISABLE_CALLBACK "disable_line_events"
5+
#define RAISE_IN_CALL(func_name, xc, const_msg) \
6+
PyErr_SetString(xc, \
7+
"in `" CYTHON_MODULE "." func_name "()`: " \
8+
const_msg)
9+
10+
TraceCallback *alloc_callback()
11+
{
12+
/* Heap-allocate a new `TraceCallback`. */
13+
TraceCallback *callback = (TraceCallback*)malloc(sizeof(TraceCallback));
14+
if (callback == NULL) RAISE_IN_CALL(
15+
// If we're here we have bigger fish to fry... but be nice and
16+
// raise an error explicitly anyway
17+
"alloc_callback",
18+
PyExc_MemoryError,
19+
"failed to allocate memory for storing the existing "
20+
"`sys` trace callback"
21+
);
22+
return callback;
23+
}
24+
25+
void free_callback(TraceCallback *callback)
26+
{
27+
/* Free a heap-allocated `TraceCallback`. */
28+
if (callback != NULL) free(callback);
29+
return;
30+
}
31+
32+
void fetch_callback(TraceCallback *callback)
33+
{
34+
/* Store the members `.c_tracefunc` and `.c_traceobj` of the
35+
* current thread on `callback`.
36+
*/
37+
// Shouldn't happen, but just to be safe
38+
if (callback == NULL) return;
39+
// No need to `Py_DECREF()` the thread callback, since it isn't a
40+
// `PyObject`
41+
PyThreadState *thread_state = PyThreadState_Get();
42+
callback->c_tracefunc = thread_state->c_tracefunc;
43+
callback->c_traceobj = thread_state->c_traceobj;
44+
// No need for NULL check with `Py_XINCREF()`
45+
Py_XINCREF(callback->c_traceobj);
46+
return;
47+
}
48+
49+
void nullify_callback(TraceCallback *callback)
50+
{
51+
// No need for NULL check with `Py_XDECREF()`
52+
Py_XDECREF(callback->c_traceobj);
53+
callback->c_tracefunc = NULL;
54+
callback->c_traceobj = NULL;
55+
return;
56+
}
57+
58+
void restore_callback(TraceCallback *callback)
59+
{
60+
/* Use `PyEval_SetTrace()` to set the trace callback on the current
61+
* thread to be consistent with the `callback`, then nullify the
62+
* pointers on `callback`.
63+
*/
64+
// Shouldn't happen, but just to be safe
65+
if (callback == NULL) return;
66+
PyEval_SetTrace(callback->c_tracefunc, callback->c_traceobj);
67+
nullify_callback(callback);
68+
return;
69+
}
70+
71+
inline int is_null_callback(TraceCallback *callback)
72+
{
73+
return (
74+
callback == NULL
75+
|| callback->c_tracefunc == NULL
76+
|| callback->c_traceobj == NULL
77+
);
78+
}
79+
80+
int call_callback(
81+
TraceCallback *callback,
82+
PyFrameObject *py_frame,
83+
int what,
84+
PyObject *arg
85+
)
86+
{
87+
/* Call the cached trace callback `callback` where appropriate, and
88+
* in a "safe" way so that:
89+
* - If it alters the `sys` trace callback, or
90+
* - If it sets `.f_trace_lines` to false,
91+
* said alterations are reverted so as not to hinder profiling.
92+
*
93+
* Returns:
94+
* - 0 if `callback` is `NULL` or has nullified members;
95+
* - -1 if an error occurs (e.g. when the disabling of line
96+
* events for the frame-local trace function failed);
97+
* - The result of calling said callback otherwise.
98+
*
99+
* Side effects:
100+
* - If the callback unsets the `sys` callback, the `sys`
101+
* callback is preserved but `callback` itself is nullified.
102+
* This is to comply with what Python usually does: if the
103+
* trace callback errors out, `sys.settrace(None)` is called.
104+
* - If a frame-local callback sets the `.f_trace_lines` to
105+
* false, `.f_trace_lines` is reverted but `.f_trace` is
106+
* wrapped so that it no loger sees line events.
107+
*
108+
* Notes:
109+
* It is tempting to assume said current callback value to be
110+
* `{ python_trace_callback, <profiler> }`, but remember that
111+
* our callback may very well be called via another callback,
112+
* much like how we call the cached callback via
113+
* `python_trace_callback()`.
114+
*/
115+
TraceCallback before, after;
116+
PyObject *mod = NULL, *dle = NULL, *f_trace = NULL;
117+
char f_trace_lines;
118+
int result;
119+
120+
if (is_null_callback(callback)) return 0;
121+
122+
f_trace_lines = py_frame->f_trace_lines;
123+
fetch_callback(&before);
124+
result = (callback->c_tracefunc)(
125+
callback->c_traceobj, py_frame, what, arg
126+
);
127+
128+
// Check if the callback has unset itself; if so, nullify `callback`
129+
fetch_callback(&after);
130+
if (is_null_callback(&after)) nullify_callback(callback);
131+
nullify_callback(&after);
132+
restore_callback(&before);
133+
134+
// Check if a callback has disabled future line events for the
135+
// frame, and if so, revert the change while withholding future line
136+
// events from the callback
137+
if (
138+
!(py_frame->f_trace_lines)
139+
&& f_trace_lines != py_frame->f_trace_lines
140+
)
141+
{
142+
py_frame->f_trace_lines = f_trace_lines;
143+
if (py_frame->f_trace != NULL && py_frame->f_trace != Py_None)
144+
{
145+
// FIXME: can we get more performance by stashing a somewhat
146+
// permanent reference to
147+
// `line_profiler._line_profiler.disable_line_events()`
148+
// somewhere?
149+
mod = PyImport_AddModuleRef(CYTHON_MODULE);
150+
if (mod == NULL)
151+
{
152+
RAISE_IN_CALL(
153+
"call_callback",
154+
PyExc_ImportError,
155+
"cannot import `" CYTHON_MODULE "`"
156+
);
157+
result = -1;
158+
goto cleanup;
159+
}
160+
dle = PyObject_GetAttrString(mod, DISABLE_CALLBACK);
161+
if (dle == NULL)
162+
{
163+
RAISE_IN_CALL(
164+
"call_callback",
165+
PyExc_AttributeError,
166+
"`line_profiler._line_profiler` has no "
167+
"attribute `" DISABLE_CALLBACK "`"
168+
);
169+
result = -1;
170+
goto cleanup;
171+
}
172+
// Note: DON'T `Py_[X]DECREF()` the pointer! Nothing else is
173+
// holding a reference to it.
174+
f_trace = PyObject_CallFunctionObjArgs(
175+
dle, py_frame->f_trace, NULL
176+
);
177+
if (f_trace == NULL)
178+
{
179+
// No need to raise another exception, it's already
180+
// raised in the call
181+
result = -1;
182+
goto cleanup;
183+
}
184+
py_frame->f_trace = f_trace;
185+
}
186+
}
187+
cleanup:
188+
Py_XDECREF(mod);
189+
Py_XDECREF(dle);
190+
return result;
191+
}

line_profiler/c_trace_callbacks.h

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
#ifndef LINE_PROFILER_C_TRACE_CALLBACKS_H
2+
#define LINE_PROFILER_C_TRACE_CALLBACKS_H
3+
4+
#include "Python_wrapper.h"
5+
#include "frameobject.h"
6+
7+
typedef struct TraceCallback
8+
{
9+
/* Notes:
10+
* - These fields are synonymous with the corresponding fields
11+
* in a `PyThreadState` object;
12+
* however, note that `PyThreadState.c_tracefunc` is
13+
* considered a CPython implementation detail.
14+
* - It is necessary to reach into the thread-state internals
15+
* like this, because `sys.gettrace()` only retrieves
16+
* `.c_traceobj`, and is thus only valid for Python-level
17+
* trace callables set via `sys.settrace()` (which implicitly
18+
* sets `.c_tracefunc` to
19+
* `Python/sysmodule.c::trace_trampoline()`).
20+
*/
21+
Py_tracefunc c_tracefunc;
22+
PyObject *c_traceobj;
23+
} TraceCallback;
24+
25+
TraceCallback *alloc_callback();
26+
void free_callback(TraceCallback *callback);
27+
void fetch_callback(TraceCallback *callback);
28+
void restore_callback(TraceCallback *callback);
29+
int call_callback(
30+
TraceCallback *callback,
31+
PyFrameObject *py_frame,
32+
int what,
33+
PyObject *arg
34+
);
35+
36+
#endif // LINE_PROFILER_C_TRACE_CALLBACKS_H

setup.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -229,7 +229,8 @@ def run_cythonize(force=False):
229229
Extension(
230230
name="line_profiler._line_profiler",
231231
sources=["line_profiler/_line_profiler.pyx",
232-
"line_profiler/timers.c"],
232+
"line_profiler/timers.c",
233+
"line_profiler/c_trace_callbacks.c"],
233234
language="c++",
234235
define_macros=[("CYTHON_TRACE", (1 if os.getenv("DEV") == "true" else 0))],
235236
),

0 commit comments

Comments
 (0)