Skip to content

Commit dd3cc6d

Browse files
topolaritytopolarity
committed
Acquire GIL around every Python API call
Majority of the API is covered by updating the `@pycheck(v/n)` macros. The rest is covered manually by `@with_gil`.
1 parent 2f600fb commit dd3cc6d

File tree

11 files changed

+117
-65
lines changed

11 files changed

+117
-65
lines changed

‎src/PyCall.jl‎

Lines changed: 23 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ import Base.Iterators: filter
3636

3737
include(joinpath(dirname(@__FILE__), "..", "deps","depsutils.jl"))
3838
include("startup.jl")
39+
include("gil.jl")
3940

4041
"""
4142
Python executable used by PyCall in the current process.
@@ -114,19 +115,21 @@ it is equivalent to a `PyNULL()` object.
114115
"""
115116
ispynull(o::PyObject) = o ≛ PyPtr_NULL
116117

117-
function pydecref_(o::Union{PyPtr,PyObject})
118-
_finalized[] || ccall(@pysym(:Py_DecRef), Cvoid, (PyPtr,), o)
118+
function pydecref_(o::PyPtr)
119+
if o !== PyPtr_NULL && !_finalized[]
120+
@with_gil ccall(@pysym(:Py_DecRef), Cvoid, (PyPtr,), o)
121+
end
119122
return o
120123
end
121124

122125
function pydecref(o::PyObject)
123-
pydecref_(o)
126+
pydecref_(getfield(o, :o))
124127
setfield!(o, :o, PyPtr_NULL)
125128
return o
126129
end
127130

128131
function pyincref_(o::Union{PyPtr,PyObject})
129-
ccall((@pysym :Py_IncRef), Cvoid, (PyPtr,), o)
132+
@with_gil ccall((@pysym :Py_IncRef), Cvoid, (PyPtr,), o)
130133
return o
131134
end
132135

@@ -165,13 +168,13 @@ function Base.copy!(dest::PyObject, src::PyObject)
165168
end
166169

167170
pyisinstance(o::PyObject, t::PyObject) =
168-
!ispynull(t) && ccall((@pysym :PyObject_IsInstance), Cint, (PyPtr,PyPtr), o, t) == 1
171+
!ispynull(t) && @with_gil(ccall((@pysym :PyObject_IsInstance), Cint, (PyPtr,PyPtr), o, t)) == 1
169172

170173
pyisinstance(o::PyObject, t::Union{Ptr{Cvoid},PyPtr}) =
171-
t != C_NULL && ccall((@pysym :PyObject_IsInstance), Cint, (PyPtr,PyPtr), o, t) == 1
174+
t != C_NULL && @with_gil(ccall((@pysym :PyObject_IsInstance), Cint, (PyPtr,PyPtr), o, t)) == 1
172175

173176
pyquery(q::Ptr{Cvoid}, o::PyObject) =
174-
ccall(q, Cint, (PyPtr,), o) == 1
177+
@with_gil(ccall(q, Cint, (PyPtr,), o)) == 1
175178

176179
# conversion to pass PyObject as ccall arguments:
177180
unsafe_convert(::Type{PyPtr}, po::PyObject) = PyPtr(po)
@@ -240,10 +243,10 @@ function pystring(o::PyObject)
240243
if ispynull(o)
241244
return "NULL"
242245
else
243-
s = ccall((@pysym :PyObject_Repr), PyPtr, (PyPtr,), o)
246+
s = @with_gil ccall((@pysym :PyObject_Repr), PyPtr, (PyPtr,), o)
244247
if (s == C_NULL)
245248
pyerr_clear()
246-
s = ccall((@pysym :PyObject_Str), PyPtr, (PyPtr,), o)
249+
s = @with_gil ccall((@pysym :PyObject_Str), PyPtr, (PyPtr,), o)
247250
if (s == C_NULL)
248251
pyerr_clear()
249252
return string(PyPtr(o))
@@ -281,7 +284,7 @@ function hash(o::PyObject, salt::UInt)
281284
# since on 64-bit Windows the Python 2.x hash is only 32 bits
282285
hash(unsafe_pyjlwrap_to_objref(o), salt)
283286
else
284-
h = ccall((@pysym :PyObject_Hash), Py_hash_t, (PyPtr,), o)
287+
h = @with_gil ccall((@pysym :PyObject_Hash), Py_hash_t, (PyPtr,), o)
285288
if h == -1 # error
286289
pyerr_clear()
287290
return hash(PyPtr(o), salt)
@@ -297,7 +300,7 @@ end
297300

298301
function _getproperty(o::PyObject, s::Union{AbstractString,Symbol})
299302
ispynull(o) && throw(ArgumentError("ref of NULL PyObject"))
300-
p = ccall((@pysym :PyObject_GetAttrString), PyPtr, (PyPtr, Cstring), o, s)
303+
p = @with_gil ccall((@pysym :PyObject_GetAttrString), PyPtr, (PyPtr, Cstring), o, s)
301304
if p == C_NULL && pyerr_occurred()
302305
e = PyError("PyObject_GetAttrString")
303306
if PyPtr(e.T) != @pyglobalobjptr(:PyExc_AttributeError)
@@ -334,7 +337,7 @@ setproperty!(o::PyObject, s::AbstractString, v) = _setproperty!(o,s,v)
334337

335338
function _setproperty!(o::PyObject, s::Union{Symbol,AbstractString}, v)
336339
ispynull(o) && throw(ArgumentError("assign of NULL PyObject"))
337-
p = ccall((@pysym :PyObject_SetAttrString), Cint, (PyPtr, Cstring, PyPtr), o, s, PyObject(v))
340+
p = @with_gil ccall((@pysym :PyObject_SetAttrString), Cint, (PyPtr, Cstring, PyPtr), o, s, PyObject(v))
338341
if p == -1 && pyerr_occurred()
339342
e = PyError("PyObject_SetAttrString")
340343
if PyPtr(e.T) != @pyglobalobjptr(:PyExc_AttributeError)
@@ -370,8 +373,8 @@ function pyhasproperty(o::PyObject, s::Union{Symbol,AbstractString})
370373
if ispynull(o)
371374
throw(ArgumentError("hasproperty of NULL PyObject"))
372375
end
373-
return 1 == ccall((@pysym :PyObject_HasAttrString), Cint,
374-
(PyPtr, Cstring), o, s)
376+
return 1 == @with_gil(ccall((@pysym :PyObject_HasAttrString), Cint,
377+
(PyPtr, Cstring), o, s))
375378
end
376379
hasproperty(o::PyObject, s::Symbol) = pyhasproperty(o, s)
377380
hasproperty(o::PyObject, s::AbstractString) = pyhasproperty(o, s)
@@ -480,7 +483,7 @@ end
480483
function _pyimport(name::AbstractString)
481484
cookie = ActivatePyActCtx()
482485
try
483-
return PyObject(ccall((@pysym :PyImport_ImportModule), PyPtr, (Cstring,), name))
486+
return @with_gil PyObject(ccall((@pysym :PyImport_ImportModule), PyPtr, (Cstring,), name))
484487
finally
485488
DeactivatePyActCtx(cookie)
486489
end
@@ -685,7 +688,7 @@ string otherwise.
685688
"""
686689
function anaconda_conda()
687690
# Anaconda Python seems to always include "Anaconda" in the version string.
688-
if conda || !occursin("conda", unsafe_string(ccall(@pysym(:Py_GetVersion), Ptr{UInt8}, ())))
691+
if conda || !occursin("conda", unsafe_string(@with_gil ccall(@pysym(:Py_GetVersion), Ptr{UInt8}, ())))
689692
return ""
690693
end
691694
aconda = joinpath(dirname(pyprogramname), "conda")
@@ -770,7 +773,7 @@ include("pyfncall.jl")
770773
# for now we can define "get".
771774

772775
function get(o::PyObject, returntype::TypeTuple, k, default)
773-
r = ccall((@pysym :PyObject_GetItem), PyPtr, (PyPtr,PyPtr), o,PyObject(k))
776+
r = @with_gil ccall((@pysym :PyObject_GetItem), PyPtr, (PyPtr,PyPtr), o,PyObject(k))
774777
if r == C_NULL
775778
pyerr_clear()
776779
default
@@ -781,13 +784,13 @@ end
781784

782785
get(o::PyObject, returntype::TypeTuple, k) =
783786
convert(returntype, PyObject(@pycheckn ccall((@pysym :PyObject_GetItem),
784-
PyPtr, (PyPtr,PyPtr), o, PyObject(k))))
787+
PyPtr, (PyPtr,PyPtr), o, PyObject(k))))
785788

786789
get(o::PyObject, k, default) = get(o, PyAny, k, default)
787790
get(o::PyObject, k) = get(o, PyAny, k)
788791

789792
function delete!(o::PyObject, k)
790-
e = ccall((@pysym :PyObject_DelItem), Cint, (PyPtr, PyPtr), o, PyObject(k))
793+
e = @with_gil ccall((@pysym :PyObject_DelItem), Cint, (PyPtr, PyPtr), o, PyObject(k))
791794
if e == -1
792795
pyerr_clear() # delete! ignores errors in Julia
793796
end
@@ -959,7 +962,7 @@ function get_real_method(obj, name)
959962
return nothing
960963
end
961964

962-
ccall((@pysym :PyCallable_Check), Cint, (PyPtr,), m) == 1 && return m
965+
@with_gil(ccall((@pysym :PyCallable_Check), Cint, (PyPtr,), m)) == 1 && return m
963966

964967
return nothing
965968
end

‎src/conversions.jl‎

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
PyPtr, (Int,), i))
1313
else
1414
PyObject(i::Unsigned) = PyObject(@pycheckn ccall(@pysym(:PyLong_FromUnsignedLongLong),
15-
PyPtr, (Culonglong,), i))
15+
PyPtr, (Culonglong,), i))
1616
PyObject(i::Integer) = PyObject(@pycheckn ccall(@pysym(:PyLong_FromLongLong),
1717
PyPtr, (Clonglong,), i))
1818
end
@@ -80,8 +80,8 @@ function PyObject(s::AbstractString)
8080
sb = String(s)
8181
if pyunicode_literals || !isascii(sb)
8282
PyObject(@pycheckn ccall(@pysym(PyUnicode_DecodeUTF8),
83-
PyPtr, (Ptr{UInt8}, Int, Ptr{UInt8}),
84-
sb, sizeof(sb), C_NULL))
83+
PyPtr, (Ptr{UInt8}, Int, Ptr{UInt8}),
84+
sb, sizeof(sb), C_NULL))
8585
else
8686
pybytes(sb)
8787
end
@@ -92,7 +92,7 @@ const _ps_len = Int[0]
9292
function convert(::Type{T}, po::PyObject) where T<:AbstractString
9393
if pyisinstance(po, @pyglobalobj :PyUnicode_Type)
9494
convert(T, PyObject(@pycheckn ccall(@pysym(PyUnicode_AsUTF8String),
95-
PyPtr, (PyPtr,), po)))
95+
PyPtr, (PyPtr,), po)))
9696
else
9797
@pycheckz ccall(@pysym(PyString_AsStringAndSize),
9898
Cint, (PyPtr, Ptr{Ptr{UInt8}}, Ptr{Int}),
@@ -223,8 +223,8 @@ function convert(tt::Type{T}, o::PyObject) where T<:Tuple
223223
end
224224
ntuple((i ->
225225
convert(tuptype(T, isva, i),
226-
PyObject(ccall((@pysym :PySequence_GetItem), PyPtr,
227-
(PyPtr, Int), o, i-1)))),
226+
PyObject(@with_gil ccall((@pysym :PySequence_GetItem), PyPtr,
227+
(PyPtr, Int), o, i-1)))),
228228
len)
229229
end
230230

@@ -346,7 +346,7 @@ function py2array(T, A::Array{TA,N}, o::PyObject,
346346
error("dimension mismatch in py2array")
347347
end
348348
s = stride(A, dim)
349-
for j = 0:len-1
349+
@with_gil for j = 0:len-1
350350
A[i+j*s] = convert(T, PyObject(ccall((@pysym :PySequence_GetItem),
351351
PyPtr, (PyPtr, Int), o, j)))
352352
end
@@ -357,7 +357,7 @@ function py2array(T, A::Array{TA,N}, o::PyObject,
357357
error("dimension mismatch in py2array")
358358
end
359359
s = stride(A, dim)
360-
for j = 0:len-1
360+
@with_gil for j = 0:len-1
361361
py2array(T, A, PyObject(ccall((@pysym :PySequence_GetItem),
362362
PyPtr, (PyPtr, Int), o, j)),
363363
dim+1, i+j*s)
@@ -376,13 +376,13 @@ function pyarray_dims(o::PyObject, forcelist=true)
376376
if len == 0
377377
return (0,)
378378
end
379-
dims0 = pyarray_dims(PyObject(ccall((@pysym :PySequence_GetItem),
380-
PyPtr, (PyPtr, Int), o, 0)),
379+
dims0 = pyarray_dims(PyObject(@with_gil(ccall((@pysym :PySequence_GetItem),
380+
PyPtr, (PyPtr, Int), o, 0))),
381381
false)
382382
if isempty(dims0) # not a nested sequence
383383
return (len,)
384384
end
385-
for j = 1:len-1
385+
@with_gil for j = 1:len-1
386386
dims = pyarray_dims(PyObject(ccall((@pysym :PySequence_GetItem),
387387
PyPtr, (PyPtr, Int), o, j)),
388388
false)
@@ -408,7 +408,7 @@ function py2array(T, o::PyObject)
408408
end
409409

410410
function py2vector(T, o::PyObject)
411-
len = ccall((@pysym :PySequence_Size), Int, (PyPtr,), o)
411+
len = @with_gil ccall((@pysym :PySequence_Size), Int, (PyPtr,), o)
412412
if len < 0 || # not a sequence
413413
len+1 < 0 # object pretending to be a sequence of infinite length
414414
pyerr_clear()
@@ -435,7 +435,7 @@ include("numpy.jl")
435435
function is_mapping_object(o::PyObject)
436436
pyisinstance(o, @pyglobalobj :PyDict_Type) ||
437437
(pyquery((@pyglobal :PyMapping_Check), o) &&
438-
ccall((@pysym :PyObject_HasAttrString), Cint, (PyPtr,Ptr{UInt8}), o, "items") == 1)
438+
@with_gil(ccall((@pysym :PyObject_HasAttrString), Cint, (PyPtr,Ptr{UInt8}), o, "items")) == 1)
439439
end
440440

441441
"""
@@ -472,13 +472,13 @@ convert(::Type{PyDict}, o::PyObject) = PyDict(o)
472472
convert(::Type{PyDict{K,V}}, o::PyObject) where {K,V} = PyDict{K,V}(o)
473473
unsafe_convert(::Type{PyPtr}, d::PyDict) = PyPtr(d.o)
474474

475-
haskey(d::PyDict{K,V,true}, key) where {K,V} = 1 == ccall(@pysym(:PyDict_Contains), Cint, (PyPtr, PyPtr), d, PyObject(key))
475+
haskey(d::PyDict{K,V,true}, key) where {K,V} = 1 == @with_gil(ccall(@pysym(:PyDict_Contains), Cint, (PyPtr, PyPtr), d, PyObject(key)))
476476
keys(::Type{T}, d::PyDict{K,V,true}) where {T,K,V} = convert(Vector{T}, PyObject(@pycheckn ccall((@pysym :PyDict_Keys), PyPtr, (PyPtr,), d)))
477477
values(::Type{T}, d::PyDict{K,V,true}) where {T,K,V} = convert(Vector{T}, PyObject(@pycheckn ccall((@pysym :PyDict_Values), PyPtr, (PyPtr,), d)))
478478

479479
keys(::Type{T}, d::PyDict{K,V,false}) where {T,K,V} = convert(Vector{T}, pycall(d.o["keys"], PyObject))
480480
values(::Type{T}, d::PyDict{K,V,false}) where {T,K,V} = convert(Vector{T}, pycall(d.o["values"], PyObject))
481-
haskey(d::PyDict{K,V,false}, key) where {K,V} = 1 == ccall(@pysym(:PyMapping_HasKey), Cint, (PyPtr, PyPtr), d, PyObject(key))
481+
haskey(d::PyDict{K,V,false}, key) where {K,V} = 1 == @with_gil(ccall(@pysym(:PyMapping_HasKey), Cint, (PyPtr, PyPtr), d, PyObject(key)))
482482

483483
similar(d::PyDict{K,V}) where {K,V} = Dict{pyany_toany(K),pyany_toany(V)}()
484484
eltype(::Type{PyDict{K,V}}) where {K,V} = Pair{pyany_toany(K),pyany_toany(V)}
@@ -515,12 +515,12 @@ function pop!(d::PyDict, k, default)
515515
end
516516

517517
function delete!(d::PyDict{K,V,true}, k) where {K,V}
518-
e = ccall(@pysym(:PyDict_DelItem), Cint, (PyPtr, PyPtr), d, PyObject(k))
518+
e = @with_gil ccall(@pysym(:PyDict_DelItem), Cint, (PyPtr, PyPtr), d, PyObject(k))
519519
e == -1 && pyerr_clear() # delete! ignores errors in Julia
520520
return d
521521
end
522522
function delete!(d::PyDict{K,V,false}, k) where {K,V}
523-
e = ccall(@pysym(:PyObject_DelItem), Cint, (PyPtr, PyPtr), d, PyObject(k))
523+
e = @with_gil ccall(@pysym(:PyObject_DelItem), Cint, (PyPtr, PyPtr), d, PyObject(k))
524524
e == -1 && pyerr_clear() # delete! ignores errors in Julia
525525
return d
526526
end
@@ -553,9 +553,9 @@ end
553553

554554
function Base.iterate(d::PyDict{K,V,true}, itr=PyDict_Iterator(Ref{PyPtr}(), Ref{PyPtr}(), Ref(0), 0, length(d))) where {K,V}
555555
itr.i >= itr.len && return nothing
556-
if 0 == ccall((@pysym :PyDict_Next), Cint,
557-
(PyPtr, Ref{Int}, Ref{PyPtr}, Ref{PyPtr}),
558-
d, itr.pa, itr.ka, itr.va)
556+
if 0 == @with_gil(ccall((@pysym :PyDict_Next), Cint,
557+
(PyPtr, Ref{Int}, Ref{PyPtr}, Ref{PyPtr}),
558+
d, itr.pa, itr.ka, itr.va))
559559
error("unexpected end of PyDict_Next")
560560
end
561561
ko = pyincref(itr.ka[]) # PyDict_Next returns
@@ -751,7 +751,7 @@ function pysequence_query(o::PyObject)
751751
# problems
752752
if pyisinstance(o, @pyglobalobj :PyTuple_Type)
753753
len = length(o)
754-
return typetuple(pytype_query(PyObject(ccall((@pysym :PySequence_GetItem), PyPtr, (PyPtr,Int), o,i-1)), PyAny) for i = 1:len)
754+
return typetuple(pytype_query(PyObject(@with_gil(ccall((@pysym :PySequence_GetItem), PyPtr, (PyPtr,Int), o,i-1))), PyAny) for i = 1:len)
755755
elseif pyisinstance(o, pyxrange[])
756756
return AbstractRange
757757
elseif ispybytearray(o)

‎src/exception.jl‎

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -51,10 +51,12 @@ end
5151

5252
# like pyerror(msg) but type-stable: always returns PyError
5353
function PyError(msg::AbstractString)
54+
# Note: callers are expected to already hold the GIL via @with_gil
5455
ptype, pvalue, ptraceback = Ref{PyPtr}(), Ref{PyPtr}(), Ref{PyPtr}()
55-
# equivalent of passing C pointers &exc[1], &exc[2], &exc[3]:
56-
ccall((@pysym :PyErr_Fetch), Cvoid, (Ref{PyPtr},Ref{PyPtr},Ref{PyPtr}), ptype, pvalue, ptraceback)
57-
ccall((@pysym :PyErr_NormalizeException), Cvoid, (Ref{PyPtr},Ref{PyPtr},Ref{PyPtr}), ptype, pvalue, ptraceback)
56+
@with_gil begin
57+
ccall((@pysym :PyErr_Fetch), Cvoid, (Ref{PyPtr},Ref{PyPtr},Ref{PyPtr}), ptype, pvalue, ptraceback)
58+
ccall((@pysym :PyErr_NormalizeException), Cvoid, (Ref{PyPtr},Ref{PyPtr},Ref{PyPtr}), ptype, pvalue, ptraceback)
59+
end
5860
return PyError(msg, PyObject(ptype[]), PyObject(pvalue[]), PyObject(ptraceback[]))
5961
end
6062

@@ -66,10 +68,10 @@ PyError(msg::AbstractString, e::PyError) =
6668
# Conversion of Python exceptions into Julia exceptions
6769

6870
# whether a Python exception has occurred
69-
pyerr_occurred() = ccall((@pysym :PyErr_Occurred), PyPtr, ()) != C_NULL
71+
pyerr_occurred() = @with_gil ccall((@pysym :PyErr_Occurred), PyPtr, ()) != C_NULL
7072

7173
# call to discard Python exceptions
72-
pyerr_clear() = ccall((@pysym :PyErr_Clear), Cvoid, ())
74+
pyerr_clear() = @with_gil ccall((@pysym :PyErr_Clear), Cvoid, ())
7375

7476
function pyerr_check(msg::AbstractString, val::Any)
7577
pyerr_occurred() && throw(pyerror(msg))
@@ -99,13 +101,13 @@ end
99101

100102
# Macros for common pyerr_check("Foo", ccall((@pysym :Foo), ...)) pattern.
101103
macro pycheck(ex)
102-
:(pyerr_check($(string(callsym(ex))), $(esc(ex))))
104+
:(pyerr_check($(string(callsym(ex))), @with_gil($(esc(ex)))))
103105
end
104106

105107
# Macros to check that ccall((@pysym :Foo), ...) returns value != bad
106108
macro pycheckv(ex, bad)
107109
quote
108-
val = $(esc(ex))
110+
val = @with_gil($(esc(ex)))
109111
if val == $(esc(bad))
110112
_handle_error($(string(callsym(ex))))
111113
end
@@ -223,14 +225,14 @@ function pyraise(e, bt = nothing)
223225
eT = typeof(e)
224226
pyeT = haskey(pyexc, eT) ? pyexc[eT] : pyexc[Exception]
225227
err = PyJlError(e, bt)
226-
ccall((@pysym :PyErr_SetObject), Cvoid, (PyPtr, PyPtr),
227-
pyeT, PyObject(err))
228+
@with_gil ccall((@pysym :PyErr_SetObject), Cvoid, (PyPtr, PyPtr),
229+
pyeT, PyObject(err))
228230
end
229231

230232
# Second argument allows for backtraces passed to `pyraise` to be ignored.
231233
function pyraise(e::PyError, ::Vector = [])
232-
ccall((@pysym :PyErr_Restore), Cvoid, (PyPtr, PyPtr, PyPtr),
233-
e.T, e.val, e.traceback)
234+
@with_gil ccall((@pysym :PyErr_Restore), Cvoid, (PyPtr, PyPtr, PyPtr),
235+
e.T, e.val, e.traceback)
234236
# refs were stolen
235237
setfield!(e.T, :o, PyPtr_NULL)
236238
setfield!(e.val, :o, PyPtr_NULL)

‎src/gil.jl‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
function gil_ensure()
2+
return ccall((@pysym :PyGILState_Ensure), Cint, ())
3+
end
4+
5+
function gil_release(state::Cint)
6+
ccall((@pysym :PyGILState_Release), Cvoid, (Cint,), state)
7+
end
8+
9+
"""
10+
@with_gil expr
11+
12+
Execute `expr` while holding the Python GIL. Safe to nest (re-entrant).
13+
"""
14+
macro with_gil(expr)
15+
quote
16+
local _gilstate = gil_ensure()
17+
try
18+
$(esc(expr))
19+
finally
20+
gil_release(_gilstate)
21+
end
22+
end
23+
end

0 commit comments

Comments
 (0)