Skip to content
Closed
Show file tree
Hide file tree
Changes from 14 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 32 additions & 1 deletion Lib/test/test_json/test_recursion.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
import weakref
import sys
Comment on lines +1 to +2
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
import weakref
import sys
import sys
import weakref

from test import support
from test.test_json import PyTest, CTest


Comment thread
picnixz marked this conversation as resolved.
class JSONTestObject:
pass

Comment thread
okiemute04 marked this conversation as resolved.

Comment thread
okiemute04 marked this conversation as resolved.
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why again has this been changed?

class TestRecursion:
def test_listrecursion(self):
x = []
Expand Down Expand Up @@ -115,6 +116,36 @@ def default(self, o):
with support.infinite_recursion(1000):
EndlessJSONEncoder(check_circular=False).encode(5j)

@support.skip_if_unlimited_stack_size
@support.skip_emscripten_stack_overflow()
@support.skip_wasi_stack_overflow()
def test_memory_leak_on_recursion_error(self):
# Test that no memory leak occurs when a RecursionError is raised.
class LeakTestObj:
pass

weak_refs = []
def default(obj):
if isinstance(obj, LeakTestObj):
new_obj = LeakTestObj()
weak_refs.append(weakref.ref(new_obj))
return [new_obj]
raise TypeError

depth = min(500, sys.getrecursionlimit() - 10)
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You should move this code inside infinite_recursion() context, since infinite_recursion() changes the recursion limit.

obj = LeakTestObj()
for _ in range(depth):
obj = [obj]

with support.infinite_recursion():
with self.assertRaises(RecursionError):
self.dumps(obj, default=default)

support.gc_collect()
self.assertTrue(weak_refs, "No objects were created to track")
for i, ref in enumerate(weak_refs):
self.assertIsNone(ref(), f"object {i} still alive")
Comment thread
picnixz marked this conversation as resolved.
Comment thread
okiemute04 marked this conversation as resolved.


class TestPyRecursion(TestRecursion, PyTest): pass
class TestCRecursion(TestRecursion, CTest): pass
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
Fix a memory leak in the :mod:`json` module when a RecursionError occurs
during encoding. Previously, objects created in the ``default()`` function
while recursively encoding JSON could remain alive due to being tracked
indefinitely in the encoder's internal circular-reference dictionary. This
change ensures such objects are properly freed after the exception.
Comment on lines +1 to +5
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
Fix a memory leak in the :mod:`json` module when a RecursionError occurs
during encoding. Previously, objects created in the ``default()`` function
while recursively encoding JSON could remain alive due to being tracked
indefinitely in the encoder's internal circular-reference dictionary. This
change ensures such objects are properly freed after the exception.
:mod:`json`: Fix a memory leak when encoding recursive structures fails.

25 changes: 20 additions & 5 deletions Modules/_json.c
Original file line number Diff line number Diff line change
Expand Up @@ -1632,8 +1632,17 @@ encoder_listencode_obj(PyEncoderObject *s, PyUnicodeWriter *writer,
}

if (_Py_EnterRecursiveCall(" while encoding a JSON object")) {
if (ident != NULL) {
int del_rv = PyDict_DelItem(s->markers, ident);
Py_DECREF(ident);
if (del_rv < 0) {
Py_DECREF(newobj);
return -1;
}

}
Comment on lines +1635 to +1637
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
if (ident != NULL) {
int del_rv = PyDict_DelItem(s->markers, ident);
Py_DECREF(ident);
if (del_rv < 0) {
Py_DECREF(newobj);
return -1;
}
}
if (ident != NULL) {
(void)PyDict_DelItem(s->markers, ident);
}

Actually since we are anyway falling through the return -1 case we can simplify this as follows. Then keep the XDECREF for indent. However explicitly suppress the error code of DelItem.


Comment thread
okiemute04 marked this conversation as resolved.
Outdated
Py_DECREF(newobj);
Py_XDECREF(ident);
return -1;
}
rv = encoder_listencode_obj(s, writer, newobj, indent_level, indent_cache);
Expand All @@ -1642,15 +1651,21 @@ encoder_listencode_obj(PyEncoderObject *s, PyUnicodeWriter *writer,
Py_DECREF(newobj);
if (rv) {
_PyErr_FormatNote("when serializing %T object", obj);
Comment thread
okiemute04 marked this conversation as resolved.
Py_XDECREF(ident);
if (ident != NULL) {
int del_rv = PyDict_DelItem(s->markers, ident);
Py_DECREF(ident);
if (del_rv < 0) {
return -1;
}
}
Comment on lines +1648 to +1650
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This one can benefit from the same optimization as my other comment as we anyway return -1.

return -1;
}
if (ident != NULL) {
if (PyDict_DelItem(s->markers, ident)) {
Py_XDECREF(ident);
int del_rv = PyDict_DelItem(s->markers, ident);
Py_DECREF(ident);
if (del_rv < 0) {
return -1;
}
Py_XDECREF(ident);
Comment on lines -1649 to -1653
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This part does not need modifications..It was already properly doing its job.

}
return rv;
}
Expand Down
Loading