fix: update test for arrow_c_stream_capsule_released to accept both destructor behaviors during decoupling from PyArrow

Author: kosiew

python/tests/test_dataframe.py

@@ -1738,10 +1738,14 @@ def test_arrow_c_stream_capsule_released(ctx):
 
     reader = pa.RecordBatchReader._import_from_c_capsule(capsule)
     reader.read_all()
-    # PyArrow may or may not clear the capsule's destructor; both behaviours are
-    # accepted temporarily while decoupling from PyArrow
+    # PyArrow may or may not clear the capsule's destructor; accept both
+    # behaviours temporarily while decoupling from PyArrow. Record the
+    # destructor value but don't assert a specific value here so the test
+    # works against pyarrow builds that clear the destructor and those that
+    # don't. The key checks are that import succeeds and cleanup does not
+    # lead to a double free (ensured by deleting the reader and forcing
+    # garbage collection below).
     _ = get_destructor(capsule)
-    # Pointer remains retrievable
     assert get_ptr(capsule, b"arrow_array_stream")
     pyerr_clear()
 

src/dataframe.rs

@@ -988,6 +988,9 @@ impl PyDataFrame {
         let reader: Box<dyn RecordBatchReader + Send> = Box::new(reader);
 
         let stream = FFI_ArrowArrayStream::new(reader);
+        // This custom destructor is defensive to avoid a double free. It is
+        // a temporary workaround until DataFusion's Python bindings decouple
+        // from PyArrow.
         // The returned capsule allows zero-copy hand-off to PyArrow. When
         // PyArrow imports the capsule it assumes ownership of the stream and
         // nulls out the capsule's internal pointer so the destructor does not