Skip to content

Commit 86a591b

Browse files
committed
we shouldn't try to act on equality delete files
1 parent 0cd0600 commit 86a591b

1 file changed

Lines changed: 6 additions & 1 deletion

File tree

pyiceberg/io/pyarrow.py

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1694,7 +1694,12 @@ def _task_to_record_batches(
16941694

16951695
def _read_all_delete_files(io: FileIO, tasks: Iterable[FileScanTask]) -> dict[str, list[ChunkedArray]]:
16961696
deletes_per_file: dict[str, list[ChunkedArray]] = {}
1697-
unique_deletes = set(itertools.chain.from_iterable([task.delete_files for task in tasks]))
1697+
unique_deletes = {
1698+
delete_file
1699+
for task in tasks
1700+
for delete_file in task.delete_files
1701+
if delete_file.content == DataFileContent.POSITION_DELETES
1702+
}
16981703
if len(unique_deletes) > 0:
16991704
executor = ExecutorFactory.get_or_create()
17001705
deletes_per_files: Iterator[dict[str, ChunkedArray]] = executor.map(

0 commit comments

Comments
 (0)