Skip to content

Commit fb003cc

Browse files
committed
we shouldn't try to act on equality delete files
1 parent 69dee3a commit fb003cc

1 file changed

Lines changed: 6 additions & 1 deletion

File tree

pyiceberg/io/pyarrow.py

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1707,7 +1707,12 @@ def _task_to_record_batches(
17071707

17081708
def _read_all_delete_files(io: FileIO, tasks: Iterable[FileScanTask]) -> dict[str, list[ChunkedArray]]:
17091709
deletes_per_file: dict[str, list[ChunkedArray]] = {}
1710-
unique_deletes = set(itertools.chain.from_iterable([task.delete_files for task in tasks]))
1710+
unique_deletes = {
1711+
delete_file
1712+
for task in tasks
1713+
for delete_file in task.delete_files
1714+
if delete_file.content == DataFileContent.POSITION_DELETES
1715+
}
17111716
if len(unique_deletes) > 0:
17121717
executor = ExecutorFactory.get_or_create()
17131718
deletes_per_files: Iterator[dict[str, ChunkedArray]] = executor.map(

0 commit comments

Comments
 (0)