Skip to content

Commit 9320fbf

Browse files
committed
perf(decompress): Batch column drops in decompress_columns()
Individual df.drop() per column caused O(N_cols × N_rows) pandas reindex on each call. For 7 columns × 82M rows, this was 12s out of 17s total (72% of runtime). Fix: Collect all compressed columns, drop once after loop. Expected: 17s → ~5s for 7-column decompression on 82M rows. Profile-driven: cProfile showed df.drop → _slice_take_blocks_ax0 → numpy.take consuming 12.1s across 7 calls.
1 parent 650a97e commit 9320fbf

1 file changed

Lines changed: 10 additions & 2 deletions

File tree

UTILS/dfextensions/AliasDataFrame/AliasDataFrame.py

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6692,6 +6692,10 @@ def decompress_columns(self, columns=None, inplace=False, keep_compressed=True,
66926692
# Filter __meta__
66936693
columns = [c for c in columns if c != "__meta__"]
66946694

6695+
# Phase 13.7: Collect columns to drop, then batch-drop once at end
6696+
# (Avoids O(N_cols × N_rows) pandas reindex per drop)
6697+
cols_to_drop = []
6698+
66956699
for col in columns:
66966700
if col not in self.compression_info:
66976701
raise ValueError(
@@ -6741,9 +6745,9 @@ def decompress_columns(self, columns=None, inplace=False, keep_compressed=True,
67416745
if col in self.aliases:
67426746
del self._schema["columns"][col]
67436747

6744-
# Step 4: Handle compressed column
6748+
# Step 4: Collect compressed column for batch drop
67456749
if not keep_compressed:
6746-
self.df.drop(columns=[compressed_col], inplace=True)
6750+
cols_to_drop.append(compressed_col)
67476751

67486752
# Step 5: Update state
67496753
if keep_schema:
@@ -6753,6 +6757,10 @@ def decompress_columns(self, columns=None, inplace=False, keep_compressed=True,
67536757
# Remove all compression metadata
67546758
del self.compression_info[col]
67556759

6760+
# Batch drop all compressed columns at once (single reindex)
6761+
if cols_to_drop:
6762+
self.df.drop(columns=cols_to_drop, inplace=True)
6763+
67566764
return self
67576765

67586766
def get_compression_info(self, column=None):

0 commit comments

Comments
 (0)