Skip to content

Commit 6394756

Browse files
committed
extract_utils: do not use ProcessPoolExecutor() for IO-bound tasks
Change-Id: I5c6c2f94e0557442b686ddf418559e44787f8177
1 parent 9ef3e93 commit 6394756

1 file changed

Lines changed: 12 additions & 21 deletions

File tree

extract_utils/extract.py

Lines changed: 12 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
import re
1010
import shutil
1111
import tarfile
12-
from concurrent.futures import ProcessPoolExecutor
1312
from os import path
1413
from tarfile import is_tarfile
1514
from typing import Callable, Dict, Iterable, List, Optional, Set, Union
@@ -399,24 +398,19 @@ def extract_ext4(file_path: str, output_path: str):
399398
# TODO: check for symlinks like the old code?
400399

401400

402-
def unzip_file(source: str, file_path: str, output_file_path: str):
403-
with ZipFile(source) as zip_file:
404-
with zip_file.open(file_path) as z:
405-
with open(output_file_path, 'wb') as f:
406-
shutil.copyfileobj(z, f)
407-
408-
409401
def extract_zip(source: str, dump_dir: str):
410402
with ZipFile(source) as zip_file:
411-
file_paths = zip_file.namelist()
403+
for info in zip_file.infolist():
404+
if info.is_dir():
405+
continue
412406

413-
with ProcessPoolExecutor() as exe:
414-
for file_path in file_paths:
415-
output_file_path = path.join(dump_dir, file_path)
407+
output_file_path = path.join(dump_dir, info.filename)
416408
output_dir = path.dirname(output_file_path)
417409
os.makedirs(output_dir, exist_ok=True)
418410

419-
exe.submit(unzip_file, source, file_path, output_file_path)
411+
with zip_file.open(info) as z:
412+
with open(output_file_path, 'wb') as f:
413+
shutil.copyfileobj(z, f)
420414

421415

422416
def extract_tar(source: str, dump_dir: str):
@@ -554,14 +548,11 @@ def extract_all_partitions(dump_dir: str, ctx: ExtractCtx):
554548
partitions = normal_partitions + firmware_partitions
555549

556550
while partitions:
557-
with ProcessPoolExecutor() as exe:
558-
for partition in partitions:
559-
if partition in firmware_partitions:
560-
fn = extract_firmware_partition
561-
else:
562-
fn = extract_partition
563-
564-
exe.submit(fn, partition, dump_dir)
551+
for partition in partitions:
552+
if partition in firmware_partitions:
553+
extract_firmware_partition(partition, dump_dir)
554+
else:
555+
extract_partition(partition, dump_dir)
565556

566557
found_partitions = find_partitions(dump_dir, ctx)
567558
partitions = find_alternate_partitions(partitions, found_partitions)

0 commit comments

Comments
 (0)