Skip to content

Commit 827767a

Browse files
authored
Improve engine's cleanup (#7813)
Fix #7812: This PR makes DeepSpeedEngine cleanup safe for partial initialization. This prevents destructor-time tracebacks by guarding access to unitialized attributes of DeepSpeed engine. Signed-off-by: Masahiro Tanaka <mtanaka@anyscale.com>
1 parent 156dcb2 commit 827767a

1 file changed

Lines changed: 12 additions & 6 deletions

File tree

deepspeed/runtime/engine.py

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -565,17 +565,23 @@ def broadcast_and_check(args, bcast_rank, bcast_group):
565565
with_kwargs=True)
566566

567567
def __del__(self):
568-
self.destroy()
568+
try:
569+
self.destroy()
570+
except Exception as exc:
571+
# Avoid destructor-time exceptions for partially initialized engines.
572+
logger.debug("DeepSpeedEngine.__del__ cleanup skipped: %s", exc, exc_info=True)
569573

570574
def destroy(self):
571-
if self.optimizer is not None and hasattr(self.optimizer, 'destroy'):
572-
self.optimizer.destroy()
575+
optimizer = getattr(self, "optimizer", None)
576+
if optimizer is not None and hasattr(optimizer, 'destroy'):
577+
optimizer.destroy()
573578
if self.is_deepcompile_active():
574579
get_deepcompile_handle().cleanup()
575580
debug_clear_module_and_param_names()
576581

577-
if self.checkpoint_engine is not None and self.checkpoint_engine.is_decoupled():
578-
self.checkpoint_engine.cleanup()
582+
checkpoint_engine = getattr(self, "checkpoint_engine", None)
583+
if checkpoint_engine is not None and checkpoint_engine.is_decoupled():
584+
checkpoint_engine.cleanup()
579585

580586
def _get_model_parameters(self):
581587
if self.autotuning_profile_model_info():
@@ -4376,7 +4382,7 @@ def is_deepcompile_enabled(self) -> bool:
43764382
return self._config.compile_config.deepcompile
43774383

43784384
def is_deepcompile_active(self) -> bool:
4379-
return self._deepcompile_active
4385+
return getattr(self, "_deepcompile_active", False)
43804386

43814387
@property
43824388
def is_compiled(self) -> bool:

0 commit comments

Comments
 (0)