[fix] fix race condition for modifying torch num_threads

Co-authored-by: 0oshowero0<o0shower0o@outlook.com>



# message auto-generated for no-merge-commit merge:
!5 merge fix_race_condition into main

[fix] fix race condition for modifying torch num_threads

Created-by: hanzhenyu8
Commit-by: 0oshowero0
Merged-by: ascend-robot
Description: fix race condition for modifying torch num_threads

See merge request: Ascend/TransferQueue!5

Author: ascend-robot

transfer_queue/utils/perf_utils.py

@@ -28,7 +28,7 @@
     handler.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(name)s - %(message)s"))
     logger.addHandler(handler)
 
-TQ_PERF_LOG_FLUSH_INTERVAL = float(os.environ.get("TQ_PERF_LOG_FLUSH_INTERVAL", 10))  # in seconds
+TQ_PERF_LOG_FLUSH_INTERVAL = float(os.environ.get("TQ_PERF_LOG_FLUSH_INTERVAL", 300))  # in seconds
 
 
 class IntervalPerfMonitor:

transfer_queue/utils/utils.py

@@ -26,6 +26,8 @@
 logger = logging.getLogger(__name__)
 logger.setLevel(os.getenv("TQ_LOGGING_LEVEL", logging.WARNING))
 
+DEFAULT_TORCH_NUM_THREADS = torch.get_num_threads()
+
 # Ensure logger has a handler
 if not logger.hasHandlers():
     handler = logging.StreamHandler()
@@ -141,10 +143,10 @@ def limit_pytorch_auto_parallel_threads(target_num_threads: Optional[int] = None
         yield
     finally:
         # Restore the original number of threads
-        torch.set_num_threads(pytorch_current_num_threads)
+        torch.set_num_threads(DEFAULT_TORCH_NUM_THREADS)
         logger.debug(
             f"{info} (pid={pid}): torch.get_num_threads() is {torch.get_num_threads()}, "
-            f"restoring to {pytorch_current_num_threads}."
+            f"restoring to {DEFAULT_TORCH_NUM_THREADS}."
         )