fix: Implement responsive scheduler shutdown using threading Event

- Added threading.Event for immediate scheduler shutdown
- Replaced blocking sleep with event.wait() for responsive termination
- Reduced thread join timeout from 30s to 5s since event enables quick shutdown
- Fixed E2E test timeout issues by making scheduler stop immediately
- Scheduler now stops in <1s instead of hanging for check_interval_minutes

This resolves the E2E test failures where scheduler.stop_scheduler()
was timing out due to long sleep() calls in the scheduler loop.

Author: RuiFSP

src/automation/retraining_scheduler.py

@@ -128,6 +128,7 @@ def __init__(
         self.last_retraining_time: datetime | None = None
         self.retraining_in_progress = False
         self.retraining_lock = threading.Lock()
+        self._shutdown_event = threading.Event()  # For responsive shutdown
 
         # Metrics tracking
         self.prediction_count_since_retraining = 0
@@ -147,6 +148,7 @@ def start_scheduler(self) -> None:
             return
 
         self.is_running = True
+        self._shutdown_event.clear()  # Reset the event
         self.scheduler_thread = threading.Thread(
             target=self._scheduler_loop, daemon=True, name="RetrainingScheduler"
         )
@@ -164,9 +166,10 @@ def stop_scheduler(self) -> None:
             return
 
         self.is_running = False
+        self._shutdown_event.set()  # Signal the thread to wake up
 
         if self.scheduler_thread and self.scheduler_thread.is_alive():
-            self.scheduler_thread.join(timeout=30)
+            self.scheduler_thread.join(timeout=5)  # Reduced timeout since we use event
 
         logger.info("Automated retraining scheduler stopped")
 
@@ -179,13 +182,22 @@ def _scheduler_loop(self) -> None:
                 self._check_retraining_triggers()
                 self.last_check_time = datetime.now()
 
-                # Sleep for the configured interval
-                time.sleep(self.config.check_interval_minutes * 60)
+                # Wait for the configured interval or until shutdown is signaled
+                sleep_time = self.config.check_interval_minutes * 60
+                if not self._shutdown_event.wait(timeout=sleep_time):
+                    # Timeout occurred (normal case), continue to next iteration
+                    continue
+                else:
+                    # Event was set (shutdown requested), exit loop
+                    break
 
             except Exception as e:
                 logger.error(f"Error in scheduler loop: {str(e)}")
                 # Continue running even if there's an error
-                time.sleep(60)  # Wait 1 minute before retrying
+                if not self._shutdown_event.wait(timeout=60):  # Wait 1 minute before retrying
+                    continue
+                else:
+                    break
 
     def _check_retraining_triggers(self) -> None:
         """Check all retraining triggers and initiate retraining if needed."""