Merge pull request #472 from FederatedAI/feature-1.11.2-deepspeed_log

fix

Author: zhihuiwan

python/fate_flow/controller/task_controller.py

@@ -207,12 +207,15 @@ def kill_task(cls, task: Task):
         kill_status = False
         try:
             # kill task executor
-            backend_engine = build_engine(
-                task.f_engine_conf.get("computing_engine"),
-                task.f_is_deepspeed
-                )
-            if backend_engine:
-                backend_engine.kill(task)
+            try:
+                backend_engine = build_engine(
+                    task.f_engine_conf.get("computing_engine"),
+                    task.f_is_deepspeed
+                    )
+                if backend_engine:
+                    backend_engine.kill(task)
+            except Exception as e:
+                schedule_logger(task.f_job_id).exception(e)
             WorkerManager.kill_task_all_workers(task)
         except Exception as e:
             schedule_logger(task.f_job_id).exception(e)

python/fate_flow/detection/detector.py

@@ -69,7 +69,7 @@ def detect_running_task(cls):
                     process_exist = build_engine(task.f_engine_conf.get("computing_engine")).is_alive(task)
                     if not process_exist:
                         msg = f"task {task.f_task_id} {task.f_task_version} on {task.f_role} {task.f_party_id}"
-                        detect_logger(job_id=task.f_job_id).info(f"{msg} with {task.f_party_status} process {task.f_run_pid} does not exist")
+                        detect_logger(job_id=task.f_job_id).error(f"{msg} with {task.f_party_status} process {task.f_run_pid} does not exist")
                         time.sleep(3)
                         _tasks = JobSaver.query_task(task_id=task.f_task_id, task_version=task.f_task_version, role=task.f_role, party_id=task.f_party_id)
                         if _tasks: