@@ -82,7 +82,13 @@ def copy_directory_data_command(self, local_dir_path: str, dest_path: str) -> Li
8282 full_command = ["sh" , "-c" , cmd ]
8383 return full_command
8484
85- def move_data (self , sleep : float = 10 , timeout : int = 600 , poll_interval : int = 5 , unknowns_grace_period : int = 60 ) -> None :
85+ def move_data (
86+ self ,
87+ sleep : float = 10 ,
88+ timeout : int = 600 ,
89+ poll_interval : int = 5 ,
90+ unknowns_grace_period : int = 60 ,
91+ ) -> None :
8692 """
8793 Moves job directory into remote storage and deletes the workload after completion.
8894 """
@@ -127,26 +133,34 @@ def move_data(self, sleep: float = 10, timeout: int = 600, poll_interval: int =
127133 while True :
128134 if time .time () - start_time > timeout :
129135 raise TimeoutError (f"Job { job_id } did not complete within { timeout } seconds." )
130-
136+
131137 current_job = client .job .get (job_id )
132138 current_job_status = current_job .status .state
133-
134139 if count > 0 :
135- if current_job_status == LeptonJobState .Completed or current_job_status == LeptonJobState .Failed :
140+ if (
141+ current_job_status == LeptonJobState .Completed
142+ or current_job_status == LeptonJobState .Failed
143+ ):
136144 break
137145 elif current_job_status == LeptonJobState .Unknown :
138146 if unknown_start_time is None :
139147 unknown_start_time = time .time ()
140- logging .warning (f"Job { job_id } entered Unknown state, giving it { unknowns_grace_period } seconds to recover..." )
148+ logging .warning (
149+ f"Job { job_id } entered Unknown state, giving it { unknowns_grace_period } seconds to recover..."
150+ )
141151
142152 elif time .time () - unknown_start_time > unknowns_grace_period :
143- logging .error (f"Job { job_id } has been in Unknown state for more than { unknowns_grace_period } seconds" )
153+ logging .error (
154+ f"Job { job_id } has been in Unknown state for more than { unknowns_grace_period } seconds"
155+ )
144156 break
145157 else :
146158 if unknown_start_time is not None :
147- logging .info (f"Job { job_id } recovered from Unknown state to { current_job_status } " )
159+ logging .info (
160+ f"Job { job_id } recovered from Unknown state to { current_job_status } "
161+ )
148162 unknown_start_time = None
149-
163+
150164 count += 1
151165 time .sleep (poll_interval )
152166
0 commit comments