diff --git a/setup.cfg b/setup.cfg index ae6f999aef9..f101c8fa7a0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -292,13 +292,10 @@ console_scripts = # TransformationSystem dirac-production-runjoblocal = DIRAC.TransformationSystem.scripts.dirac_production_runjoblocal:main dirac-transformation-add-files = DIRAC.TransformationSystem.scripts.dirac_transformation_add_files:main [admin] - dirac-transformation-archive = DIRAC.TransformationSystem.scripts.dirac_transformation_archive:main [admin] - dirac-transformation-clean = DIRAC.TransformationSystem.scripts.dirac_transformation_clean:main [admin] dirac-transformation-cli = DIRAC.TransformationSystem.scripts.dirac_transformation_cli:main [admin] dirac-transformation-get-files = DIRAC.TransformationSystem.scripts.dirac_transformation_get_files:main [admin] dirac-transformation-information = DIRAC.TransformationSystem.scripts.dirac_transformation_information:main [admin] dirac-transformation-recover-data = DIRAC.TransformationSystem.scripts.dirac_transformation_recover_data:main [admin] - dirac-transformation-remove-output = DIRAC.TransformationSystem.scripts.dirac_transformation_remove_output:main [admin] dirac-transformation-replication = DIRAC.TransformationSystem.scripts.dirac_transformation_replication:main [admin] dirac-transformation-verify-outputdata = DIRAC.TransformationSystem.scripts.dirac_transformation_verify_outputdata:main [admin] dirac-transformation-update-derived = DIRAC.TransformationSystem.scripts.dirac_transformation_update_derived:main [admin] diff --git a/src/DIRAC/TransformationSystem/Agent/TransformationCleaningAgent.py b/src/DIRAC/TransformationSystem/Agent/TransformationCleaningAgent.py index 2d9f815262e..fac7599b07c 100644 --- a/src/DIRAC/TransformationSystem/Agent/TransformationCleaningAgent.py +++ b/src/DIRAC/TransformationSystem/Agent/TransformationCleaningAgent.py @@ -32,9 +32,12 @@ from DIRAC.Resources.Storage.StorageElement import StorageElement from DIRAC.TransformationSystem.Client import TransformationStatus from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient -from DIRAC.WorkloadManagementSystem.Client.JobMonitoringClient import JobMonitoringClient -from DIRAC.WorkloadManagementSystem.Client.WMSClient import WMSClient from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB +from DIRAC.WorkloadManagementSystem.Service.JobPolicy import ( + RIGHT_DELETE, + RIGHT_KILL, +) +from DIRAC.WorkloadManagementSystem.Utilities.jobAdministration import kill_delete_jobs # # agent's name AGENT_NAME = "Transformation/TransformationCleaningAgent" @@ -58,8 +61,6 @@ def __init__(self, *args, **kwargs): # # transformation client self.transClient = None - # # wms client - self.wmsClient = None # # request client self.reqClient = None # # file catalog client @@ -120,14 +121,10 @@ def initialize(self): # # transformation client self.transClient = TransformationClient() - # # wms client - self.wmsClient = WMSClient() # # request client self.reqClient = ReqClient() # # file catalog client self.metadataClient = FileCatalogClient() - # # job monitoring client - self.jobMonitoringClient = JobMonitoringClient() # # job DB self.jobDB = JobDB() @@ -271,7 +268,7 @@ def finalize(self): # Remove JobIDs that were unknown to the TransformationSystem jobGroupsToCheck = [str(transDict["TransformationID"]).zfill(8) for transDict in toClean + toArchive] - res = self.jobMonitoringClient.getJobs({"JobGroup": jobGroupsToCheck}) + res = self.jobDB.selectJobs({"JobGroup": jobGroupsToCheck}) if not res["OK"]: return res jobIDsToRemove = [int(jobID) for jobID in res["Value"]] @@ -613,8 +610,8 @@ def __removeWMSTasks(self, transJobIDs): # Prevent 0 job IDs jobIDs = [int(j) for j in transJobIDs if int(j)] allRemove = True - for jobList in breakListIntoChunks(jobIDs, 500): - res = self.wmsClient.killJob(jobList, force=True) + for jobList in breakListIntoChunks(jobIDs, 1000): + res = kill_delete_jobs(RIGHT_KILL, jobList, force=True) if res["OK"]: self.log.info(f"Successfully killed {len(jobList)} jobs from WMS") elif ("InvalidJobIDs" in res) and ("NonauthorizedJobIDs" not in res) and ("FailedJobIDs" not in res): @@ -626,7 +623,7 @@ def __removeWMSTasks(self, transJobIDs): self.log.error("Failed to kill jobs", f"(n={len(res['FailedJobIDs'])})") allRemove = False - res = self.wmsClient.deleteJob(jobList) + res = kill_delete_jobs(RIGHT_DELETE, jobList, force=True) if res["OK"]: self.log.info("Successfully deleted jobs from WMS", f"(n={len(jobList)})") elif ("InvalidJobIDs" in res) and ("NonauthorizedJobIDs" not in res) and ("FailedJobIDs" not in res): diff --git a/src/DIRAC/TransformationSystem/scripts/dirac_transformation_archive.py b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_archive.py deleted file mode 100755 index 0115a322f89..00000000000 --- a/src/DIRAC/TransformationSystem/scripts/dirac_transformation_archive.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python -""" -Archive a transformation -""" -from DIRAC.Core.Base.Script import Script - - -@Script() -def main(): - # Registering arguments will automatically add their description to the help menu - Script.registerArgument(["transID: transformation ID"]) - _, args = Script.parseCommandLine() - - transIDs = [int(arg) for arg in args] - - from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent - - agent = TransformationCleaningAgent( - "Transformation/TransformationCleaningAgent", - "Transformation/TransformationCleaningAgent", - "dirac-transformation-archive", - ) - agent.initialize() - - for transID in transIDs: - agent.archiveTransformation(transID) - - -if __name__ == "__main__": - main() diff --git a/src/DIRAC/TransformationSystem/scripts/dirac_transformation_clean.py b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_clean.py deleted file mode 100755 index 63f2988f2d2..00000000000 --- a/src/DIRAC/TransformationSystem/scripts/dirac_transformation_clean.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python -""" -Clean a tranformation -""" -from DIRAC.Core.Base.Script import Script - - -@Script() -def main(): - # Registering arguments will automatically add their description to the help menu - Script.registerArgument(["transID: transformation ID"]) - _, args = Script.parseCommandLine() - - from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent - - transIDs = [int(arg) for arg in args] - - agent = TransformationCleaningAgent( - "Transformation/TransformationCleaningAgent", - "Transformation/TransformationCleaningAgent", - "dirac-transformation-clean", - ) - agent.initialize() - - for transID in transIDs: - agent.cleanTransformation(transID) - - -if __name__ == "__main__": - main() diff --git a/src/DIRAC/TransformationSystem/scripts/dirac_transformation_remove_output.py b/src/DIRAC/TransformationSystem/scripts/dirac_transformation_remove_output.py deleted file mode 100755 index cda85e5b755..00000000000 --- a/src/DIRAC/TransformationSystem/scripts/dirac_transformation_remove_output.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python -""" -Remove the outputs produced by a transformation -""" -from DIRAC.Core.Base.Script import Script - - -@Script() -def main(): - # Registering arguments will automatically add their description to the help menu - Script.registerArgument(["transID: transformation ID"]) - _, args = Script.parseCommandLine() - - transIDs = [int(arg) for arg in args] - - from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent - - agent = TransformationCleaningAgent( - "Transformation/TransformationCleaningAgent", - "Transformation/TransformationCleaningAgent", - "dirac-transformation-remove-output", - ) - agent.initialize() - - for transID in transIDs: - agent.removeTransformationOutput(transID) - - -if __name__ == "__main__": - main() diff --git a/src/DIRAC/WorkloadManagementSystem/Agent/JobCleaningAgent.py b/src/DIRAC/WorkloadManagementSystem/Agent/JobCleaningAgent.py index 24728897175..020f4e25118 100644 --- a/src/DIRAC/WorkloadManagementSystem/Agent/JobCleaningAgent.py +++ b/src/DIRAC/WorkloadManagementSystem/Agent/JobCleaningAgent.py @@ -35,10 +35,12 @@ from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient from DIRAC.RequestManagementSystem.Client.Request import Request from DIRAC.WorkloadManagementSystem.Client import JobStatus -from DIRAC.WorkloadManagementSystem.Client.JobMonitoringClient import JobMonitoringClient from DIRAC.WorkloadManagementSystem.Client.WMSClient import WMSClient from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB from DIRAC.WorkloadManagementSystem.DB.SandboxMetadataDB import SandboxMetadataDB +from DIRAC.WorkloadManagementSystem.Service.JobPolicy import RIGHT_DELETE +from DIRAC.WorkloadManagementSystem.Utilities.jobAdministration import kill_delete_jobs +from DIRAC.WorkloadManagementSystem.Utilities.JobParameters import getJobParameters class JobCleaningAgent(AgentModule): @@ -230,11 +232,11 @@ def _deleteRemoveJobs(self, jobList, remove=False): if not res["OK"]: self.log.error("No DN found", f"for {user}") return res - wmsClient = WMSClient(useCertificates=True, delegatedDN=res["Value"][0], delegatedGroup=ownerGroup) if remove: + wmsClient = WMSClient(useCertificates=True, delegatedDN=res["Value"][0], delegatedGroup=ownerGroup) result = wmsClient.removeJob(jobsList) else: - result = wmsClient.deleteJob(jobsList) + result = kill_delete_jobs(RIGHT_DELETE, jobsList) if not result["OK"]: self.log.error( f"Could not {'remove' if remove else 'delete'} jobs", @@ -294,7 +296,8 @@ def deleteJobOversizedSandbox(self, jobIDList): failed = {} successful = {} - result = JobMonitoringClient().getJobParameters(jobIDList, ["OutputSandboxLFN"]) + jobIDs = [int(jobID) for jobID in jobIDList] + result = getJobParameters(jobIDs, "OutputSandboxLFN") if not result["OK"]: return result osLFNDict = result["Value"] diff --git a/src/DIRAC/WorkloadManagementSystem/Agent/StalledJobAgent.py b/src/DIRAC/WorkloadManagementSystem/Agent/StalledJobAgent.py index d3608fbee38..643824eefcd 100755 --- a/src/DIRAC/WorkloadManagementSystem/Agent/StalledJobAgent.py +++ b/src/DIRAC/WorkloadManagementSystem/Agent/StalledJobAgent.py @@ -14,16 +14,16 @@ from DIRAC import S_ERROR, S_OK, gConfig from DIRAC.AccountingSystem.Client.Types.Job import Job from DIRAC.ConfigurationSystem.Client.Helpers import cfgPath -from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getDNForUsername from DIRAC.Core.Base.AgentModule import AgentModule from DIRAC.Core.Utilities import DErrno from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd from DIRAC.Core.Utilities.TimeUtilities import fromString, second, toEpoch from DIRAC.WorkloadManagementSystem.Client import JobMinorStatus, JobStatus -from DIRAC.WorkloadManagementSystem.Client.WMSClient import WMSClient from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB from DIRAC.WorkloadManagementSystem.DB.JobLoggingDB import JobLoggingDB from DIRAC.WorkloadManagementSystem.DB.PilotAgentsDB import PilotAgentsDB +from DIRAC.WorkloadManagementSystem.Service.JobPolicy import RIGHT_KILL +from DIRAC.WorkloadManagementSystem.Utilities.jobAdministration import kill_delete_jobs from DIRAC.WorkloadManagementSystem.Utilities.JobParameters import getJobParameters from DIRAC.WorkloadManagementSystem.Utilities.Utils import rescheduleJobs @@ -235,7 +235,7 @@ def _failStalledJobs(self, jobID): # Set the jobs Failed, send them a kill signal in case they are not really dead # and send accounting info if setFailed: - res = self._sendKillCommand(jobID) + res = kill_delete_jobs(RIGHT_KILL, [jobID], nonauthJobList=[], force=True) if not res["OK"]: self.log.error("Failed to kill job", jobID) @@ -574,26 +574,3 @@ def _failSubmittingJobs(self): continue return S_OK() - - def _sendKillCommand(self, job): - """Send a kill signal to the job such that it cannot continue running. - - :param int job: ID of job to send kill command - """ - - res = self.jobDB.getJobAttribute(job, "Owner") - if not res["OK"]: - return res - owner = res["Value"] - - res = self.jobDB.getJobAttribute(job, "OwnerGroup") - if not res["OK"]: - return res - ownerGroup = res["Value"] - - wmsClient = WMSClient( - useCertificates=True, - delegatedDN=getDNForUsername(owner)["Value"][0] if owner else None, - delegatedGroup=ownerGroup, - ) - return wmsClient.killJob(job) diff --git a/src/DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_JobCleaningAgent.py b/src/DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_JobCleaningAgent.py index f885ce9071a..310addd6363 100644 --- a/src/DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_JobCleaningAgent.py +++ b/src/DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_JobCleaningAgent.py @@ -1,10 +1,11 @@ """ Test class for Job Cleaning Agent """ -import pytest from unittest.mock import MagicMock +import pytest + # DIRAC Components -from DIRAC import gLogger, S_OK +from DIRAC import S_OK, gLogger from DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent import JobCleaningAgent gLogger.setLevel("DEBUG") @@ -32,7 +33,6 @@ def jca(mocker): mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.JobDB.selectJobs", side_effect=mockReply) mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.JobDB.__init__", side_effect=mockNone) mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.ReqClient", return_value=mockNone) - mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.JobMonitoringClient", return_value=mockJMC) jca = JobCleaningAgent() jca.log = gLogger @@ -98,7 +98,7 @@ def test_deleteJobsByStatus(jca, conditions, mockReplyInput, expected): "inputs, params, expected", [ ([], {"OK": True, "Value": {}}, {"OK": True, "Value": {"Failed": {}, "Successful": {}}}), - (["a", "b"], {"OK": True, "Value": {}}, {"OK": True, "Value": {"Failed": {}, "Successful": {}}}), + (["123", "456"], {"OK": True, "Value": {}}, {"OK": True, "Value": {"Failed": {}, "Successful": {}}}), ( [], {"OK": True, "Value": {1: {"OutputSandboxLFN": "/some/lfn/1.txt"}}}, @@ -113,11 +113,11 @@ def test_deleteJobsByStatus(jca, conditions, mockReplyInput, expected): {"OK": True, "Value": {"Failed": {}, "Successful": {1: "/some/lfn/1.txt", 2: "/some/other/lfn/2.txt"}}}, ), ( - ["a", "b"], + ["123", "456"], {"OK": True, "Value": {1: {"OutputSandboxLFN": "/some/lfn/1.txt"}}}, {"OK": True, "Value": {"Failed": {}, "Successful": {1: "/some/lfn/1.txt"}}}, ), - (["a", "b"], {"OK": False}, {"OK": False}), + (["123", "456"], {"OK": False}, {"OK": False}), ], ) def test_deleteJobOversizedSandbox(mocker, inputs, params, expected): @@ -127,10 +127,10 @@ def test_deleteJobOversizedSandbox(mocker, inputs, params, expected): mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.AgentModule.am_getOption", return_value=mockAM) mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.JobDB", return_value=mockNone) mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.ReqClient", return_value=mockNone) - mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.JobMonitoringClient", return_value=mockJMC) mocker.patch( "DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.getDNForUsername", return_value=S_OK(["/bih/boh/DN"]) ) + mocker.patch("DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent.getJobParameters", return_value=params) jobCleaningAgent = JobCleaningAgent() jobCleaningAgent.log = gLogger @@ -138,8 +138,6 @@ def test_deleteJobOversizedSandbox(mocker, inputs, params, expected): jobCleaningAgent._AgentModule__configDefaults = mockAM jobCleaningAgent.initialize() - mockJMC.getJobParameters.return_value = params - result = jobCleaningAgent.deleteJobOversizedSandbox(inputs) assert result == expected diff --git a/src/DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_StalledJobAgent.py b/src/DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_StalledJobAgent.py index 58dc4653153..e182a64c707 100644 --- a/src/DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_StalledJobAgent.py +++ b/src/DIRAC/WorkloadManagementSystem/Agent/test/Test_Agent_StalledJobAgent.py @@ -28,8 +28,6 @@ def sja(mocker): mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.rescheduleJobs", return_value=MagicMock()) mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.PilotAgentsDB", return_value=MagicMock()) mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.getJobParameters", return_value=MagicMock()) - mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.WMSClient", return_value=MagicMock()) - mocker.patch("DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent.getDNForUsername", return_value=MagicMock()) stalledJobAgent = StalledJobAgent() stalledJobAgent._AgentModule__configDefaults = mockAM diff --git a/src/DIRAC/WorkloadManagementSystem/Client/WMSClient.py b/src/DIRAC/WorkloadManagementSystem/Client/WMSClient.py index d5d70c653c6..9526d9b1cae 100755 --- a/src/DIRAC/WorkloadManagementSystem/Client/WMSClient.py +++ b/src/DIRAC/WorkloadManagementSystem/Client/WMSClient.py @@ -2,11 +2,10 @@ methods necessary to communicate with the Workload Management System """ import os -from io import StringIO import time +from io import StringIO -from DIRAC import S_OK, S_ERROR, gLogger - +from DIRAC import S_ERROR, S_OK, gLogger from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.Core.Utilities import File from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd diff --git a/src/DIRAC/WorkloadManagementSystem/Service/JobManagerHandler.py b/src/DIRAC/WorkloadManagementSystem/Service/JobManagerHandler.py index 2a02bdade9a..7264285627f 100755 --- a/src/DIRAC/WorkloadManagementSystem/Service/JobManagerHandler.py +++ b/src/DIRAC/WorkloadManagementSystem/Service/JobManagerHandler.py @@ -21,9 +21,7 @@ from DIRAC.Core.Utilities.JEncode import strToIntDict from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager -from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient from DIRAC.WorkloadManagementSystem.Client import JobStatus -from DIRAC.WorkloadManagementSystem.Client.JobStatus import filterJobStateTransition from DIRAC.WorkloadManagementSystem.Service.JobPolicy import ( RIGHT_DELETE, RIGHT_KILL, @@ -32,6 +30,7 @@ RIGHT_SUBMIT, JobPolicy, ) +from DIRAC.WorkloadManagementSystem.Utilities.jobAdministration import kill_delete_jobs from DIRAC.WorkloadManagementSystem.Utilities.JobModel import JobDescriptionModel from DIRAC.WorkloadManagementSystem.Utilities.ParametricJob import generateParametricJobs, getParameterVectorLength from DIRAC.WorkloadManagementSystem.Utilities.Utils import rescheduleJobs @@ -430,131 +429,28 @@ def export_removeJob(self, jobIDs): return S_OK(validJobList) - def __deleteJob(self, jobID, force=False): - """Set the job status to "Deleted" - and remove the pilot that ran and its logging info if the pilot is finished. - - :param int jobID: job ID - :return: S_OK()/S_ERROR() - """ - if not (result := self.jobDB.setJobStatus(jobID, JobStatus.DELETED, "Checking accounting", force=force))["OK"]: - return result - - if not (result := self.taskQueueDB.deleteJob(jobID))["OK"]: - self.log.warn("Failed to delete job from the TaskQueue") - - # if it was the last job for the pilot - result = self.pilotAgentsDB.getPilotsForJobID(jobID) - if not result["OK"]: - self.log.error("Failed to get Pilots for JobID", result["Message"]) - return result - for pilot in result["Value"]: - res = self.pilotAgentsDB.getJobsForPilot(pilot) - if not res["OK"]: - self.log.error("Failed to get jobs for pilot", res["Message"]) - return res - if not res["Value"]: # if list of jobs for pilot is empty, delete pilot - result = self.pilotAgentsDB.getPilotInfo(pilotID=pilot) - if not result["OK"]: - self.log.error("Failed to get pilot info", result["Message"]) - return result - ret = self.pilotAgentsDB.deletePilot(result["Value"]["PilotJobReference"]) - if not ret["OK"]: - self.log.error("Failed to delete pilot from PilotAgentsDB", ret["Message"]) - return ret - - return S_OK() + ########################################################################### + types_deleteJob = [] - def __killJob(self, jobID, sendKillCommand=True, force=False): - """Kill one job + def export_deleteJob(self, jobIDs, force=False): + """Delete jobs specified in the jobIDs list - :param int jobID: job ID - :param bool sendKillCommand: send kill command + :param list jobIDs: list of job IDs - :return: S_OK()/S_ERROR() + :return: S_OK/S_ERROR """ - if sendKillCommand: - if not (result := self.jobDB.setJobCommand(jobID, "Kill"))["OK"]: - return result - - self.log.info("Job marked for termination", jobID) - if not (result := self.jobDB.setJobStatus(jobID, JobStatus.KILLED, "Marked for termination", force=force))[ - "OK" - ]: - self.log.warn("Failed to set job Killed status", result["Message"]) - if not (result := self.taskQueueDB.deleteJob(jobID))["OK"]: - self.log.warn("Failed to delete job from the TaskQueue", result["Message"]) - - return S_OK() - - def _kill_delete_jobs(self, jobIDList, right, force=False): - """Kill (== set the status to "KILLED") or delete (== set the status to "DELETED") jobs as necessary - :param list jobIDList: job IDs - :param str right: RIGHT_KILL or RIGHT_DELETE - - :return: S_OK()/S_ERROR() - """ - jobList = self.__getJobList(jobIDList) + jobList = self.__getJobList(jobIDs) if not jobList: self.log.warn("No jobs specified") return S_OK([]) - validJobList, invalidJobList, nonauthJobList, ownerJobList = self.jobPolicy.evaluateJobRights(jobList, right) - - badIDs = [] - - killJobList = [] - deleteJobList = [] - if validJobList: - # Get the jobs allowed to transition to the Killed state - filterRes = filterJobStateTransition(validJobList, JobStatus.KILLED) - if not filterRes["OK"]: - return filterRes - killJobList.extend(filterRes["Value"]) - - if not right == RIGHT_KILL: - # Get the jobs allowed to transition to the Deleted state - filterRes = filterJobStateTransition(validJobList, JobStatus.DELETED) - if not filterRes["OK"]: - return filterRes - deleteJobList.extend(filterRes["Value"]) - - # Look for jobs that are in the Staging state to send kill signal to the stager - result = self.jobDB.getJobsAttributes(killJobList, ["Status"]) - if not result["OK"]: - return result - stagingJobList = [jobID for jobID, sDict in result["Value"].items() if sDict["Status"] == JobStatus.STAGING] - - for jobID in killJobList: - result = self.__killJob(jobID, force=force) - if not result["OK"]: - badIDs.append(jobID) - - for jobID in deleteJobList: - result = self.__deleteJob(jobID, force=force) - if not result["OK"]: - badIDs.append(jobID) - - if stagingJobList: - stagerClient = StorageManagerClient() - self.log.info("Going to send killing signal to stager as well!") - result = stagerClient.killTasksBySourceTaskID(stagingJobList) - if not result["OK"]: - self.log.warn("Failed to kill some Stager tasks", result["Message"]) + validJobList, invalidJobList, nonauthJobList, ownerJobList = self.jobPolicy.evaluateJobRights( + jobList, RIGHT_DELETE + ) - if nonauthJobList or badIDs: - result = S_ERROR("Some jobs failed deletion") - if nonauthJobList: - self.log.warn("Non-authorized JobIDs won't be deleted", str(nonauthJobList)) - result["NonauthorizedJobIDs"] = nonauthJobList - if badIDs: - self.log.warn("JobIDs failed to be deleted", str(badIDs)) - result["FailedJobIDs"] = badIDs - return result + result = kill_delete_jobs(RIGHT_DELETE, validJobList, nonauthJobList, force=force) - jobsList = killJobList if right == RIGHT_KILL else deleteJobList - result = S_OK(jobsList) result["requireProxyUpload"] = len(ownerJobList) > 0 and self.__checkIfProxyUploadIsRequired() if invalidJobList: @@ -563,30 +459,33 @@ def _kill_delete_jobs(self, jobIDList, right, force=False): return result ########################################################################### - types_deleteJob = [] + types_killJob = [] - def export_deleteJob(self, jobIDs, force=False): - """Delete jobs specified in the jobIDs list + def export_killJob(self, jobIDs, force=False): + """Kill jobs specified in the jobIDs list :param list jobIDs: list of job IDs :return: S_OK/S_ERROR """ - return self._kill_delete_jobs(jobIDs, RIGHT_DELETE, force=force) + jobList = self.__getJobList(jobIDs) + if not jobList: + self.log.warn("No jobs specified") + return S_OK([]) - ########################################################################### - types_killJob = [] + validJobList, invalidJobList, nonauthJobList, ownerJobList = self.jobPolicy.evaluateJobRights( + jobList, RIGHT_KILL + ) - def export_killJob(self, jobIDs, force=False): - """Kill jobs specified in the jobIDs list + result = kill_delete_jobs(RIGHT_KILL, validJobList, nonauthJobList, force=force) - :param list jobIDs: list of job IDs + result["requireProxyUpload"] = len(ownerJobList) > 0 and self.__checkIfProxyUploadIsRequired() - :return: S_OK/S_ERROR - """ + if invalidJobList: + result["InvalidJobIDs"] = invalidJobList - return self._kill_delete_jobs(jobIDs, RIGHT_KILL, force=force) + return result ########################################################################### types_resetJob = [] diff --git a/src/DIRAC/WorkloadManagementSystem/Utilities/jobAdministration.py b/src/DIRAC/WorkloadManagementSystem/Utilities/jobAdministration.py new file mode 100644 index 00000000000..fe4bf9b4138 --- /dev/null +++ b/src/DIRAC/WorkloadManagementSystem/Utilities/jobAdministration.py @@ -0,0 +1,129 @@ +from DIRAC import S_ERROR, S_OK, gLogger +from DIRAC.StorageManagementSystem.DB.StorageManagementDB import StorageManagementDB +from DIRAC.WorkloadManagementSystem.Client import JobStatus +from DIRAC.WorkloadManagementSystem.Client.JobStatus import filterJobStateTransition +from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB +from DIRAC.WorkloadManagementSystem.DB.PilotAgentsDB import PilotAgentsDB +from DIRAC.WorkloadManagementSystem.DB.TaskQueueDB import TaskQueueDB +from DIRAC.WorkloadManagementSystem.Service.JobPolicy import RIGHT_KILL, RIGHT_DELETE + + +def _deleteJob(jobID, force=False): + """Set the job status to "Deleted" + and remove the pilot that ran and its logging info if the pilot is finished. + + :param int jobID: job ID + :return: S_OK()/S_ERROR() + """ + if not (result := JobDB().setJobStatus(jobID, JobStatus.DELETED, "Checking accounting", force=force))["OK"]: + gLogger.warn("Failed to set job Deleted status", result["Message"]) + return result + + if not (result := TaskQueueDB().deleteJob(jobID))["OK"]: + gLogger.warn("Failed to delete job from the TaskQueue") + + # if it was the last job for the pilot + result = PilotAgentsDB().getPilotsForJobID(jobID) + if not result["OK"]: + gLogger.error("Failed to get Pilots for JobID", result["Message"]) + return result + for pilot in result["Value"]: + res = PilotAgentsDB().getJobsForPilot(pilot) + if not res["OK"]: + gLogger.error("Failed to get jobs for pilot", res["Message"]) + return res + if not res["Value"]: # if list of jobs for pilot is empty, delete pilot + result = PilotAgentsDB().getPilotInfo(pilotID=pilot) + if not result["OK"]: + gLogger.error("Failed to get pilot info", result["Message"]) + return result + ret = PilotAgentsDB().deletePilot(result["Value"]["PilotJobReference"]) + if not ret["OK"]: + gLogger.error("Failed to delete pilot from PilotAgentsDB", ret["Message"]) + return ret + + return S_OK() + + +def _killJob(jobID, sendKillCommand=True, force=False): + """Kill one job + + :param int jobID: job ID + :param bool sendKillCommand: send kill command + + :return: S_OK()/S_ERROR() + """ + if sendKillCommand: + if not (result := JobDB().setJobCommand(jobID, "Kill"))["OK"]: + gLogger.warn("Failed to set job Kill command", result["Message"]) + return result + + gLogger.info("Job marked for termination", jobID) + if not (result := JobDB().setJobStatus(jobID, JobStatus.KILLED, "Marked for termination", force=force))["OK"]: + gLogger.warn("Failed to set job Killed status", result["Message"]) + if not (result := TaskQueueDB().deleteJob(jobID))["OK"]: + gLogger.warn("Failed to delete job from the TaskQueue", result["Message"]) + + return S_OK() + + +def kill_delete_jobs(right, validJobList, nonauthJobList=[], force=False): + """Kill (== set the status to "KILLED") or delete (== set the status to "DELETED") jobs as necessary + + :param str right: RIGHT_KILL or RIGHT_DELETE + + :return: S_OK()/S_ERROR() + """ + badIDs = [] + + killJobList = [] + deleteJobList = [] + if validJobList: + # Get the jobs allowed to transition to the Killed state + filterRes = filterJobStateTransition(validJobList, JobStatus.KILLED) + if not filterRes["OK"]: + return filterRes + killJobList.extend(filterRes["Value"]) + + if right == RIGHT_DELETE: + # Get the jobs allowed to transition to the Deleted state + filterRes = filterJobStateTransition(validJobList, JobStatus.DELETED) + if not filterRes["OK"]: + return filterRes + deleteJobList.extend(filterRes["Value"]) + + for jobID in killJobList: + result = _killJob(jobID, force=force) + if not result["OK"]: + badIDs.append(jobID) + + for jobID in deleteJobList: + result = _deleteJob(jobID, force=force) + if not result["OK"]: + badIDs.append(jobID) + + # Look for jobs that are in the Staging state to send kill signal to the stager + result = JobDB().getJobsAttributes(killJobList, ["Status"]) + if not result["OK"]: + return result + stagingJobList = [jobID for jobID, sDict in result["Value"].items() if sDict["Status"] == JobStatus.STAGING] + + if stagingJobList: + stagerDB = StorageManagementDB() + gLogger.info("Going to send killing signal to stager as well!") + result = stagerDB.killTasksBySourceTaskID(stagingJobList) + if not result["OK"]: + gLogger.warn("Failed to kill some Stager tasks", result["Message"]) + + if nonauthJobList or badIDs: + result = S_ERROR("Some jobs failed deletion") + if nonauthJobList: + gLogger.warn("Non-authorized JobIDs won't be deleted", str(nonauthJobList)) + result["NonauthorizedJobIDs"] = nonauthJobList + if badIDs: + gLogger.warn("JobIDs failed to be deleted", str(badIDs)) + result["FailedJobIDs"] = badIDs + return result + + jobsList = killJobList if right == RIGHT_KILL else deleteJobList + return S_OK(jobsList) diff --git a/src/DIRAC/WorkloadManagementSystem/Utilities/test/Test_JobAdministration.py b/src/DIRAC/WorkloadManagementSystem/Utilities/test/Test_JobAdministration.py new file mode 100644 index 00000000000..3d484a7bf99 --- /dev/null +++ b/src/DIRAC/WorkloadManagementSystem/Utilities/test/Test_JobAdministration.py @@ -0,0 +1,40 @@ +""" unit test (pytest) of JobAdministration module +""" + +from unittest.mock import MagicMock + +import pytest + +# sut +from DIRAC.WorkloadManagementSystem.Utilities.jobAdministration import kill_delete_jobs + + +@pytest.mark.parametrize( + "jobIDs_list, right, filtered_jobs, expected_res, expected_value", + [ + ([], "Kill", [], True, []), + ([], "Delete", [], True, []), + (1, "Kill", [], True, []), + (1, "Kill", [1], True, [1]), + ([1, 2], "Kill", [], True, []), + ([1, 2], "Kill", [1], True, [1]), + (1, "Kill", [1], True, [1]), + ([1, 2], "Kill", [1], True, [1]), + ([1, 2], "Kill", [2], True, [2]), + ([1, 2], "Kill", [], True, []), + ([1, 2], "Kill", [1, 2], True, [1, 2]), + ], +) +def test___kill_delete_jobs(mocker, jobIDs_list, right, filtered_jobs, expected_res, expected_value): + mocker.patch("DIRAC.WorkloadManagementSystem.Utilities.jobAdministration.JobDB", MagicMock()) + mocker.patch("DIRAC.WorkloadManagementSystem.Utilities.jobAdministration.TaskQueueDB", MagicMock()) + mocker.patch("DIRAC.WorkloadManagementSystem.Utilities.jobAdministration.PilotAgentsDB", MagicMock()) + mocker.patch("DIRAC.WorkloadManagementSystem.Utilities.jobAdministration.StorageManagementDB", MagicMock()) + mocker.patch( + "DIRAC.WorkloadManagementSystem.Utilities.jobAdministration.filterJobStateTransition", + return_value={"OK": True, "Value": filtered_jobs}, + ) + + res = kill_delete_jobs(right, jobIDs_list) + assert res["OK"] == expected_res + assert res["Value"] == expected_value diff --git a/src/DIRAC/WorkloadManagementSystem/Utilities/test/Test_JobManager.py b/src/DIRAC/WorkloadManagementSystem/Utilities/test/Test_JobManager.py deleted file mode 100644 index fb1b5f64552..00000000000 --- a/src/DIRAC/WorkloadManagementSystem/Utilities/test/Test_JobManager.py +++ /dev/null @@ -1,58 +0,0 @@ -""" unit test (pytest) of JobManager service -""" - -from unittest.mock import MagicMock -import pytest - -from DIRAC import gLogger - -gLogger.setLevel("DEBUG") - -from DIRAC.WorkloadManagementSystem.Service.JobPolicy import ( - RIGHT_DELETE, - RIGHT_KILL, -) - -# sut -from DIRAC.WorkloadManagementSystem.Service.JobManagerHandler import JobManagerHandlerMixin - -# mocks -jobPolicy_mock = MagicMock() -jobDB_mock = MagicMock() -jobDB_mock.getJobsAttributes.return_value = {"OK": True, "Value": {}} - - -@pytest.mark.parametrize( - "jobIDs_list, right, lists, filteredJobsList, expected_res, expected_value", - [ - ([], RIGHT_KILL, ([], [], [], []), [], True, []), - ([], RIGHT_DELETE, ([], [], [], []), [], True, []), - (1, RIGHT_KILL, ([], [], [], []), [], True, []), - (1, RIGHT_KILL, ([1], [], [], []), [], True, []), - ([1, 2], RIGHT_KILL, ([], [], [], []), [], True, []), - ([1, 2], RIGHT_KILL, ([1], [], [], []), [], True, []), - (1, RIGHT_KILL, ([1], [], [], []), [1], True, [1]), - ([1, 2], RIGHT_KILL, ([1], [], [], []), [1], True, [1]), - ([1, 2], RIGHT_KILL, ([1], [2], [], []), [1], True, [1]), - ([1, 2], RIGHT_KILL, ([1], [2], [], []), [], True, []), - ([1, 2], RIGHT_KILL, ([1, 2], [], [], []), [1, 2], True, [1, 2]), - ], -) -def test___kill_delete_jobs(mocker, jobIDs_list, right, lists, filteredJobsList, expected_res, expected_value): - mocker.patch( - "DIRAC.WorkloadManagementSystem.Service.JobManagerHandler.filterJobStateTransition", - return_value={"OK": True, "Value": filteredJobsList}, - ) - - JobManagerHandlerMixin.log = gLogger - JobManagerHandlerMixin.jobPolicy = jobPolicy_mock - JobManagerHandlerMixin.jobDB = jobDB_mock - JobManagerHandlerMixin.taskQueueDB = MagicMock() - - jobPolicy_mock.evaluateJobRights.return_value = lists - - jm = JobManagerHandlerMixin() - - res = jm._kill_delete_jobs(jobIDs_list, right) - assert res["OK"] == expected_res - assert res["Value"] == expected_value diff --git a/tests/Integration/WorkloadManagementSystem/Test_Client_WMS.py b/tests/Integration/WorkloadManagementSystem/Test_Client_WMS.py index 0c76f900e19..fc3e5ec6c22 100644 --- a/tests/Integration/WorkloadManagementSystem/Test_Client_WMS.py +++ b/tests/Integration/WorkloadManagementSystem/Test_Client_WMS.py @@ -451,7 +451,13 @@ def test_JobStateUpdateAndJobMonitoringMultiple(lfn: str) -> None: assert res["OK"], res["Message"] assert res["Value"]["Status"] == JobStatus.RUNNING finally: - jobManagerClient.removeJob(jobIDs) + res = jobManagerClient.killJob(jobIDs) + assert res["OK"], res["Message"] + time.sleep(5) + res = jobManagerClient.deleteJob(jobIDs) + assert res["OK"], res["Message"] + res = jobManagerClient.removeJob(jobIDs) + assert res["OK"], res["Message"] def test_JobManagerClient_removeJob() -> None: