From aab3caef49375df750c0db9be702f8bc19f09d90 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Thu, 23 Jun 2022 14:16:52 +0200 Subject: [PATCH 0001/1135] Revert "fix (FS): Support receiving proxies from clients running v7.3.x" This reverts commit ce3963dd600e6919868d8a05ddcac5d848edc504. --- src/DIRAC/FrameworkSystem/Service/ProxyManagerHandler.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/DIRAC/FrameworkSystem/Service/ProxyManagerHandler.py b/src/DIRAC/FrameworkSystem/Service/ProxyManagerHandler.py index 985a81dcd9b..5c86fb5c180 100644 --- a/src/DIRAC/FrameworkSystem/Service/ProxyManagerHandler.py +++ b/src/DIRAC/FrameworkSystem/Service/ProxyManagerHandler.py @@ -105,9 +105,6 @@ def export_completeDelegationUpload(self, requestId, pemChain): :return: S_OK(dict)/S_ERROR() -- dict contain proxies """ - if isinstance(pemChain, bytes): - # The client is running v7.3.x and we need to decode for backwards compatibility - pemChain = pemChain.decode("ascii") credDict = self.getRemoteCredentials() userId = f'{credDict["username"]}:{credDict["group"]}' retVal = self.__proxyDB.completeDelegation(requestId, credDict["DN"], pemChain) From dd095006b0895f6c07902f2ab7bf126593b4b09b Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Thu, 23 Jun 2022 14:26:53 +0200 Subject: [PATCH 0002/1135] feat: Allow semantic CI check to accept git-revert commits --- .github/workflows/semantic.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/semantic.yml b/.github/workflows/semantic.yml index c30c89ab772..07af3517572 100644 --- a/.github/workflows/semantic.yml +++ b/.github/workflows/semantic.yml @@ -15,7 +15,7 @@ jobs: - name: Check Commit Format uses: gsactions/commit-message-checker@v1 with: - pattern: '^(docs|feat|fix|refactor|style|test|sweep)( ?\(.*\))?: .+$' + pattern: '^((docs|feat|fix|refactor|style|test|sweep)( ?\(.*\))?: .+|Revert ".+")$' excludeDescription: 'true' # optional: this excludes the description body of a pull request excludeTitle: 'true' # optional: this excludes the title of a pull request checkAllCommitMessages: 'true' # optional: this checks all commits associated with a pull request From 9aee58822cf6421ea3e5ca51a2b85ac8bd65241c Mon Sep 17 00:00:00 2001 From: Simon Mazenoux Date: Mon, 11 Jul 2022 17:46:16 +0200 Subject: [PATCH 0003/1135] fix: testing the presence of Owner instead of OwnerName in jobManifest --- .../scripts/dirac_wms_job_get_jdl.py | 1 - .../TransformationSystem/test/Test_JobInfo.py | 4 - .../Client/JobState/JobManifest.py | 2 +- .../WorkloadManagementSystem/DB/JobDB.py | 2 +- .../WorkloadManagementSystem/Test_JobDB.py | 383 ++++++++++-------- 5 files changed, 216 insertions(+), 176 deletions(-) diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_jdl.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_jdl.py index d7fa8181ba3..57e06d9ae7e 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_jdl.py +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_jdl.py @@ -29,7 +29,6 @@ 'Owner': 'vhamar', 'OwnerDN': '/O=GRID-FR/C=FR/O=CNRS/OU=CPPM/CN=Vanessa Hamar', 'OwnerGroup': 'eela_user', - 'OwnerName': 'vhamar', 'Priority': '1'} """ import DIRAC diff --git a/src/DIRAC/TransformationSystem/test/Test_JobInfo.py b/src/DIRAC/TransformationSystem/test/Test_JobInfo.py index 360ed1ee1d3..74cb93ad20c 100644 --- a/src/DIRAC/TransformationSystem/test/Test_JobInfo.py +++ b/src/DIRAC/TransformationSystem/test/Test_JobInfo.py @@ -74,7 +74,6 @@ def setUp(self): "jobDescription.xml", "SB:ProductionSandboxSE2|/SandBox/i/ilc_prod/5d3/92f/5d392f5266a796018ab6774ef84cbd31.tar.bz2", ], - "OwnerName": "sailer", "StdOutput": "std.out", "JobType": "MCReconstruction_Overlay", "GridEnv": "/cvmfs/grid.cern.ch/emi-ui-3.7.3-1_sl6v2/etc/profile.d/setup-emi3-ui-example", @@ -144,7 +143,6 @@ def setUp(self): "jobDescription.xml", "SB:ProductionSandboxSE2|/SandBox/i/ilc_prod/5d3/92f/5d392f5266a796018ab6774ef84cbd31.tar.bz2", ], - "OwnerName": "sailer", "StdOutput": "std.out", "JobType": "MCReconstruction_Overlay", "GridEnv": "/cvmfs/grid.cern.ch/emi-ui-3.7.3-1_sl6v2/etc/profile.d/setup-emi3-ui-example", @@ -218,7 +216,6 @@ def setUp(self): "jobDescription.xml", "SB:ProductionSandboxSE2|/SandBox/i/ilc_prod/042/d64/042d64cb0fe73720cbd114a73506c582.tar.bz2", ], - "OwnerName": "sailer", "StdOutput": "std.out", "JobType": "MCSimulation", "GridEnv": "/cvmfs/grid.cern.ch/emi-ui-3.7.3-1_sl6v2/etc/profile.d/setup-emi3-ui-example", @@ -278,7 +275,6 @@ def setUp(self): "jobDescription.xml", "SB:ProductionSandboxSE2|/SandBox/i/ilc_prod/b2a/d98/b2ad98c3e240361a4253c4bb277be478.tar.bz2", ], - "OwnerName": "sailer", "StdOutput": "std.out", "JobType": "MCGeneration", "GridEnv": "/cvmfs/grid.cern.ch/emi-ui-3.7.3-1_sl6v2/etc/profile.d/setup-emi3-ui-example", diff --git a/src/DIRAC/WorkloadManagementSystem/Client/JobState/JobManifest.py b/src/DIRAC/WorkloadManagementSystem/Client/JobState/JobManifest.py index 848f55ad53b..b03f60b1144 100644 --- a/src/DIRAC/WorkloadManagementSystem/Client/JobState/JobManifest.py +++ b/src/DIRAC/WorkloadManagementSystem/Client/JobState/JobManifest.py @@ -154,7 +154,7 @@ def check(self): """ Check that the manifest is OK """ - for k in ["OwnerName", "OwnerDN", "OwnerGroup", "DIRACSetup"]: + for k in ["Owner", "OwnerDN", "OwnerGroup", "DIRACSetup"]: if k not in self.__manifest: return S_ERROR("Missing var %s in manifest" % k) diff --git a/src/DIRAC/WorkloadManagementSystem/DB/JobDB.py b/src/DIRAC/WorkloadManagementSystem/DB/JobDB.py index a271cf08f51..f16da87b509 100755 --- a/src/DIRAC/WorkloadManagementSystem/DB/JobDB.py +++ b/src/DIRAC/WorkloadManagementSystem/DB/JobDB.py @@ -932,7 +932,7 @@ def insertNewJobIntoDB( if not result["OK"]: return result jobManifest.setOptionsFromDict( - {"OwnerName": owner, "OwnerDN": ownerDN, "OwnerGroup": ownerGroup, "DIRACSetup": diracSetup} + {"Owner": owner, "OwnerDN": ownerDN, "OwnerGroup": ownerGroup, "DIRACSetup": diracSetup} ) result = jobManifest.check() if not result["OK"]: diff --git a/tests/Integration/WorkloadManagementSystem/Test_JobDB.py b/tests/Integration/WorkloadManagementSystem/Test_JobDB.py index cd459bf9c06..4b0cd08986a 100644 --- a/tests/Integration/WorkloadManagementSystem/Test_JobDB.py +++ b/tests/Integration/WorkloadManagementSystem/Test_JobDB.py @@ -14,7 +14,7 @@ DIRAC.initialize() # Initialize configuration -from DIRAC import gLogger, S_OK +from DIRAC import S_OK, gLogger from DIRAC.WorkloadManagementSystem.Client import JobStatus from DIRAC.WorkloadManagementSystem.Client import JobMinorStatus @@ -22,100 +22,32 @@ from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB jdl = """[ - Executable = "dirac-jobexec"; - StdError = "std.err"; - LogLevel = "info"; - Site = "ANY"; - JobName = "helloWorld"; - Priority = "1"; - InputSandbox = - { - "../../Integration/WorkloadManagementSystem/exe-script.py", - "exe-script.py", - "/tmp/tmpMQEink/jobDescription.xml", - "SB:FedericoSandboxSE|/SandBox/f/fstagni.lhcb_user/0c2/9f5/0c29f53a47d051742346b744c793d4d0.tar.bz2" - }; - Arguments = "jobDescription.xml -o LogLevel=info"; - JobGroup = "lhcb"; - OutputSandbox = - { - "helloWorld.log", - "std.err", - "std.out" - }; - StdOutput = "std.out"; - InputData = ""; - JobType = "User"; -]""" - -originalJDL = """[ - Executable = "dirac-jobexec"; - StdError = "std.err"; - LogLevel = "info"; - Site = "ANY"; - JobName = "helloWorld"; - Priority = "1"; - InputSandbox = - { - "../../Integration/WorkloadManagementSystem/exe-script.py", - "exe-script.py", - "/tmp/tmpMQEink/jobDescription.xml", - "SB:FedericoSandboxSE|/SandBox/f/fstagni.lhcb_user/0c2/9f5/0c29f53a47d051742346b744c793d4d0.tar.bz2" - }; - Arguments = "jobDescription.xml -o LogLevel=info"; - JobGroup = "lhcb"; - OutputSandbox = - { - "helloWorld.log", - "std.err", - "std.out" - }; - StdOutput = "std.out"; - InputData = ""; - JobType = "User"; -]""" - - -def getExpectedJDL(jobID): - return f"""[ Arguments = "jobDescription.xml -o LogLevel=info"; CPUTime = 86400; DIRACSetup = "someSetup"; Executable = "dirac-jobexec"; InputData = ""; InputSandbox = - {{ + { "../../Integration/WorkloadManagementSystem/exe-script.py", "exe-script.py", "/tmp/tmpMQEink/jobDescription.xml", "SB:FedericoSandboxSE|/SandBox/f/fstagni.lhcb_user/0c2/9f5/0c29f53a47d051742346b744c793d4d0.tar.bz2" - }}; + }; JobGroup = "lhcb"; - JobID = {jobID}; JobName = "helloWorld"; - JobRequirements = - [ - CPUTime = 86400; - OwnerDN = "/DN/OF/owner"; - OwnerGroup = "ownerGroup"; - Setup = "someSetup"; - UserPriority = 1; - VirtualOrganization = "vo"; - ]; JobType = "User"; LogLevel = "info"; OutputSandbox = - {{ + { "helloWorld.log", "std.err", "std.out" - }}; + }; Owner = "owner"; OwnerDN = "/DN/OF/owner"; OwnerGroup = "ownerGroup"; - OwnerName = "owner"; Priority = 1; - Site = "ANY"; StdError = "std.err"; StdOutput = "std.out"; VirtualOrganization = "vo"; @@ -135,114 +67,227 @@ def fixturejobDB(): # remove the job entries res = jobDB.selectJobs({}) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] jobs = res["Value"] for job in jobs: res = jobDB.removeJobFromDB(job) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] + + +def test_isValid(jobDB: JobDB): + """Test to check that the JobDB is correctly initialized""" + + assert jobDB.isValid() -def test_insertAndRemoveJobIntoDB(jobDB): +def test_insertNewJobIntoDB(jobDB): + """Test the insertNewJobIntoDB method""" + # Act res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup", "someSetup") - assert res["OK"] is True, res["Message"] + + # Assert + assert res["OK"], res["Message"] jobID = int(res["JobID"]) res = jobDB.getJobAttribute(jobID, "Status") - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] assert res["Value"] == JobStatus.RECEIVED res = jobDB.getJobAttribute(jobID, "MinorStatus") - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] assert res["Value"] == "Job accepted" res = jobDB.getJobAttributes(jobID, ["Status", "MinorStatus"]) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] assert res["Value"] == {"Status": JobStatus.RECEIVED, "MinorStatus": "Job accepted"} res = jobDB.getJobsAttributes(jobID, ["Status", "MinorStatus"]) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] assert res["Value"] == {jobID: {"Status": JobStatus.RECEIVED, "MinorStatus": "Job accepted"}} res = jobDB.getJobOptParameters(jobID) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] assert res["Value"] == {} + +def test_removeJobFromDB(jobDB): + + # Arrange + res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup", "someSetup") + assert res["OK"], res["Message"] + jobID = int(res["JobID"]) + + # Act + res = jobDB.removeJobFromDB(jobID) + + # Assert + assert res["OK"], res["Message"] + + +def test_getJobJDL_original(jobDB: JobDB): + """Test of the getJobJDL method with the original parameter set to True""" + + # Arrange + res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup", "someSetup") + assert res["OK"], res["Message"] + jobID = int(res["JobID"]) + + # Act res = jobDB.getJobJDL(jobID, original=True) - print(" ".join(res["Value"].split())) - assert res["OK"] is True, res["Message"] - assert " ".join(res["Value"].split()) == " ".join(originalJDL.split()) - res = jobDB.getJobJDL(jobID) - print(" ".join(res["Value"].split())) - assert res["OK"] is True, res["Message"] + # Assert + assert res["OK"], res["Message"] print(res["Value"]) - assert " ".join(res["Value"].split()) == " ".join(getExpectedJDL(jobID).split()) + assert " ".join(res["Value"].split()) == " ".join(jdl.split()) + + +def test_getJobJDL_nonOriginal(jobDB: JobDB): + """Test of the getJobJDL method with the original parameter set to True""" + # Arrange res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup", "someSetup") - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] + jobID = int(res["JobID"]) + + # Act + res = jobDB.getJobJDL(jobID, original=False) + + # Assert + assert res["OK"], res["Message"] + print(res["Value"]) + assert " ".join(res["Value"].split()) == " ".join( + f"""[ + Arguments = "jobDescription.xml -o LogLevel=info"; + CPUTime = 86400; + DIRACSetup = "someSetup"; + Executable = "dirac-jobexec"; + InputData = ""; + InputSandbox = + {{ + "../../Integration/WorkloadManagementSystem/exe-script.py", + "exe-script.py", + "/tmp/tmpMQEink/jobDescription.xml", + "SB:FedericoSandboxSE|/SandBox/f/fstagni.lhcb_user/0c2/9f5/0c29f53a47d051742346b744c793d4d0.tar.bz2" + }}; + JobGroup = "lhcb"; + JobID = {jobID}; + JobName = "helloWorld"; + JobRequirements = + [ + CPUTime = 86400; + OwnerDN = "/DN/OF/owner"; + OwnerGroup = "ownerGroup"; + Setup = "someSetup"; + UserPriority = 1; + VirtualOrganization = "vo"; + ]; + JobType = "User"; + LogLevel = "info"; + OutputSandbox = + {{ + "helloWorld.log", + "std.err", + "std.out" + }}; + Owner = "owner"; + OwnerDN = "/DN/OF/owner"; + OwnerGroup = "ownerGroup"; + Priority = 1; + StdError = "std.err"; + StdOutput = "std.out"; + VirtualOrganization = "vo"; + ]""".split() + ) + + +def test_getJobsAttributes(jobDB): + + # Arrange + res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup", "someSetup") + assert res["OK"], res["Message"] + jobID_1 = int(res["JobID"]) + + res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup", "someSetup") + assert res["OK"], res["Message"] jobID_2 = int(res["JobID"]) - # Check that the original jdl is insensitive to the jobID - assert jobDB.getJobJDL(jobID, original=True) == jobDB.getJobJDL(jobID_2, original=True) + # Act + res = jobDB.getJobsAttributes([jobID_1, jobID_2], ["Status", "MinorStatus"]) - res = jobDB.getJobsAttributes([jobID, jobID_2], ["Status", "MinorStatus"]) - assert res["OK"] is True, res["Message"] + # Assert + assert res["OK"], res["Message"] assert res["Value"] == { - jobID: {"Status": JobStatus.RECEIVED, "MinorStatus": "Job accepted"}, + jobID_1: {"Status": JobStatus.RECEIVED, "MinorStatus": "Job accepted"}, jobID_2: {"Status": JobStatus.RECEIVED, "MinorStatus": "Job accepted"}, } - res = jobDB.selectJobs({}) - assert res["OK"] is True, res["Message"] - jobs = res["Value"] - for job in jobs: - res = jobDB.removeJobFromDB(job) - assert res["OK"] is True, res["Message"] - def test_rescheduleJob(jobDB): + # Arrange res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup", "someSetup") - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] jobID = res["JobID"] res = jobDB.getJobAttribute(jobID, "Status") - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] assert res["Value"] == JobStatus.RECEIVED + res = jobDB.getJobAttribute(jobID, "MinorStatus") + assert res["OK"], res["Message"] + assert res["Value"] == "Job accepted" + + res = jobDB.getJobJDL(jobID, original=False) + print(res["Value"]) + assert res["OK"], res["Message"] + jdlWithJobID = res["Value"] + + # Act res = jobDB.rescheduleJob(jobID) - assert res["OK"] is True, res["Message"] + + # Assert + assert res["OK"], res["Message"] + + res = jobDB.getJobJDL(jobID, original=True) + assert res["OK"], res["Message"] + assert " ".join(res["Value"].split()) == " ".join(jdl.split()) + + res = jobDB.getJobJDL(jobID, original=False) + assert res["OK"], res["Message"] + print(res["Value"]) + assert " ".join(res["Value"].split()) == " ".join(jdlWithJobID.split()) res = jobDB.getJobAttribute(jobID, "Status") - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] assert res["Value"] == JobStatus.RECEIVED + res = jobDB.getJobAttribute(jobID, "MinorStatus") - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] assert res["Value"] == JobMinorStatus.RESCHEDULED def test_getCounters(jobDB): res = jobDB.getCounters("Jobs", ["Status", "MinorStatus"], {}, "2007-04-22 00:00:00") - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] def test_heartBeatLogging(jobDB): res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup", "someSetup") - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] jobID = res["JobID"] res = jobDB.setJobStatus(jobID, status=JobStatus.CHECKING) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] res = jobDB.setJobStatus(jobID, status=JobStatus.WAITING) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] res = jobDB.setJobStatus(jobID, status=JobStatus.MATCHED) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] res = jobDB.setJobStatus(jobID, status=JobStatus.RUNNING) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] res = jobDB.setHeartBeatData(jobID, dynamicDataDict={"CPU": 2345}) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] res = jobDB.setHeartBeatData(jobID, dynamicDataDict={"Memory": 5555}) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] res = jobDB.getHeartBeatData(jobID) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] assert len(res["Value"]) == 2, str(res) for name, value, _hbt in res["Value"]: @@ -254,168 +299,168 @@ def test_heartBeatLogging(jobDB): assert False, "Unknown entry: %s: %s" % (name, value) res = jobDB.setJobStatus(jobID, status=JobStatus.DONE) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] tomorrow = datetime.today() + timedelta(1) delTime = datetime.strftime(tomorrow, "%Y-%m-%d") res = jobDB.removeInfoFromHeartBeatLogging(status=JobStatus.DONE, delTime=delTime, maxLines=100) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] res = jobDB.getHeartBeatData(jobID) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] assert not res["Value"], str(res) -def test_jobParameters(jobDB): +def test_getJobParameters(jobDB): res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup", "someSetup") - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] jobID = res["JobID"] res = jobDB.getJobParameters(jobID) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] assert res["Value"] == {}, res["Value"] res = jobDB.getJobParameters([jobID]) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] assert res["Value"] == {}, res["Value"] res = jobDB.getJobParameters(jobID, "not") - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] assert res["Value"] == {}, res["Value"] def test_setJobsMajorStatus(jobDB): - res = jobDB.insertNewJobIntoDB(jdl, "owner_1", "/DN/OF/owner", "ownerGroup", "someSetup") - assert res["OK"] is True, res["Message"] + res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup", "someSetup") + assert res["OK"], res["Message"] jobID_1 = res["JobID"] - res = jobDB.insertNewJobIntoDB(jdl, "owner_2", "/DN/OF/owner", "ownerGroup", "someSetup") - assert res["OK"] is True, res["Message"] + res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup", "someSetup") + assert res["OK"], res["Message"] jobID_2 = res["JobID"] res = jobDB.getJobAttribute(jobID_1, "Status") - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] assert res["Value"] == JobStatus.RECEIVED res = jobDB.getJobAttribute(jobID_2, "Status") - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] assert res["Value"] == JobStatus.RECEIVED res = jobDB.setJobsMajorStatus([jobID_1, jobID_2], JobStatus.CHECKING) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] res = jobDB.getJobsAttributes([jobID_1, jobID_2], ["Status"]) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] assert res["Value"] == {jobID_1: {"Status": JobStatus.CHECKING}, jobID_2: {"Status": JobStatus.CHECKING}} res = jobDB.setJobsMajorStatus([jobID_1, jobID_2], JobStatus.RUNNING) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] res = jobDB.getJobsAttributes([jobID_1, jobID_2], ["Status"]) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] assert res["Value"] == {jobID_1: {"Status": JobStatus.CHECKING}, jobID_2: {"Status": JobStatus.CHECKING}} res = jobDB.setJobsMajorStatus([jobID_1], JobStatus.WAITING) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] res = jobDB.getJobsAttributes([jobID_1, jobID_2], ["Status"]) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] assert res["Value"] == {jobID_1: {"Status": JobStatus.WAITING}, jobID_2: {"Status": JobStatus.CHECKING}} res = jobDB.setJobsMajorStatus([jobID_1], JobStatus.KILLED) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] res = jobDB.getJobsAttributes([jobID_1, jobID_2], ["Status"]) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] assert res["Value"] == {jobID_1: {"Status": JobStatus.WAITING}, jobID_2: {"Status": JobStatus.CHECKING}} res = jobDB.setJobsMajorStatus([jobID_1], JobStatus.KILLED, force=True) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] res = jobDB.getJobsAttributes([jobID_1, jobID_2], ["Status"]) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] assert res["Value"] == {jobID_1: {"Status": JobStatus.KILLED}, jobID_2: {"Status": JobStatus.CHECKING}} def test_attributes(jobDB): - res = jobDB.insertNewJobIntoDB(jdl, "owner_1", "/DN/OF/owner", "ownerGroup", "someSetup") - assert res["OK"] is True, res["Message"] + res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup", "someSetup") + assert res["OK"], res["Message"] jobID_1 = res["JobID"] - res = jobDB.insertNewJobIntoDB(jdl, "owner_2", "/DN/OF/owner", "ownerGroup", "someSetup") - assert res["OK"] is True, res["Message"] + res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup", "someSetup") + assert res["OK"], res["Message"] jobID_2 = res["JobID"] res = jobDB.getJobAttribute(jobID_1, "Status") - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] assert res["Value"] == JobStatus.RECEIVED res = jobDB.getJobAttribute(jobID_2, "Status") - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] assert res["Value"] == JobStatus.RECEIVED res = jobDB.getJobsAttributes([jobID_1, jobID_2], ["Status"]) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] assert res["Value"] == {jobID_1: {"Status": JobStatus.RECEIVED}, jobID_2: {"Status": JobStatus.RECEIVED}} res = jobDB.setJobAttributes(jobID_1, ["ApplicationStatus"], ["ApplicationStatus_1"], True) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] res = jobDB.getJobAttribute(jobID_1, "ApplicationStatus") - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] assert res["Value"] == "ApplicationStatus_1" res = jobDB.setJobAttributes(jobID_1, ["ApplicationStatus"], ["ApplicationStatus_1_2"], True) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] res = jobDB.getJobAttribute(jobID_1, "ApplicationStatus") - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] assert res["Value"] == "ApplicationStatus_1_2" res = jobDB.setJobAttributes(jobID_1, ["JobName", "Site"], ["JobName_1", "DIRAC.Client.org"], True) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] res = jobDB.getJobAttribute(jobID_1, "Site") - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] assert res["Value"] == "DIRAC.Client.org" res = jobDB.setJobAttributes(jobID_1, ["Status"], [JobStatus.CHECKING], True) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] res = jobDB.setJobAttributes(jobID_1, ["Status"], [JobStatus.WAITING], True) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] res = jobDB.getJobAttribute(jobID_1, "Status") - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] assert res["Value"] == JobStatus.WAITING res = jobDB.setJobAttributes(jobID_1, ["Status", "MinorStatus"], [JobStatus.MATCHED, "minor"], True) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] res = jobDB.getJobAttributes(jobID_1, ["Status", "MinorStatus"]) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] assert res["Value"]["Status"] == JobStatus.MATCHED assert res["Value"]["MinorStatus"] == "minor" res = jobDB.getJobAttributes(jobID_2, ["Status"]) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] assert res["Value"]["Status"] == JobStatus.RECEIVED res = jobDB.getJobsAttributes([jobID_1, jobID_2], ["Status"]) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] assert res["Value"] == {jobID_1: {"Status": JobStatus.MATCHED}, jobID_2: {"Status": JobStatus.RECEIVED}} res = jobDB.setJobAttributes(jobID_2, ["Status"], [JobStatus.CHECKING], True) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] res = jobDB.setJobAttributes(jobID_2, ["Status"], [JobStatus.WAITING], True) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] res = jobDB.setJobAttributes(jobID_2, ["Status"], [JobStatus.MATCHED], True) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] res = jobDB.setJobAttributes([jobID_1, jobID_2], ["Status", "MinorStatus"], [JobStatus.RUNNING, "minor_2"], True) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] res = jobDB.getJobAttributes(jobID_1, ["Status", "MinorStatus"]) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] assert res["Value"]["Status"] == JobStatus.RUNNING assert res["Value"]["MinorStatus"] == "minor_2" res = jobDB.getJobAttributes(jobID_2, ["Status", "MinorStatus"]) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] assert res["Value"]["Status"] == JobStatus.RUNNING assert res["Value"]["MinorStatus"] == "minor_2" jobDB.setJobAttributes(jobID_1, ["Status"], [JobStatus.DONE], True) jobDB.setJobAttributes([jobID_1, jobID_2], ["Status", "MinorStatus"], [JobStatus.COMPLETED, "minor_3"], True) res = jobDB.getJobAttributes(jobID_1, ["Status", "MinorStatus"]) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] assert res["Value"]["Status"] == JobStatus.DONE assert res["Value"]["MinorStatus"] == "minor_3" res = jobDB.getJobAttributes(jobID_2, ["Status", "MinorStatus"]) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] assert res["Value"]["Status"] == JobStatus.RUNNING assert res["Value"]["MinorStatus"] == "minor_3" res = jobDB.getJobsAttributes([jobID_1, jobID_2], ["Status"]) - assert res["OK"] is True, res["Message"] + assert res["OK"], res["Message"] assert res["Value"] == {jobID_1: {"Status": JobStatus.DONE}, jobID_2: {"Status": JobStatus.RUNNING}} From c47ce83a56f8e1f5c0e331d8843c65a7ddff6046 Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Thu, 18 Aug 2022 11:08:47 +0200 Subject: [PATCH 0004/1135] feat: Use Python's default SSL ciphers by default --- src/DIRAC/Core/Tornado/Client/private/TornadoBaseClient.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/DIRAC/Core/Tornado/Client/private/TornadoBaseClient.py b/src/DIRAC/Core/Tornado/Client/private/TornadoBaseClient.py index 28c4bde81ef..f8e3c29d573 100644 --- a/src/DIRAC/Core/Tornado/Client/private/TornadoBaseClient.py +++ b/src/DIRAC/Core/Tornado/Client/private/TornadoBaseClient.py @@ -651,10 +651,8 @@ def init_poolmanager(self, *args, **kwargs): @convertToReturnValue def _create_session(verified=True): ctx = ssl.create_default_context() - # Python 3.10+ sets DEFAULT:@SECLEVEL=2 which prevents the use of 1024 bit RSA for proxies. - # In DIRAC 8.0 the default proxy length has been increased to 2048 bits however we need to - # downgrade to DEFAULT:@SECLEVEL=1 until all users have uploaded a new proxy. - ctx.set_ciphers(os.environ.get("DIRAC_HTTPS_SSL_CIPHERS", "DEFAULT:@SECLEVEL=1")) + if ssl_ciphers := os.environ.get("DIRAC_HTTPS_SSL_CIPHERS"): + ctx.set_ciphers(ssl_ciphers) if minimum_tls_version := os.environ.get("DIRAC_HTTPS_SSL_METHOD_MIN"): ctx.minimum_version = getattr(ssl.TLSVersion, minimum_tls_version) if maximum_tls_version := os.environ.get("DIRAC_HTTPS_SSL_METHOD_MAX"): From b7fd5310cfac82e63edbfd68ce714126c68255f9 Mon Sep 17 00:00:00 2001 From: chaen Date: Fri, 16 Sep 2022 10:52:38 +0200 Subject: [PATCH 0005/1135] sweep: #6365 docs: document input data resolution --- dirac.cfg | 30 ++++ .../InputDataResolution.rst | 35 ++++ .../Systems/WorkloadManagement/index.rst | 1 + .../UserJobs/JDLReference/index.rst | 154 +++++++++--------- 4 files changed, 145 insertions(+), 75 deletions(-) create mode 100644 docs/source/AdministratorGuide/Systems/WorkloadManagement/InputDataResolution.rst diff --git a/dirac.cfg b/dirac.cfg index 46475fb12a3..1247730d4f9 100644 --- a/dirac.cfg +++ b/dirac.cfg @@ -930,6 +930,36 @@ Operations } } } + # Specify how job access their data + # None of these fields is mandatory + # See https://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/WorkloadManagement/InputDataResolution.html + InputDataPolicy + { + # Default policy + Default = DIRAC.WorkloadManagementSystem.Client.InputDataByProtocol + # A job running at CERN would stream the data + LCG.CERN.cern = DIRAC.WorkloadManagementSystem.Client.InputDataByProtocol + # A job running at GRIDKA would download the files on the WN + LCG.GRIDKA.de = DIRAC.WorkloadManagementSystem.Client.DownloadInputData + # Shortcut for the JobAPI: job.setInputDataPolicy('Download') + Download = DIRAC.WorkloadManagementSystem.Client.DownloadInputData + # Shortcut for the JobAPI: job.setInputDataPolicy('Protocol') + Protocol = DIRAC.WorkloadManagementSystem.Client.InputDataByProtocol + # Used to limit or not the replicas considered by a Job in case of streaming + # See src/DIRAC/WorkloadManagementSystem/Client/InputDataByProtocol.py + AllReplicas = True + # List of protocols to use for streaming + Protocols + { + # This list is used if the we are getting a file from a + # StorageElement local to the site we are running on + Local = file, xroot, root + # This list is used if the SE is not local + Remote = xroot, root + } + # Module used for InputData resolution if not specified in the JDL + InputDataModule = DIRAC.Core.Utilities.InputDataResolution + } # Options for the pilot3 # See https://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/WorkloadManagement/Pilots/Pilots3.html Pilot diff --git a/docs/source/AdministratorGuide/Systems/WorkloadManagement/InputDataResolution.rst b/docs/source/AdministratorGuide/Systems/WorkloadManagement/InputDataResolution.rst new file mode 100644 index 00000000000..8b9ffa1cb9c --- /dev/null +++ b/docs/source/AdministratorGuide/Systems/WorkloadManagement/InputDataResolution.rst @@ -0,0 +1,35 @@ +.. _InputDataResolution: + +================================================== +InputDataResolution: giving job access to the data +================================================== + +When a job needs access to data, there are two ways data can be accessed: + +* either by downloading the file on the local worker node +* or by reading the data remotely, aka ``streaming``. + +The resolution is done in the ``JobWrapper`` (see :ref:`jobs`). By default, the resolution logic is implemented in :py:mod:`~DIRAC.WorkloadManagementSystem.Client.InputDataResolution`. It can be overwritten by the Job JDL (see ``InputDataModule`` in :ref:`jdlDescription`), or by the ``/Operations/<>/InputDataPolicy/InputDataModule`` parameter. + + +You can look into this class for more details, but to summarize: + +* it will look into the ``job`` JDL if it can find ``InputDataPolicy`` option. If so, it will use that as the module. +* If not, it will check whether a policy is defined for the site we are running on (in ``/Operations/InputDataPolicy/``). +* If not, it will run the default policy specified in ``/Operations/InputDataPolicy/Default`` + +The ``InputDataPolicy`` parameter can either be set directly in the JDL, in which case it should be a full module, or it can be set using the ``Job`` class (see :py:meth:`~DIRAC.Interfaces.API.Job.Job.setInputDataPolicy`) + +DownloadInputData +================= + +This module will download the files locally on the worker node for processing. + +See :py:mod:`~DIRAC.WorkloadManagementSystem.Client.DownloadInputData` for details. + +InputDataByProtocol +=================== + +This module will generate the URLs necessary to access the files remotely. + +See :py:mod:`~DIRAC.WorkloadManagementSystem.Client.InputDataByProtocol` for details. diff --git a/docs/source/AdministratorGuide/Systems/WorkloadManagement/index.rst b/docs/source/AdministratorGuide/Systems/WorkloadManagement/index.rst index 22d1e0bdbcf..5ceaa110b52 100644 --- a/docs/source/AdministratorGuide/Systems/WorkloadManagement/index.rst +++ b/docs/source/AdministratorGuide/Systems/WorkloadManagement/index.rst @@ -90,3 +90,4 @@ The following sections add some detail for the WMS systems. tagsAndJobs multiProcessorJobs VMDIRAC + InputDataResolution diff --git a/docs/source/UserGuide/GettingStarted/UserJobs/JDLReference/index.rst b/docs/source/UserGuide/GettingStarted/UserJobs/JDLReference/index.rst index 7c98925fa55..c0301b624ab 100644 --- a/docs/source/UserGuide/GettingStarted/UserJobs/JDLReference/index.rst +++ b/docs/source/UserGuide/GettingStarted/UserJobs/JDLReference/index.rst @@ -6,84 +6,88 @@ .. role:: subtitle +.. _jdlDescription: + ========================================= Job Description Language Reference ========================================= In this section all the attributes that can be used in the DIRAC JDL job descriptions are presented. -+---------------------+---------------------------------------------+-----------------------------------------------+ -| | -| :subtitle:`The basic JDL parameters` | -| | -| These are the parameters giving the basic job description | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| **Attribute Name** | **Description** | **Example** | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *Executable* | Name of the executable file | Executable = ``"/bin/ls";`` | -| | | | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *Arguments* | String of arguments for the job | Arguments = ``"-ltr";`` | -| | executable | | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *StdError* | Name of the file to get the standard error | StdError = ``"std.err";`` | -| | stream of the user application | | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *StdOutput* | Name of the file to get the standard output | StdOutput = ``"std.out";`` | -| | stream of the user application | | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *InputSandbox* | A list of input sandbox files | InputSandbox = ``{"jobScript.sh"};`` | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *OutputSandbox* | A list of output sandbox files | OutputSandbox = ``{"std.err","std.out"};`` | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| | -| :subtitle:`Job Requirements` | -| | -| These parameters are interpreted as job requirements | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| **Attribute Name** | **Description** | **Example** | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *CPUTime* | Max CPU time required by the job in | CPUTime = 18000; | -| | HEPSPEC06 seconds | | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *Site* | Job destination site | Site = ``{"EGI.CPPM.fr"};`` | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *BannedSites* | Sites where the job must not go | BannedSites = ``{"EGI.LAPP.fr"};`` | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *GridCE* | Job destination CE | GridCE = ``{"some.ce.lapp.fr"};`` | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *Platform* | Target Operating System | Platform = ``"Linux_x86_64_glibc-2.17";`` | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| | -| :subtitle:`Data` | -| | -| Describing job data | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| **Attribute Name** | **Description** | **Example** | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *InputData* | Job input data files | InputData = ``{"/dirac/user/a/atsareg/data1", | -| | | "/dirac/user/a/atsareg/data1"};`` | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *InputDataPolicy* | Job input data policy | InputDataPolicy = ``"Download";`` | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *OutputData* | Job output data files | OutputData = ``{"output1","output2"};`` | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *OutputPath* | The output data path in the File Catalog | OutputPath = ``{"/myjobs/output"};`` | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *OutputSE* | The output data Storage Element | OutputSE = ``{"DIRAC-USER"};`` | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| | -| :subtitle:`Parametric Jobs` | -| | -| Bulk submission parameters | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| **Attribute Name** | **Description** | **Example** | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *Parameters* | Number of parameters or a list of values | Parameters = 10; | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *ParameterStart* | Value of the first parameter | ParameterStart = 0.; | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *ParameterStep* | Parameter increment | ParameterStep = 0.1; (default 0.) | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *ParameterFactor* | Parameter multiplier | ParameterFactor = 1.1; (default 1.) | -+---------------------+---------------------------------------------+-----------------------------------------------+ ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| | +| :subtitle:`The basic JDL parameters` | +| | +| These are the parameters giving the basic job description | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| **Attribute Name** | **Description** | **Example** | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *Executable* | Name of the executable file | Executable = ``"/bin/ls";`` | +| | | | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *Arguments* | String of arguments for the job | Arguments = ``"-ltr";`` | +| | executable | | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *StdError* | Name of the file to get the standard error | StdError = ``"std.err";`` | +| | stream of the user application | | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *StdOutput* | Name of the file to get the standard output | StdOutput = ``"std.out";`` | +| | stream of the user application | | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *InputSandbox* | A list of input sandbox files | InputSandbox = ``{"jobScript.sh"};`` | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *OutputSandbox* | A list of output sandbox files | OutputSandbox = ``{"std.err","std.out"};`` | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| | +| :subtitle:`Job Requirements` | +| | +| These parameters are interpreted as job requirements | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| **Attribute Name** | **Description** | **Example** | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *CPUTime* | Max CPU time required by the job in | CPUTime = 18000; | +| | HEPSPEC06 seconds | | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *Site* | Job destination site | Site = ``{"EGI.CPPM.fr"};`` | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *BannedSites* | Sites where the job must not go | BannedSites = ``{"EGI.LAPP.fr"};`` | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *GridCE* | Job destination CE | GridCE = ``{"some.ce.lapp.fr"};`` | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *Platform* | Target Operating System | Platform = ``"Linux_x86_64_glibc-2.17";`` | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| | +| :subtitle:`Data` | +| | +| Describing job data | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| **Attribute Name** | **Description** | **Example** | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *InputData* | Job input data files | InputData = ``{"/dirac/user/a/atsareg/data1", | +| | | "/dirac/user/a/atsareg/data1"};`` | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *InputDataModule* | Job input data module | InputDataModule = ``"DIRAC.WorkloadManagementSystem.Client.InputDataResolution"`` | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *InputDataPolicy* | Job input data policy | InputDataPolicy = ``"DIRAC.WorkloadManagementSystem.Client.DownloadInputData";`` | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *OutputData* | Job output data files | OutputData = ``{"output1","output2"};`` | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *OutputPath* | The output data path in the File Catalog | OutputPath = ``{"/myjobs/output"};`` | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *OutputSE* | The output data Storage Element | OutputSE = ``{"DIRAC-USER"};`` | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| | +| :subtitle:`Parametric Jobs` | +| | +| Bulk submission parameters | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| **Attribute Name** | **Description** | **Example** | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *Parameters* | Number of parameters or a list of values | Parameters = 10; | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *ParameterStart* | Value of the first parameter | ParameterStart = 0.; | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *ParameterStep* | Parameter increment | ParameterStep = 0.1; (default 0.) | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *ParameterFactor* | Parameter multiplier | ParameterFactor = 1.1; (default 1.) | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ From 03fc6c5550e3c70137111503bf6e2e50d269b66c Mon Sep 17 00:00:00 2001 From: chaen Date: Wed, 21 Sep 2022 07:28:36 +0200 Subject: [PATCH 0006/1135] sweep: #6376 [v7r3]fix (SE): free space can be negative, so set it to 0 --- src/DIRAC/Resources/Storage/StorageElement.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/DIRAC/Resources/Storage/StorageElement.py b/src/DIRAC/Resources/Storage/StorageElement.py index e240e06f0cd..2cbe913c413 100755 --- a/src/DIRAC/Resources/Storage/StorageElement.py +++ b/src/DIRAC/Resources/Storage/StorageElement.py @@ -489,6 +489,12 @@ def checkOccupancy(self, occupancyDict, unit): log.error(msg) return S_ERROR(msg) + # It can happen that Used space > total space (quota enforcement on EOS are async) + # In that case, just set it to 0, and issue a warning + if occupancyDict["Free"] < 0: + log.warn("Negative free value in occupancy dict", str(occupancyDict["Free"])) + occupancyDict["Free"] = 0 + # Since plugins return Bytes, we do not need to convert if that's what we want if unit != "B": for space in ["Total", "Free"]: From 87b6900713e0549cd4861e5568388d9edc14afad Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Wed, 21 Sep 2022 07:32:34 +0200 Subject: [PATCH 0007/1135] sweep: #6375 Update dependencies in setup.cfg --- environment.yml | 1 - setup.cfg | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/environment.yml b/environment.yml index c69c864971e..f6cf3e7feb7 100644 --- a/environment.yml +++ b/environment.yml @@ -22,7 +22,6 @@ dependencies: - opensearch-py - opensearch-dsl - fts3 - - future - gitpython >=2.1.0 - m2crypto >=0.36,!=0.38.0 - matplotlib diff --git a/setup.cfg b/setup.cfg index 49470ac465f..3e58a8866d6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,8 +33,8 @@ install_requires = cachetools certifi diraccfg + db12 fts3 - future gfal2-python importlib_metadata >=4.4 M2Crypto >=0.36 From 5e74da2b9db92871bb3a94139b4ec1109295c70a Mon Sep 17 00:00:00 2001 From: chaen Date: Mon, 26 Sep 2022 12:22:12 +0200 Subject: [PATCH 0008/1135] sweep: #6382 cancel the FTS transfers when an RMS request is canceled --- .../DataManagementSystem/Agent/FTS3Agent.py | 16 +++++++++++++- .../DataManagementSystem/Client/FTS3Job.py | 22 +++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/src/DIRAC/DataManagementSystem/Agent/FTS3Agent.py b/src/DIRAC/DataManagementSystem/Agent/FTS3Agent.py index 954a4ed166c..ac32b8822a4 100644 --- a/src/DIRAC/DataManagementSystem/Agent/FTS3Agent.py +++ b/src/DIRAC/DataManagementSystem/Agent/FTS3Agent.py @@ -415,6 +415,20 @@ def _treatOperation(self, operation): operation.status = "Canceled" continueOperationProcessing = False + log.info("Canceling the associated FTS3 jobs") + + for ftsJob in operation.ftsJobs: + res = self.getFTS3Context( + ftsJob.username, ftsJob.userGroup, ftsJob.ftsServer, threadID=threadID + ) + if not res["OK"]: + log.error("Error getting context", res) + continue + + context = res["Value"] + # We ignore the return on purpose + ftsJob.cancel(context) + if continueOperationProcessing: res = operation.prepareNewJobs( @@ -468,7 +482,7 @@ def _treatOperation(self, operation): % (operation.operationID, len(submittedFileIds)) ) - # new jobs are put in the DB at the same time + # new jobs are put in the DB at the same time res = self.fts3db.persistOperation(operation) if not res["OK"]: diff --git a/src/DIRAC/DataManagementSystem/Client/FTS3Job.py b/src/DIRAC/DataManagementSystem/Client/FTS3Job.py index 69b8886b2b5..5606bf1bca7 100644 --- a/src/DIRAC/DataManagementSystem/Client/FTS3Job.py +++ b/src/DIRAC/DataManagementSystem/Client/FTS3Job.py @@ -218,6 +218,28 @@ def monitor(self, context=None, ftsServer=None, ucert=None): return S_OK(filesStatus) + def cancel(self, context): + """Cancel the job on the FTS server. Note that it will cancel all the files. + See https://fts3-docs.web.cern.ch/fts3-docs/fts-rest/docs/api.html#delete-jobsjobidlist + for behavior details + """ + + try: + cancelDict = fts3.cancel(context, self.ftsGUID) + newStatus = cancelDict["job_state"].capitalize() + # If the status is already canceled + # (for a reason or another, don't change the error message) + # If the new status is Canceled, set it, and update the reason + if newStatus == "Canceled" and self.status != "Canceled": + self.status = "Canceled" + self.error = "Matching RMS Request was canceled" + return S_OK() + # The job is not found + except NotFound: + return S_ERROR(errno.ESRCH, "FTSGUID %s not found on %s" % (self.ftsGUID, self.ftsServer)) + except FTS3ClientException as e: + return S_ERROR("Error canceling the job %s" % e) + @staticmethod def __fetchSpaceToken(seName, vo): """Fetch the space token of storage element From c2435e1aa9d7c9c5ed0c543bc1ad8799e03d96fd Mon Sep 17 00:00:00 2001 From: Chris Burr Date: Mon, 26 Sep 2022 16:54:40 +0200 Subject: [PATCH 0009/1135] sweep: #6377 Fix deployment of 8.0.x and add protections against deploying incorrect versions --- .github/workflows/deployment.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index e3a15079fb1..09f975c5ece 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -62,18 +62,29 @@ jobs: run: | set -xeuo pipefail IFS=$'\n\t' + # Only do a real release for workflow_dispatch events from DIRACGrid/DIRAC for integration for Python 3 compatible branches if [[ "${{ github.repository }}" == "DIRACGrid/DIRAC" ]]; then if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then - if [[ "${{ github.event.ref }}" =~ ^refs/heads/(integration|rel-v([8-9]|[1-9][0-9]+)\.[0-9]+)$ ]]; then + if [[ "${{ github.event.ref }}" =~ ^refs/heads/(integration|rel-v([8-9]|[1-9][0-9]+)r[0-9]+)$ ]]; then echo "Will create a real release" export NEW_VERSION="v${{ github.event.inputs.version }}" if [[ "${NEW_VERSION}" == "v" ]]; then + # If version wasn't given as an input to the workflow, use setuptools_scm to guess while removing "dev" portion of the version number NEW_VERSION=v$(python -m setuptools_scm | sed 's@Guessed Version @@g' | sed -E 's@(\.dev|\+g).+@@g') export NEW_VERSION fi echo "Release will be named $NEW_VERSION" # Validate the version + # Ensure the version doesn't look like a PEP-440 "dev release" (which might happen if the automatic version bump has issues) python -c $'from packaging.version import Version; v = Version('"'$NEW_VERSION'"$')\nif v.is_devrelease:\n raise ValueError(v)' + if [[ "${{ github.event.ref }}" =~ ^refs/heads/integration$ ]]; then + # If we're releasing from integration it must be a pre-release + python -c $'from packaging.version import Version; v = Version('"'$NEW_VERSION'"$')\nif not v.is_prerelease:\n raise ValueError("integration should only be used for pre-releases")' + elif [[ "${{ github.event.ref }}" != "$(python -c $'from packaging.version import Version; v = Version('"'$NEW_VERSION'"$')\nprint(f"refs/heads/rel-v{v.major}r{v.minor}")')" ]]; then + # If we're not releasing from integration the version should match the rel-vXrY branch name + echo "$NEW_VERSION is an invalid version for ${{ github.event.ref }}" + exit 2 + fi # Commit the release notes mv release.notes release.notes.old { From 0c377a60465b81761573754ee89af39cdcfa1dce Mon Sep 17 00:00:00 2001 From: chaen Date: Mon, 26 Sep 2022 16:58:40 +0200 Subject: [PATCH 0010/1135] sweep: #6364 Fix data monitoring and introduce FailedDataOperation monitoring type --- .../Systems/MonitoringSystem/index.rst | 4 +- .../Client/DataOperationSender.py | 54 ++++++++++++++----- .../Client/Types/FailedDataOperation.py | 43 +++++++++++++++ src/DIRAC/Resources/Storage/StorageElement.py | 38 ++++++++++--- 4 files changed, 119 insertions(+), 20 deletions(-) create mode 100644 src/DIRAC/MonitoringSystem/Client/Types/FailedDataOperation.py diff --git a/docs/source/AdministratorGuide/Systems/MonitoringSystem/index.rst b/docs/source/AdministratorGuide/Systems/MonitoringSystem/index.rst index 2588e7850b6..f6301b09b1f 100644 --- a/docs/source/AdministratorGuide/Systems/MonitoringSystem/index.rst +++ b/docs/source/AdministratorGuide/Systems/MonitoringSystem/index.rst @@ -18,7 +18,7 @@ The Monitoring system is used to monitor various components of DIRAC. Currently, - Service Monitoring: for monitoring the activity of DIRAC services. - RMS Monitoring: for monitoring the DIRAC RequestManagement System (mostly the Request Executing Agent). - PilotSubmission Monitoring: for monitoring the DIRAC pilot submission statistics from SiteDirector agents. - - DataOperation Monitoring: for monitoring the DIRAC data operation statistics. + - DataOperation Monitoring: for monitoring the DIRAC data operation statistics as well as individual failures from interactive use of ``StorageElement``. It is based on Elasticsearch distributed search and analytics NoSQL database. If you want to use it, you have to install the Monitoring service, and of course connect to a ElasticSearch instance. @@ -161,6 +161,8 @@ Data Operation Monitoring This monitoring enables the reporting of information about the data operation such as the cumulative transfer size or the number of succeded and failed transfers. +It will also fill an index called ``faileddataoperation_index`` containing entries for individual interactive failures (CLI, Job, etc). + Accessing the Monitoring information ===================================== diff --git a/src/DIRAC/MonitoringSystem/Client/DataOperationSender.py b/src/DIRAC/MonitoringSystem/Client/DataOperationSender.py index b3816719b9a..2c0c57e5505 100644 --- a/src/DIRAC/MonitoringSystem/Client/DataOperationSender.py +++ b/src/DIRAC/MonitoringSystem/Client/DataOperationSender.py @@ -26,12 +26,12 @@ class DataOperationSender: # Initialize the object so that the Reporters are created only once def __init__(self): - monitoringType = "DataOperation" # Will use the `MonitoringBackends/Default` value # as monitoring backend unless a flag for `MonitoringBackends/DataOperation` is set. - self.monitoringOptions = Operations().getMonitoringBackends(monitoringType) + self.monitoringOptions = Operations().getMonitoringBackends("DataOperation") if "Monitoring" in self.monitoringOptions: - self.dataOperationReporter = MonitoringReporter(monitoringType) + self.dataOperationReporter = MonitoringReporter("DataOperation") + self.failedDataOperationReporter = MonitoringReporter("FailedDataOperation") if "Accounting" in self.monitoringOptions: self.dataOp = DataOperation() @@ -41,16 +41,27 @@ def __init__(self): self._sendDataMethods.append(getattr(self, f"_sendData{backend}")) self._commitMethods.append(getattr(self, f"_commit{backend}")) - def _sendDataMonitoring(self, baseDict, commitFlag=False, delayedCommit=False, startTime=False, endTime=False): + def _sendDataMonitoring( + self, baseDict, commitFlag=False, delayedCommit=False, startTime=False, endTime=False, failedRecords=None + ): """Send the data to the monitoring system""" - # Some fields added here are not known to the Accounting, so we have to make a copy - monitoringDict = copy.deepcopy(baseDict) - monitoringDict["Channel"] = monitoringDict["Source"] + "->" + monitoringDict["Destination"] + # Since we are adding elements that the accounting + # may not like, work on a copy + baseDict = copy.copy(baseDict) + baseDict["Channel"] = baseDict["Source"] + "->" + baseDict["Destination"] # Add timestamp if not already added - if "timestamp" not in monitoringDict: - monitoringDict["timestamp"] = int(toEpochMilliSeconds()) - self.dataOperationReporter.addRecord(monitoringDict) + if "timestamp" not in baseDict: + baseDict["timestamp"] = int(toEpochMilliSeconds()) + self.dataOperationReporter.addRecord(baseDict) + + # If there were failedRecords, send them right away + if failedRecords: + for failedRec in failedRecords: + self.failedDataOperationReporter.addRecord(failedRec) + + self.failedDataOperationReporter.commit() + if commitFlag: result = self.dataOperationReporter.commit() sLog.debug("Committing data operation to monitoring") @@ -59,11 +70,18 @@ def _sendDataMonitoring(self, baseDict, commitFlag=False, delayedCommit=False, s else: sLog.debug("Done committing to monitoring") return result + return S_OK() @convertToReturnValue - def _sendDataAccounting(self, baseDict, commitFlag=False, delayedCommit=False, startTime=False, endTime=False): + def _sendDataAccounting( + self, baseDict, commitFlag=False, delayedCommit=False, startTime=False, endTime=False, failedRecords=None + ): """Send the data to the accounting system""" + + # Only work with the keys we know about + baseDict = {key: baseDict[key] for key in self.dataOp.fieldsList if key in baseDict} + returnValueOrRaise(self.dataOp.setValuesFromDict(baseDict)) if startTime: @@ -93,7 +111,9 @@ def _sendDataAccounting(self, baseDict, commitFlag=False, delayedCommit=False, s sLog.error("Could not delay-commit data operation to accounting") return result - def sendData(self, baseDict, commitFlag=False, delayedCommit=False, startTime=False, endTime=False): + def sendData( + self, baseDict, commitFlag=False, delayedCommit=False, startTime=False, endTime=False, failedRecords=None + ): """ Sends the input to Monitoring or Accounting based on the monitoringOptions @@ -102,14 +122,22 @@ def sendData(self, baseDict, commitFlag=False, delayedCommit=False, startTime=Fa :param bool delayedCommit: decides whether to commit the record with delay (only for sending to Accounting) :param int startTime: epoch time, start time of the plot :param int endTime: epoch time, end time of the plot + :param list failedRecords: list of records for the failed operation """ baseDict["ExecutionSite"] = DIRAC.siteName() # Send data and commit prioritizing the first monitoring option in the list for methId, _sendDataMeth in enumerate(self._sendDataMethods): + # Some fields added here are not known to the Accounting, so we have to make a copy + # of the baseDict res = _sendDataMeth( - baseDict, commitFlag=commitFlag, delayedCommit=delayedCommit, startTime=startTime, endTime=endTime + baseDict, + commitFlag=commitFlag, + delayedCommit=delayedCommit, + startTime=startTime, + endTime=endTime, + failedRecords=failedRecords, ) if not res["OK"]: sLog.error("DataOperationSender.sendData: could not send data", f"{res}") diff --git a/src/DIRAC/MonitoringSystem/Client/Types/FailedDataOperation.py b/src/DIRAC/MonitoringSystem/Client/Types/FailedDataOperation.py new file mode 100644 index 00000000000..cc2b1175a2b --- /dev/null +++ b/src/DIRAC/MonitoringSystem/Client/Types/FailedDataOperation.py @@ -0,0 +1,43 @@ +""" +Monitoring Type for storing individual failures of data operations on LFN +""" + +from DIRAC.MonitoringSystem.Client.Types.BaseType import BaseType + + +class FailedDataOperation(BaseType): + def __init__(self): + + super().__init__() + + self.keyFields = [ + "LFN", # LFN of the file with the failure + "URL", # URL used in the operation + "OperationType", # Type of the operation (getFile, putFile, etc) + "User", # Identity used + "ExecutionSite", # Where the operation was executed + "TargetSE", # Name of the SE that was targeted + "Protocol", # Protocol used + "Error", # Error message + "Component", # Which component sent the error (StorageElement, Stager, etc) + ] + + self.monitoringFields = ["Errno"] # if self.monitoringFields is empty, MonitoringDB complains + + self.index = "faileddataoperation_index" + + self.addMapping( + { + "LFN": {"type": "keyword"}, + "URL": {"type": "keyword"}, + "OperationType": {"type": "keyword"}, + "User": {"type": "keyword"}, + "ExecutionSite": {"type": "keyword"}, + "TargetSE": {"type": "keyword"}, + "Protocol": {"type": "keyword"}, + "Error": {"type": "text"}, + "Component": {"type": "keyword"}, + } + ) + + self.checkType() diff --git a/src/DIRAC/Resources/Storage/StorageElement.py b/src/DIRAC/Resources/Storage/StorageElement.py index 2cbe913c413..67fcd402c5c 100755 --- a/src/DIRAC/Resources/Storage/StorageElement.py +++ b/src/DIRAC/Resources/Storage/StorageElement.py @@ -21,6 +21,7 @@ from DIRAC.Core.Utilities.File import convertSizeUnits from DIRAC.Core.Utilities.List import getIndexInList from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR, returnSingleResult +from DIRAC.Core.Utilities.TimeUtilities import toEpochMilliSeconds from DIRAC.Resources.Storage.StorageFactory import StorageFactory from DIRAC.Core.Utilities.Pfn import pfnparse from DIRAC.Core.Utilities.SiteSEMapping import getSEsForSite @@ -1295,7 +1296,7 @@ def __executeMethod(self, lfn, *args, **kwargs): res = fcn(urlsToUse, *args, **kwargs) elapsedTime = time.time() - startTime - self.addAccountingOperation(urlsToUse, startDate, elapsedTime, storageParameters, res) + self.addAccountingOperation(urlDict, startDate, elapsedTime, storageParameters, res) if not res["OK"]: errStr = "Completely failed to perform %s." % self.methodName @@ -1343,11 +1344,11 @@ def __getattr__(self, name): raise AttributeError("StorageElement does not have a method '%s'" % name) - def addAccountingOperation(self, lfns, startDate, elapsedTime, storageParameters, callRes): + def addAccountingOperation(self, urlDict, startDate, elapsedTime, storageParameters, callRes): """ Generates a DataOperationSender instance and sends the operation data filled in accountingDict. - :param lfns: list of lfns on which we attempted the operation + :param urlDict: {url:lfn} on which we attempted the operation :param startDate: datetime, start of the operation :param elapsedTime: time (seconds) the operation took :param storageParameters: the parameters of the plugins used to perform the operation @@ -1387,13 +1388,15 @@ def addAccountingOperation(self, lfns, startDate, elapsedTime, storageParameters endDate = startDate + datetime.timedelta(seconds=elapsedTime) + totalSucc = 0 if not callRes["OK"]: # Everything failed - accountingDict["TransferTotal"] = len(lfns) + accountingDict["TransferTotal"] = len(urlDict) accountingDict["FinalStatus"] = "Failed" else: succ = callRes.get("Value", {}).get("Successful", {}) + failed = callRes.get("Value", {}).get("Failed", {}) totalSize = 0 @@ -1423,11 +1426,34 @@ def addAccountingOperation(self, lfns, startDate, elapsedTime, storageParameters failedAccountingDict["TransferOK"] = 0 failedAccountingDict["TransferSize"] = 0 failedAccountingDict["FinalStatus"] = "Failed" - res = self.dataOpSender.sendData(failedAccountingDict, startTime=startDate, endTime=endDate) + + # Send also the list of failures, only if we send to monitoring + failedRecords = [] + if "Monitoring" in self.dataOpSender.monitoringOptions: + + for failedURL, errorMsg in failed.items(): + failedRecord = { + "timestamp": int(toEpochMilliSeconds()), + "LFN": urlDict[failedURL], + "URL": failedURL, + "OperationType": accountingDict["OperationType"], + "User": accountingDict["User"], + "ExecutionSite": siteName(), + "TargetSE": self.name, + "Protocol": accountingDict["Protocol"], + "Error": errorMsg, + "Component": "StorageElement", + } + failedRecords.append(failedRecord) + res = self.dataOpSender.sendData( + failedAccountingDict, startTime=startDate, endTime=endDate, failedRecords=failedRecords + ) if not res["OK"]: self.log.error("Could not send failed accounting report", res["Message"]) - self.dataOpSender.sendData(accountingDict, commitFlag=False, startTime=startDate, endTime=endDate) + # Only send if there are successes + if totalSucc: + self.dataOpSender.sendData(accountingDict, commitFlag=False, startTime=startDate, endTime=endDate) StorageElement = StorageElementCache() From bfecdae192c8e099af3ba91cd6f48ed706a7004b Mon Sep 17 00:00:00 2001 From: aldbr Date: Tue, 27 Sep 2022 14:37:07 +0200 Subject: [PATCH 0011/1135] sweep: #6387 fix (wms): JobPath opPath referenced before assignment --- .../WorkloadManagementSystem/Executor/JobPath.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/DIRAC/WorkloadManagementSystem/Executor/JobPath.py b/src/DIRAC/WorkloadManagementSystem/Executor/JobPath.py index 9c1aa2fdee0..b828f2a1198 100755 --- a/src/DIRAC/WorkloadManagementSystem/Executor/JobPath.py +++ b/src/DIRAC/WorkloadManagementSystem/Executor/JobPath.py @@ -38,7 +38,7 @@ def optimizeJob(self, jid, jobState): self.jobLog.info("Job defines its own optimizer chain", opChain) else: # Construct path - opPath = self.ex_getOption("BasePath", ["JobPath", "JobSanity"]) + opChain = self.ex_getOption("BasePath", ["JobPath", "JobSanity"]) result = jobState.getInputData() if not result["OK"]: @@ -47,20 +47,20 @@ def optimizeJob(self, jid, jobState): if result["Value"]: # if the returned tuple is not empty it will evaluate true self.jobLog.info("Input data requirement found") - opPath.extend(self.ex_getOption("InputData", ["InputData"])) + opChain.extend(self.ex_getOption("InputData", ["InputData"])) else: self.jobLog.info("No input data requirement") # End of path - opPath.extend(self.ex_getOption("EndPath", ["JobScheduling"])) + opChain.extend(self.ex_getOption("EndPath", ["JobScheduling"])) uPath = [] - for opN in opPath: + for opN in opChain: if opN not in uPath: uPath.append(opN) - opPath = uPath - self.jobLog.info("Constructed path is", "%s" % "->".join(opPath)) + opChain = uPath + self.jobLog.info("Constructed path is", "%s" % "->".join(opChain)) - result = self.__setOptimizerChain(jobState, opPath) + result = self.__setOptimizerChain(jobState, opChain) if not result["OK"]: self.jobLog.error("Failed to set optimizer chain", result["Message"]) return result From 8d8524a305d9277f7b801d056306d60b4c642c56 Mon Sep 17 00:00:00 2001 From: Simon Mazenoux Date: Wed, 10 Aug 2022 10:43:51 +0200 Subject: [PATCH 0012/1135] fix: stop storing any in the jdl --- src/DIRAC/Interfaces/API/Job.py | 8 +- src/DIRAC/Interfaces/API/test/Test_JobAPI.py | 84 +++++++++++++++---- src/DIRAC/Interfaces/API/test/testWF.xml | 1 - src/DIRAC/Interfaces/API/test/testWFSIO.jdl | 1 - .../TransformationSystem/test/Test_JobInfo.py | 4 - .../Agent/test/Test_Agent_JobAgent.py | 1 - 6 files changed, 73 insertions(+), 26 deletions(-) diff --git a/src/DIRAC/Interfaces/API/Job.py b/src/DIRAC/Interfaces/API/Job.py index 2c8c9fbe09c..353d0a25525 100755 --- a/src/DIRAC/Interfaces/API/Job.py +++ b/src/DIRAC/Interfaces/API/Job.py @@ -512,13 +512,13 @@ def setDestination(self, destination): else: description = "List of sites selected by user" if isinstance(destination, list): - sites = {site for site in destination if site.lower() != "any"} + sites = {site for site in destination if site.lower() not in ("any", "")} if sites: result = self._checkSiteIsValid(sites) if not result["OK"]: - return self._reportError("%s is not a valid destination site" % (destination), **kwargs) - destSites = ";".join(destination) - self._addParameter(self.workflow, "Site", "JDL", destSites, description) + return result + destSites = ";".join(sites) + self._addParameter(self.workflow, "Site", "JDL", destSites, description) else: return self._reportError("Invalid destination site, expected string or list", **kwargs) return S_OK() diff --git a/src/DIRAC/Interfaces/API/test/Test_JobAPI.py b/src/DIRAC/Interfaces/API/test/Test_JobAPI.py index 0f757090518..13df743e10a 100644 --- a/src/DIRAC/Interfaces/API/test/Test_JobAPI.py +++ b/src/DIRAC/Interfaces/API/test/Test_JobAPI.py @@ -1,12 +1,14 @@ """ Basic unit tests for the Job API """ -from os.path import dirname, join -import pytest +# pylint: disable=missing-docstring, protected-access + from io import StringIO +from os.path import dirname, join +import pytest -from DIRAC.Interfaces.API.Job import Job from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd +from DIRAC.Interfaces.API.Job import Job def test_basicJob(): @@ -83,17 +85,69 @@ def test_SimpleParametricJob(): (None, 12, 8, 8, None, 8), # non-sense ], ) -def test_MPJob(proc, minProc, maxProc, expectedProc, expectedMinProc, expectedMaxProc): +def test_setNumberOfProcessors(proc, minProc, maxProc, expectedProc, expectedMinProc, expectedMaxProc): + # Arrange + job = Job() + # Act + res = job.setNumberOfProcessors(proc, minProc, maxProc) + + # Assert + assert res["OK"], res["Message"] + jobDescription = ClassAd(f"[{job._toJDL()}]") + assert expectedProc == jobDescription.getAttributeInt("NumberOfProcessors") + assert expectedMinProc == jobDescription.getAttributeInt("MinNumberOfProcessors") + assert expectedMaxProc == jobDescription.getAttributeInt("MaxNumberOfProcessors") + + +@pytest.mark.parametrize( + "sites, expectedSites", + [ + ("", ""), + ("Any", ""), + ("ANY", ""), + ([""], ""), + (["ANY"], ""), + (["", "ANY"], ""), + ("LCG.CERN.ch", ["LCG.CERN.ch"]), + (["LCG.CERN.ch", "ANY", ""], ["LCG.CERN.ch"]), + (["LCG.CERN.ch", "LCG.IN2P3.fr"], ["LCG.CERN.ch", "LCG.IN2P3.fr"]), + (["LCG.CERN.ch", "ANY", "LCG.IN2P3.fr"], ["LCG.CERN.ch", "LCG.IN2P3.fr"]), + ], +) +def test_setDestination_successful(sites, expectedSites): + # Arrange job = Job() - job.setExecutable("myExec") - job.setLogLevel("DEBUG") - job.setNumberOfProcessors(proc, minProc, maxProc) - jdl = job._toJDL() - clad = ClassAd("[" + jdl + "]") - processors = clad.getAttributeInt("NumberOfProcessors") - minProcessors = clad.getAttributeInt("MinNumberOfProcessors") - maxProcessors = clad.getAttributeInt("MaxNumberOfProcessors") - assert processors == expectedProc - assert minProcessors == expectedMinProc - assert maxProcessors == expectedMaxProc + job._siteSet = {"LCG.CERN.ch", "LCG.IN2P3.fr"} + + # Act + res = job.setDestination(sites) + + # Assert + assert res["OK"], res["Message"] + jobDescription = ClassAd(f"[{job._toJDL()}]") + + if expectedSites: + assert jobDescription.lookupAttribute("Site") + assert set(jobDescription.getListFromExpression("Site")) == set(expectedSites) + else: + assert not jobDescription.lookupAttribute("Site"), jobDescription.getListFromExpression("Site") + + +@pytest.mark.parametrize( + "sites", + [ + (["LCG.NCBJ.pl"]), + (["LCG.CERN.ch", "LCG.NCBJ.pl"]), + ], +) +def test_setDestination_unsuccessful(sites): + # Arrange + job = Job() + job._siteSet = {"LCG.CERN.ch"} + + # Act + res = job.setDestination(sites) + + # Assert + assert not res["OK"], res["Value"] diff --git a/src/DIRAC/Interfaces/API/test/testWF.xml b/src/DIRAC/Interfaces/API/test/testWF.xml index 25845b52eea..06996903d1a 100644 --- a/src/DIRAC/Interfaces/API/test/testWF.xml +++ b/src/DIRAC/Interfaces/API/test/testWF.xml @@ -18,7 +18,6 @@ - Date: Wed, 28 Sep 2022 18:04:28 +0200 Subject: [PATCH 0013/1135] docs: DIRAC release v7r3p30 --- releases.cfg | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/releases.cfg b/releases.cfg index 2e347783fe2..e4e22cbbe74 100644 --- a/releases.cfg +++ b/releases.cfg @@ -28,6 +28,11 @@ Releases Modules = DIRAC, RESTDIRAC:v0r5, COMDIRAC:v0r20, WebAppDIRAC:v4r4-pre1 DIRACOS = v1r27 } + v7r3p30 + { + Modules = DIRAC, RESTDIRAC:v0r7, COMDIRAC:v0r20, WebAppDIRAC:v4r3p11 + DIRACOS = v1r27 + } v7r3p29 { Modules = DIRAC, RESTDIRAC:v0r7, COMDIRAC:v0r20, WebAppDIRAC:v4r3p11 From 68beb41d347d5840f24d2809990d921c62162ee2 Mon Sep 17 00:00:00 2001 From: Simon Mazenoux Date: Fri, 24 Jun 2022 11:23:59 +0200 Subject: [PATCH 0014/1135] refactor: replacing voimport and ModuleFactory by ObjectLoader --- setup.cfg | 1 - .../AccountingSystem/Client/AccountingCLI.py | 64 +++++++-------- src/DIRAC/Core/Base/AgentModule.py | 22 +++-- src/DIRAC/Core/Base/Client.py | 15 +++- src/DIRAC/Core/Base/SQLAlchemyDB.py | 76 +++++------------ src/DIRAC/Core/Utilities/Extensions.py | 9 +-- src/DIRAC/Core/Utilities/ModuleFactory.py | 47 ----------- src/DIRAC/Core/Utilities/ObjectLoader.py | 81 +++++++++---------- .../Core/Utilities/Plotting/ObjectLoader.py | 4 - .../Client/ComponentInstaller.py | 10 +-- src/DIRAC/Interfaces/API/Dirac.py | 39 +++++---- .../private/RequestTask.py | 43 ++++------ .../Command/CommandCaller.py | 25 +++--- .../DB/ResourceManagementDB.py | 25 +++--- .../DB/ResourceStatusDB.py | 41 ++++++---- .../Actions/LogPolicyResultAction.py | 12 +-- .../ResourceStatusSystem/PolicySystem/PEP.py | 25 +++--- .../PolicySystem/PolicyCaller.py | 16 ++-- .../Service/PublisherHandler.py | 27 ++++--- .../Utilities/InfoGetter.py | 7 +- .../Utilities/Synchronizer.py | 12 +-- .../ResourceStatusSystem/Utilities/Utils.py | 17 +--- .../scripts/dirac_rss_query_dtcache.py | 10 +-- .../BatchSystems/TimeLeft/TimeLeft.py | 56 +++++-------- .../TimeLeft/test/Test_TimeLeft.py | 26 +++--- .../Agent/TransformationAgent.py | 29 ++++--- .../Client/WorkflowTasks.py | 73 +++++++---------- .../scripts/dirac_production_runjoblocal.py | 3 +- .../Workflow/Utilities/test/Test_Utilities.py | 18 ++--- .../Agent/JobAgent.py | 12 +-- .../Client/InputDataResolution.py | 40 +++------ .../DB/PilotsLoggingDB.py | 38 ++++----- .../Executor/JobPath.py | 1 + .../JobWrapper/JobWrapper.py | 12 +-- .../JobWrapper/test/Test_JobWrapper.py | 4 +- 35 files changed, 378 insertions(+), 562 deletions(-) delete mode 100755 src/DIRAC/Core/Utilities/ModuleFactory.py delete mode 100644 src/DIRAC/Core/Utilities/Plotting/ObjectLoader.py diff --git a/setup.cfg b/setup.cfg index 3e58a8866d6..74bebc90a94 100644 --- a/setup.cfg +++ b/setup.cfg @@ -84,7 +84,6 @@ server = suds tornado ~=5.1.1 tornado-m2crypto - importlib_resources testing = flaky hypothesis diff --git a/src/DIRAC/AccountingSystem/Client/AccountingCLI.py b/src/DIRAC/AccountingSystem/Client/AccountingCLI.py index e67f9036f53..a9c50b3a34a 100644 --- a/src/DIRAC/AccountingSystem/Client/AccountingCLI.py +++ b/src/DIRAC/AccountingSystem/Client/AccountingCLI.py @@ -8,12 +8,14 @@ from DIRAC import gLogger from DIRAC.Core.Base.CLI import CLI, colorize from DIRAC.AccountingSystem.Client.DataStoreClient import DataStoreClient +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader class AccountingCLI(CLI): def __init__(self): CLI.__init__(self) self.do_connect(None) + self.objectLoader = ObjectLoader() def start(self): """ @@ -70,14 +72,11 @@ def do_registerType(self, args): gLogger.error("No type name specified") return # Try to import the type - try: - typeModule = __import__( - "DIRAC.AccountingSystem.Client.Types.%s" % typeName, globals(), locals(), typeName - ) - typeClass = getattr(typeModule, typeName) - except Exception as e: - gLogger.error(f"Can't load type {typeName}: {str(e)}") - return + result = self.objectLoader.loadObject(f"DIRAC.AccountingSystem.Client.Types.{typeName}") + if not result["OK"]: + return result + typeClass = result["Value"] + gLogger.info("Loaded type %s" % typeClass.__name__) typeDef = typeClass().getDefinition() acClient = DataStoreClient() @@ -103,16 +102,13 @@ def do_resetBucketLength(self, args): else: gLogger.error("No type name specified") return + # Try to import the type - try: - typeModule = __import__( - "DIRAC.AccountingSystem.Client.Types.%s" % typeName, globals(), locals(), typeName - ) - typeClass = getattr(typeModule, typeName) - except Exception as e: - gLogger.error(f"Can't load type {typeName}: {str(e)}") - return - gLogger.info("Loaded type %s" % typeClass.__name__) + result = self.objectLoader.loadObject(f"DIRAC.AccountingSystem.Client.Types.{typeName}") + if not result["OK"]: + return result + typeClass = result["Value"] + gLogger.info(f"Loaded type {typeClass.__name__}") typeDef = typeClass().getDefinition() acClient = DataStoreClient() retVal = acClient.setBucketsLength(typeDef[0], typeDef[3]) @@ -137,16 +133,13 @@ def do_regenerateBuckets(self, args): else: gLogger.error("No type name specified") return + # Try to import the type - try: - typeModule = __import__( - "DIRAC.AccountingSystem.Client.Types.%s" % typeName, globals(), locals(), typeName - ) - typeClass = getattr(typeModule, typeName) - except Exception as e: - gLogger.error(f"Can't load type {typeName}: {str(e)}") - return - gLogger.info("Loaded type %s" % typeClass.__name__) + result = self.objectLoader.loadObject(f"DIRAC.AccountingSystem.Client.Types.{typeName}") + if not result["OK"]: + return result + typeClass = result["Value"] + gLogger.info(f"Loaded type {typeClass.__name__}") typeDef = typeClass().getDefinition() acClient = DataStoreClient() retVal = acClient.regenerateBuckets(typeDef[0]) @@ -192,16 +185,15 @@ def do_deleteType(self, args): else: gLogger.error("No type name specified") return - while True: - choice = input( - "Are you completely sure you want to delete type %s and all it's data? yes/no [no]: " % typeName - ) - choice = choice.lower() - if choice in ("yes", "y"): - break - else: - print("Delete aborted") - return + + choice = input( + "Are you completely sure you want to delete type %s and all it's data? yes/no [no]: " % typeName + ) + choice = choice.lower() + if choice not in ("yes", "y"): + print("Delete aborted") + return + acClient = DataStoreClient() retVal = acClient.deleteType(typeName) if not retVal["OK"]: diff --git a/src/DIRAC/Core/Base/AgentModule.py b/src/DIRAC/Core/Base/AgentModule.py index e36aebc8a5c..22b3cefbb34 100644 --- a/src/DIRAC/Core/Base/AgentModule.py +++ b/src/DIRAC/Core/Base/AgentModule.py @@ -119,7 +119,7 @@ def __init__(self, agentName, loadName, baseAgentName=False, properties={}): self.__moduleProperties["system"], self.__moduleProperties["agentName"] = agentName.split("/") self.__configDefaults = {} self.__configDefaults["MonitoringEnabled"] = self.am_getOption("MonitoringEnabled", True) - self.__configDefaults["Enabled"] = self.am_getOption("Status", "Active").lower() in ("active") + self.__configDefaults["Enabled"] = self.am_getOption("Status", "Active").lower() == "active" self.__configDefaults["PollingTime"] = self.am_getOption("PollingTime", 120) self.__configDefaults["MaxCycles"] = self.am_getOption("MaxCycles", 500) self.__configDefaults["WatchdogTime"] = self.am_getOption("WatchdogTime", 0) @@ -155,15 +155,21 @@ def __getCodeInfo(self): except Exception: self.log.exception(f"Failed to find version for {self!r}") self.__codeProperties["version"] = "unset" + try: - self.__agentModule = __import__(self.__class__.__module__, globals(), locals(), "__doc__") - except Exception as excp: - self.log.exception("Cannot load agent module", lException=excp) - try: - self.__codeProperties["description"] = getattr(self.__agentModule, "__doc__") - except Exception: - self.log.error("Missing property __doc__") + try: + self.__agentModule = importlib.import_module(self.__class__.__module__) + except ImportError as e: + self.log.exception("Cannot load agent module", lException=e) + raise e + try: + self.__codeProperties["description"] = getattr(self.__agentModule, "__doc__") + except AttributeError as e: + self.log.error("Missing property __doc__", lException=e) + raise e + except (ImportError, AttributeError): self.__codeProperties["description"] = "unset" + self.__codeProperties["DIRACVersion"] = DIRAC.version self.__codeProperties["platform"] = DIRAC.getPlatform() diff --git a/src/DIRAC/Core/Base/Client.py b/src/DIRAC/Core/Base/Client.py index 6ce57def625..fcdfd5b62c7 100644 --- a/src/DIRAC/Core/Base/Client.py +++ b/src/DIRAC/Core/Base/Client.py @@ -3,8 +3,19 @@ This class exposes possible RPC calls, given a url of a service. """ import ast -import importlib_resources -from functools import partial, partialmethod +from functools import partial +from importlib import resources + +try: + from functools import partialmethod +except ImportError: + + class partialmethod(partial): + def __get__(self, instance, owner): + if instance is None: + return self + return partial(self.func, instance, *(self.args or ()), **(self.keywords or {})) + from DIRAC.Core.Tornado.Client.ClientSelector import RPCClientSelector from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient diff --git a/src/DIRAC/Core/Base/SQLAlchemyDB.py b/src/DIRAC/Core/Base/SQLAlchemyDB.py index 0c940ebbee6..9f24eed4218 100644 --- a/src/DIRAC/Core/Base/SQLAlchemyDB.py +++ b/src/DIRAC/Core/Base/SQLAlchemyDB.py @@ -12,6 +12,7 @@ from DIRAC import gConfig, gLogger, S_OK, S_ERROR from DIRAC.Core.Base.DIRACDB import DIRACDB from DIRAC.ConfigurationSystem.Client.Utilities import getDBParameters +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader class SQLAlchemyDB(DIRACDB): @@ -30,6 +31,8 @@ def __init__(self, *args, **kwargs): self.extensions = gConfig.getValue("DIRAC/Extensions", []) self.tablesList = [] + self.objectLoader = ObjectLoader() + def _initializeConnection(self, dbPath): """ Collect from the CS all the info needed to connect to the DB. @@ -64,26 +67,10 @@ def _createTablesIfNotThere(self, tablesList): for table in tablesList: if table not in tablesInDB: - found = False - # is it in the extension? (fully or extended) - for ext in self.extensions: - try: - getattr( - __import__(ext + self.__class__.__module__, globals(), locals(), [table]), table - ).__table__.create( - self.engine - ) # pylint: disable=no-member - found = True - break - except (ImportError, AttributeError): - continue - # If not found in extensions, import it from DIRAC base. - if not found: - getattr( - __import__(self.__class__.__module__, globals(), locals(), [table]), table - ).__table__.create( - self.engine - ) # pylint: disable=no-member + result = self.objectLoader.loadObject(self.__class__.__module__, table) + if not result["OK"]: + return result + result["Value"].__table__.create(self.engine) else: gLogger.debug("Table %s already exists" % table) @@ -100,18 +87,10 @@ def insert(self, table, params): # expire_on_commit is set to False so that we can still use the object after we close the session session = self.sessionMaker_o(expire_on_commit=False) # FIXME: should we use this flag elsewhere? - found = False - for ext in self.extensions: - try: - tableRow_o = getattr(__import__(ext + self.__class__.__module__, globals(), locals(), [table]), table)() - found = True - break - except (ImportError, AttributeError): - continue - # If not found in extensions, import it from DIRAC base (this same module). - if not found: - tableRow_o = getattr(__import__(self.__class__.__module__, globals(), locals(), [table]), table)() - + result = self.objectLoader.loadObject(self.__class__.__module__, table) + if not result["OK"]: + return result + tableRow_o = result["Value"]() tableRow_o.fromDict(params) try: @@ -128,6 +107,8 @@ def insert(self, table, params): finally: session.close() + return S_OK() + def select(self, table, params): """ Uses params to build conditional SQL statement ( WHERE ... ). @@ -141,18 +122,10 @@ def select(self, table, params): session = self.sessionMaker_o() - # finding the table - found = False - for ext in self.extensions: - try: - table_c = getattr(__import__(ext + self.__class__.__module__, globals(), locals(), [table]), table) - found = True - break - except (ImportError, AttributeError): - continue - # If not found in extensions, import it from DIRAC base (this same module). - if not found: - table_c = getattr(__import__(self.__class__.__module__, globals(), locals(), [table]), table) + result = self.objectLoader.loadObject(self.__class__.__module__, table) + if not result["OK"]: + return result + table_c = result["Value"] # handling query conditions found in 'Meta' columnNames = [column.lower() for column in params.get("Meta", {}).get("columns", [])] @@ -236,17 +209,10 @@ def delete(self, table, params): """ session = self.sessionMaker_o() - found = False - for ext in self.extensions: - try: - table_c = getattr(__import__(ext + self.__class__.__module__, globals(), locals(), [table]), table) - found = True - break - except (ImportError, AttributeError): - continue - # If not found in extensions, import it from DIRAC base (this same module). - if not found: - table_c = getattr(__import__(self.__class__.__module__, globals(), locals(), [table]), table) + result = self.objectLoader.loadObject(self.__class__.__module__, table) + if not result["OK"]: + return result + table_c = result["Value"] # handling query conditions found in 'Meta' older = params.get("Meta", {}).get("older", None) diff --git a/src/DIRAC/Core/Utilities/Extensions.py b/src/DIRAC/Core/Utilities/Extensions.py index 7deb87632b8..4364feac9c5 100644 --- a/src/DIRAC/Core/Utilities/Extensions.py +++ b/src/DIRAC/Core/Utilities/Extensions.py @@ -4,7 +4,6 @@ import fnmatch from importlib.machinery import PathFinder import functools -import importlib import os import pkgutil import sys @@ -29,7 +28,7 @@ def newFunc(modules, *args, **kwargs): results = set() for module in modules: if isinstance(module, str): - module = importlib.import_module(module) + module = import_module(module) results |= set(func(module, *args, **kwargs)) return sorted(results) @@ -144,7 +143,7 @@ def recurseImport(modName, parentModule=None, hideExceptions=False): if parentModule is not None: raise NotImplementedError(parentModule) try: - return S_OK(importlib.import_module(modName)) + return S_OK(import_module(modName)) except ImportError as excp: # name of the module reported as not found notFoundModule = excp.name @@ -169,7 +168,7 @@ def _findFile(module, submoduleName, pattern="*"): """Implementation of findDatabases""" for system in _findSystems(module): try: - dbModule = importlib_resources.files(".".join([module.__name__, system.name, submoduleName])) + dbModule = resources.files(".".join([module.__name__, system.name, submoduleName])) except ImportError: continue for file in dbModule.iterdir(): @@ -188,7 +187,7 @@ def parseArgs(): subparser.set_defaults(func=func) args = parser.parse_args() # Get the result and print it - extensions = [importlib.import_module(e) for e in args.extensions] + extensions = [import_module(e) for e in args.extensions] for result in args.func(extensions): if not isinstance(result, str): result = " ".join(result) diff --git a/src/DIRAC/Core/Utilities/ModuleFactory.py b/src/DIRAC/Core/Utilities/ModuleFactory.py deleted file mode 100755 index 623708a5def..00000000000 --- a/src/DIRAC/Core/Utilities/ModuleFactory.py +++ /dev/null @@ -1,47 +0,0 @@ -######################################################################## -# File : ModuleFactory.py -# Author : Stuart Paterson -######################################################################## -""" The Module Factory instantiates a given Module based on a given input - string and set of arguments to be passed. This allows for VO specific - module utilities to be used in various contexts. -""" -from DIRAC import S_OK, S_ERROR, gLogger - - -class ModuleFactory: - - ############################################################################# - def __init__(self): - """Standard constructor""" - self.log = gLogger - - ############################################################################# - def getModule(self, importString, argumentsDict): - """This method returns the Module instance given the import string and - arguments dictionary. - """ - try: - moduleName = importString.split(".")[-1] - modulePath = importString.replace(".%s" % (moduleName), "") - importModule = __import__(f"{modulePath}.{moduleName}", globals(), locals(), [moduleName]) - except Exception as x: - msg = f"ModuleFactory could not import {modulePath}.{moduleName}" - self.log.warn(x) - self.log.warn(msg) - return S_ERROR(msg) - - try: - # FIXME: should we use imp module? - moduleStr = "importModule.%s(argumentsDict)" % (moduleName) - moduleInstance = eval(moduleStr) - except Exception as x: - msg = "ModuleFactory could not instantiate %s()" % (moduleName) - self.log.warn(x) - self.log.warn(msg) - return S_ERROR(msg) - - return S_OK(moduleInstance) - - -# EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF# diff --git a/src/DIRAC/Core/Utilities/ObjectLoader.py b/src/DIRAC/Core/Utilities/ObjectLoader.py index 21d0df33a6c..fbbdcca5a6e 100644 --- a/src/DIRAC/Core/Utilities/ObjectLoader.py +++ b/src/DIRAC/Core/Utilities/ObjectLoader.py @@ -1,9 +1,13 @@ -""" An utility to load modules and objects in DIRAC and extensions, being sure that the extensions are considered +""" +An utility to load modules and objects in DIRAC and extensions, +being sure that the extensions are considered """ import collections +from importlib import import_module import os import re import pkgutil +from typing import Any import DIRAC from DIRAC import gLogger, S_OK, S_ERROR @@ -21,25 +25,13 @@ class ObjectLoader(metaclass=DIRACSingleton): ol.loadObject('TransformationSystem.Client.TransformationClient') """ - def __init__(self, baseModules=False): - """init""" + def __init__(self, baseModules: list[str] = None) -> None: # We save the original arguments in case # we need to reinitialize the rootModules - # CAUTION: we cant do it after doing - # baseModules = ['DIRAC'] - # because then baseModules, self.baseModules, and __rootModules - # are the same and edited in place by __generateRootModules !! - # (Think of it, it's a binding to a list) - self.originalBaseModules = baseModules - self._init(baseModules) - - def _init(self, baseModules): - """Actually performs the initialization""" if not baseModules: baseModules = ["DIRAC"] - self.__rootModules = baseModules - self.__objs = {} - self.__generateRootModules(baseModules) + self.__baseModules = baseModules + self.__rootModules = self.__generateRootModules(baseModules) def reloadRootModules(self): """Retrigger the initialization of the rootModules. @@ -49,19 +41,15 @@ def reloadRootModules(self): the initialization after the CS has been fully initialized in LocalConfiguration.enableCS """ - # Load the original baseModule argument that was given - # to the constructor - baseModules = self.originalBaseModules - # and replay the init sequence - self._init(baseModules) + self.__rootModules = self.__generateRootModules(self.__baseModules) - def __rootImport(self, modName, hideExceptions=False): + def __rootImport(self, modName: str, hideExceptions: bool = False): """Auto search which root module has to be used""" for rootModule in self.__rootModules: impName = modName if rootModule: impName = f"{rootModule}.{impName}" - gLogger.debug("Trying to load %s" % impName) + gLogger.debug(f"Trying to load {impName}") result = recurseImport(impName, hideExceptions=hideExceptions) if not result["OK"]: return result @@ -69,27 +57,29 @@ def __rootImport(self, modName, hideExceptions=False): return S_OK((impName, result["Value"])) return S_OK() - def __generateRootModules(self, baseModules): + def __generateRootModules(self, baseModules: list[str]) -> list[str]: """Iterate over all the possible root modules""" - self.__rootModules = baseModules + rootModules = baseModules for rootModule in reversed(extensionsByPriority()): - if rootModule not in self.__rootModules: - self.__rootModules.append(rootModule) - self.__rootModules.append("") + if rootModule not in rootModules: + rootModules.append(rootModule) + rootModules.append("") # Reversing the order because we want first to look in the extension(s) - self.__rootModules.reverse() + rootModules.reverse() + + return rootModules - def loadModule(self, importString, hideExceptions=False): + def loadModule(self, importString: str, hideExceptions: bool = False): """Load a module from an import string""" result = self.__rootImport(importString, hideExceptions=hideExceptions) if not result["OK"]: return result if not result["Value"]: - return S_ERROR(DErrno.EIMPERR, "No module %s found" % importString) + return S_ERROR(DErrno.EIMPERR, f"No module {importString} found") return S_OK(result["Value"][1]) - def loadObject(self, importString, objName=False, hideExceptions=False): + def loadObject(self, importString: str, objName: str = "", hideExceptions: bool = False): """Load an object from inside a module""" if not objName: objName = importString.split(".")[-1] @@ -105,7 +95,9 @@ def loadObject(self, importString, objName=False, hideExceptions=False): except AttributeError: return S_ERROR(DErrno.EIMPERR, f"{importString} does not contain a {objName} object") - def getObjects(self, modulePath, reFilter=None, parentClass=None, recurse=False, continueOnError=False): + def getObjects( + self, modulePath: str, reFilter=None, parentClass=None, recurse: bool = False, continueOnError: bool = False + ): """Search for modules under a certain path modulePath is the import string needed to access the parent module. @@ -124,7 +116,7 @@ def getObjects(self, modulePath, reFilter=None, parentClass=None, recurse=False, impPath = modulePath if rootModule: impPath = f"{rootModule}.{impPath}" - gLogger.debug("Trying to load %s" % impPath) + gLogger.debug(f"Trying to load {impPath}") result = recurseImport(impPath) if not result["OK"]: @@ -175,9 +167,9 @@ def getObjects(self, modulePath, reFilter=None, parentClass=None, recurse=False, return S_OK(modules) -def loadObjects(path, reFilter=None, parentClass=None): +def loadObjects(path: str, reFilter=None, parentClass: object = None) -> dict[str, Any]: """ - :param str path: the path to the syetem for example: DIRAC/AccountingSystem + :param str path: the path to the system for example: DIRAC/AccountingSystem :param object reFilter: regular expression used to found the class :param object parentClass: class instance :return: dictionary containing the name of the class and its instance @@ -202,23 +194,26 @@ def loadObjects(path, reFilter=None, parentClass=None): # Load them! loadedObjects = {} - for pythonClassName in objectsToLoad: - parentModule = objectsToLoad[pythonClassName] + for pythonClassName, parentModule in objectsToLoad.items(): try: # Where parentModule can be DIRAC, pathList is something like [ "AccountingSystem", "Client", "Types" ] # And the python class name is.. well, the python class name - objPythonPath = "{}.{}.{}".format(parentModule, ".".join(pathList), pythonClassName) - objModule = __import__(objPythonPath, globals(), locals(), pythonClassName) + objPythonPath = f"{parentModule}.{'.'.join(pathList)}.{pythonClassName}" + objModule = import_module(objPythonPath) + except ImportError as e: + gLogger.error(f"No module {objPythonPath} found", str(e)) + continue + try: objClass = getattr(objModule, pythonClassName) - except Exception as e: - gLogger.error("Can't load type", f"{parentModule}/{pythonClassName}: {str(e)}") + except AttributeError as e: + gLogger.error(f"{objPythonPath} does not contain a {pythonClassName} object", str(e)) continue if parentClass == objClass: continue if parentClass and not issubclass(objClass, parentClass): gLogger.warn(f"{objClass} is not a subclass of {parentClass}. Skipping") continue - gLogger.info("Loaded %s" % objPythonPath) + gLogger.info(f"Loaded {objPythonPath}") loadedObjects[pythonClassName] = objClass return loadedObjects diff --git a/src/DIRAC/Core/Utilities/Plotting/ObjectLoader.py b/src/DIRAC/Core/Utilities/Plotting/ObjectLoader.py deleted file mode 100644 index a3c478a7c08..00000000000 --- a/src/DIRAC/Core/Utilities/Plotting/ObjectLoader.py +++ /dev/null @@ -1,4 +0,0 @@ -""" -It is used to load classes from a specific system. -""" -from DIRAC.Core.Utilities.ObjectLoader import loadObjects diff --git a/src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py b/src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py index 7859da404e0..0626927e03e 100644 --- a/src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py +++ b/src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py @@ -47,11 +47,11 @@ /LocalInstallation/VirtualOrganization: Name of the main Virtual Organization (default: None) """ +from collections import defaultdict import glob -import importlib +from importlib import metadata, resources, import_module import inspect import io -import MySQLdb import os import pkgutil import re @@ -798,7 +798,7 @@ def getComponentCfg( for ext in extensions: cfgTemplateModule = f"{ext}.{system}System" try: - cfgTemplate = importlib_resources.read_text(cfgTemplateModule, "ConfigTemplate.cfg") + cfgTemplate = resources.read_text(cfgTemplateModule, "ConfigTemplate.cfg") except (ImportError, OSError): continue gLogger.notice("Loading configuration template from", cfgTemplateModule) @@ -2066,7 +2066,7 @@ def getAvailableESDatabases(self, extensions): # Introspect all possible ones for a ElasticDB attribute try: - module = importlib.import_module(".".join([extension, systemName, "DB", dbName])) + module = import_module(".".join([extension, systemName, "DB", dbName])) dbClass = getattr(module, dbName) except (AttributeError, ImportError): continue @@ -2229,7 +2229,7 @@ def _createMySQLCMDLines(self, dbSql): sourcedDBbFileName = line.split(" ")[1].replace("\n", "") gLogger.info("Found file to source: %s" % sourcedDBbFileName) module, filename = sourcedDBbFileName.rsplit("/", 1) - dbSourced = importlib_resources.read_text(module.replace("/", "."), filename) + dbSourced = resources.read_text(module.replace("/", "."), filename) for lineSourced in dbSourced.split("\n"): if lineSourced.strip(): cmdLines.append(lineSourced.strip()) diff --git a/src/DIRAC/Interfaces/API/Dirac.py b/src/DIRAC/Interfaces/API/Dirac.py index fa59fea0480..812f4c5d0ed 100755 --- a/src/DIRAC/Interfaces/API/Dirac.py +++ b/src/DIRAC/Interfaces/API/Dirac.py @@ -33,11 +33,11 @@ from DIRAC.Core.Base.Client import Client from DIRAC.Core.Utilities.File import mkDir from DIRAC.Core.Utilities.List import breakListIntoChunks +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader from DIRAC.Core.Utilities.SiteSEMapping import getSEsForSite from DIRAC.Core.Utilities.PrettyPrint import printTable, printDict from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd from DIRAC.Core.Utilities.Subprocess import systemCall -from DIRAC.Core.Utilities.ModuleFactory import ModuleFactory from DIRAC.ConfigurationSystem.Client.PathFinder import getSystemSection, getServiceURL from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.Interfaces.API.JobRepository import JobRepository @@ -91,6 +91,8 @@ def __init__(self, withRepo=False, repoLocation="", useCertificates=False, vo=No self.defaultFileCatalog = gConfig.getValue(self.section + "/FileCatalog", None) self.vo = vo + self.objectLoader = ObjectLoader() + def _checkFileArgument(self, fnList, prefix=None, single=False): if prefix is None: prefix = "LFN" @@ -466,14 +468,12 @@ def getInputDataCatalog(self, lfns, siteName="", fileName="pool_xml_catalog.xml" if ignoreMissing: argumentsDict["IgnoreMissing"] = True self.log.verbose(argumentsDict) - moduleFactory = ModuleFactory() - self.log.verbose("Input Data Policy Module: %s" % inputDataPolicy) - moduleInstance = moduleFactory.getModule(inputDataPolicy, argumentsDict) - if not moduleInstance["OK"]: - self.log.warn("Could not create InputDataModule") - return moduleInstance - - module = moduleInstance["Value"] + + result = self.objectLoader.loadObject(inputDataPolicy) + if not result["OK"]: + return result + module = result["Value"](argumentsDict) + result = module.execute() self.log.debug(result) if not result["OK"]: @@ -575,13 +575,12 @@ def runLocal(self, job): "Job": parameters, } self.log.verbose(argumentsDict) - moduleFactory = ModuleFactory() - moduleInstance = moduleFactory.getModule(inputDataPolicy, argumentsDict) - if not moduleInstance["OK"]: - self.log.warn("Could not create InputDataModule") - return moduleInstance - module = moduleInstance["Value"] + result = self.objectLoader.loadObject(inputDataPolicy) + if not result["OK"]: + return result + module = result["Value"](argumentsDict) + result = module.execute() if not result["OK"]: self.log.warn("Input data resolution failed") @@ -589,13 +588,11 @@ def runLocal(self, job): softwarePolicy = Operations().getValue("SoftwareDistModule") if softwarePolicy: - moduleFactory = ModuleFactory() - moduleInstance = moduleFactory.getModule(softwarePolicy, {"Job": parameters}) - if not moduleInstance["OK"]: - self.log.warn("Could not create SoftwareDistModule") - return moduleInstance + result = self.objectLoader.loadObject(softwarePolicy) + if not result["OK"]: + return result + module = result["Value"]({"Job": parameters}) - module = moduleInstance["Value"] result = module.execute() if not result["OK"]: self.log.warn("Software installation failed with result:\n%s" % (result)) diff --git a/src/DIRAC/RequestManagementSystem/private/RequestTask.py b/src/DIRAC/RequestManagementSystem/private/RequestTask.py index db214da33d4..892ea005905 100644 --- a/src/DIRAC/RequestManagementSystem/private/RequestTask.py +++ b/src/DIRAC/RequestManagementSystem/private/RequestTask.py @@ -1,9 +1,3 @@ -######################################################################## -# File: RequestTask.py -# Author: Krzysztof.Ciba@NOSPAMgmail.com -# Date: 2013/03/13 12:42:45 -######################################################################## - """ :mod: RequestTask ================= @@ -14,19 +8,13 @@ .. moduleauthor:: Krzysztof.Ciba@NOSPAMgmail.com - request processing task to be used inside ProcessTask created in RequesteExecutingAgent + request processing task to be used inside ProcessTask created in RequestExecutingAgent """ -# # -# @file RequestTask.py -# @author Krzysztof.Ciba@NOSPAMgmail.com -# @date 2013/03/13 12:42:54 -# @brief Definition of RequestTask class. -# # imports + +from importlib import import_module import os import time -import datetime -# # from DIRAC from DIRAC import gLogger, S_OK, S_ERROR, gConfig from DIRAC.Core.Utilities import DErrno, TimeUtilities from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData @@ -40,7 +28,6 @@ from DIRAC.RequestManagementSystem.private.OperationHandlerBase import OperationHandlerBase from DIRAC.WorkloadManagementSystem.Client.JobMonitoringClient import JobMonitoringClient -######################################################################## class RequestTask: @@ -194,20 +181,20 @@ def loadHandler(self, pluginPath): If above conditions aren't meet, function is throwing exceptions: - :raises ImportError: when class cannot be imported + :raises ImportError: when the module isn't found + :raises AttributeError: when class isn't found in the module :raises TypeError: when class isn't inherited from OperationHandlerBase """ if "/" in pluginPath: pluginPath = ".".join([chunk for chunk in pluginPath.split("/") if chunk]) pluginName = pluginPath.split(".")[-1] - if pluginName not in globals(): - mod = __import__(pluginPath, globals(), fromlist=[pluginName]) - pluginClassObj = getattr(mod, pluginName) - else: - pluginClassObj = globals()[pluginName] - if not issubclass(pluginClassObj, OperationHandlerBase): - raise TypeError("operation handler '%s' isn't inherited from OperationHandlerBase class" % pluginName) + module = import_module(pluginPath) + pluginClassObj = getattr(module, pluginName) + + if not issubclass(pluginClassObj, OperationHandlerBase): + pluginName = pluginPath.split(".")[-1] + raise TypeError(f"operation handler '{pluginName}' isn't inherited from OperationHandlerBase class") # # return an instance return pluginClassObj @@ -222,11 +209,11 @@ def getHandler(self, operation): handler = self.handlers.get(operation.Type, None) if not handler: try: - handlerCls = self.loadHandler(self.handlersDict[operation.Type]) - self.handlers[operation.Type] = handlerCls(csPath=f"{self.csPath}/OperationHandlers/{operation.Type}") + handlerClass = self.loadHandler(self.handlersDict[operation.Type]) + self.handlers[operation.Type] = handlerClass(csPath=f"{self.csPath}/OperationHandlers/{operation.Type}") handler = self.handlers[operation.Type] - except (ImportError, TypeError) as error: - self.log.exception("Error getting Handler", "%s" % error, lException=error) + except (ImportError, AttributeError, TypeError) as error: + self.log.exception("Error getting Handler", str(error)) return S_ERROR(str(error)) # # set operation for this handler handler.setOperation(operation) diff --git a/src/DIRAC/ResourceStatusSystem/Command/CommandCaller.py b/src/DIRAC/ResourceStatusSystem/Command/CommandCaller.py index 34cb8cc49f9..140d96eb847 100644 --- a/src/DIRAC/ResourceStatusSystem/Command/CommandCaller.py +++ b/src/DIRAC/ResourceStatusSystem/Command/CommandCaller.py @@ -5,8 +5,8 @@ """ import copy -from DIRAC import S_ERROR, S_OK -from DIRAC.ResourceStatusSystem.Utilities import Utils +from DIRAC import S_OK +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader def commandInvocation(commandTuple, pArgs=None, decisionParams=None, clients=None): @@ -30,20 +30,17 @@ def commandInvocation(commandTuple, pArgs=None, decisionParams=None, clients=Non if pArgs is None: pArgs = {} - try: - cModule = commandTuple[0] - cClass = commandTuple[1] - commandModule = Utils.voimport("DIRAC.ResourceStatusSystem.Command." + cModule) - except ImportError: - return S_ERROR("Import error for command %s." % (cModule)) - - if not hasattr(commandModule, cClass): - return S_ERROR(f"{cModule} has no {cClass}") - # We merge decision parameters and policy arguments. newArgs = copy.deepcopy(decisionParams) newArgs.update(pArgs) - commandObject = getattr(commandModule, cClass)(newArgs, clients) + cModule = commandTuple[0] + cClass = commandTuple[1] + + result = ObjectLoader().loadObject(f"DIRAC.ResourceStatusSystem.Command.{cModule}", cClass) + if not result["OK"]: + return result + commandAttribute = result["Value"] + command = commandAttribute(newArgs, clients) - return S_OK(commandObject) + return S_OK(command) diff --git a/src/DIRAC/ResourceStatusSystem/DB/ResourceManagementDB.py b/src/DIRAC/ResourceStatusSystem/DB/ResourceManagementDB.py index 60a9bb0ff0d..7d21abdd885 100644 --- a/src/DIRAC/ResourceStatusSystem/DB/ResourceManagementDB.py +++ b/src/DIRAC/ResourceStatusSystem/DB/ResourceManagementDB.py @@ -23,8 +23,7 @@ from DIRAC import S_OK, S_ERROR from DIRAC.Core.Base.SQLAlchemyDB import SQLAlchemyDB -from DIRAC.ResourceStatusSystem.Utilities import Utils - +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader # Defining the tables @@ -404,9 +403,14 @@ def __init__(self, parentLogger=None): super().__init__(parentLogger=parentLogger) + self.objectLoader = ObjectLoader() + # This is the list of tables that will be created. # It can be extended in an extension module - self.tablesList = getattr(Utils.voimport("DIRAC.ResourceStatusSystem.DB.ResourceManagementDB"), "TABLESLIST") + result = self.objectLoader.loadObject(__name__, "TABLESLIST") + if not result["OK"]: + raise Exception(result["Message"]) + self.tablesList = result["Value"] self._initializeConnection("ResourceStatus/ResourceManagementDB") # Create required tables @@ -429,17 +433,10 @@ def addOrModify(self, table, params): session = self.sessionMaker_o() - found = False - for ext in self.extensions: - try: - table_c = getattr(__import__(ext + __name__, globals(), locals(), [table]), table) - found = True - break - except (ImportError, AttributeError): - continue - # If not found in extensions, import it from DIRAC base (this same module). - if not found: - table_c = getattr(__import__(__name__, globals(), locals(), [table]), table) + result = self.objectLoader.loadObject(__name__, table) + if not result["OK"]: + return result["Message"] + table_c = result["Value"] columns = [key.name for key in class_mapper(table_c).columns] primaryKeys = [key.name for key in class_mapper(table_c).primary_key] diff --git a/src/DIRAC/ResourceStatusSystem/DB/ResourceStatusDB.py b/src/DIRAC/ResourceStatusSystem/DB/ResourceStatusDB.py index d8e427eb263..1e9c398db2f 100644 --- a/src/DIRAC/ResourceStatusSystem/DB/ResourceStatusDB.py +++ b/src/DIRAC/ResourceStatusSystem/DB/ResourceStatusDB.py @@ -23,7 +23,7 @@ from DIRAC import S_OK, S_ERROR, gConfig from DIRAC.Core.Base.SQLAlchemyDB import SQLAlchemyDB -from DIRAC.ResourceStatusSystem.Utilities import Utils +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader TABLESLIST = ["SiteStatus", "ResourceStatus", "NodeStatus"] @@ -299,12 +299,19 @@ def __init__(self, parentLogger=None): super().__init__(parentLogger=parentLogger) + self.objectLoader = ObjectLoader() + # These are the list of tables that will be created. # They can be extended in an extension module - self.tablesList = getattr(Utils.voimport("DIRAC.ResourceStatusSystem.DB.ResourceStatusDB"), "TABLESLIST") - self.tablesListWithID = getattr( - Utils.voimport("DIRAC.ResourceStatusSystem.DB.ResourceStatusDB"), "TABLESLISTWITHID" - ) + result = self.objectLoader.loadObject(__name__, "TABLESLIST") + if not result["OK"]: + raise Exception(result["Message"]) + self.tablesList = result["Value"] + + result = self.objectLoader.loadObject(__name__, "TABLESLISTWITHID") + if not result["OK"]: + raise Exception(result["Message"]) + self.tablesListWithID = result["Value"] self.extensions = gConfig.getValue("DIRAC/Extensions", []) self._initializeConnection("ResourceStatus/ResourceStatusDB") @@ -344,17 +351,12 @@ def addOrModify(self, table, params): """ session = self.sessionMaker_o() - found = False - for ext in self.extensions: - try: - table_c = getattr(__import__(ext + __name__, globals(), locals(), [table]), table) - found = True - break - except (ImportError, AttributeError): - continue - # If not found in extensions, import it from DIRAC base (this same module). - if not found: - table_c = getattr(__import__(__name__, globals(), locals(), [table]), table) + + result = self.objectLoader.loadObject(__name__, table) + if not result["OK"]: + return result + table_c = result["Value"] + primaryKeys = [key.name for key in class_mapper(table_c).primary_key] try: @@ -424,7 +426,12 @@ def addIfNotThere(self, table, params): """ session = self.sessionMaker_o() - table_c = getattr(__import__(__name__, globals(), locals(), [table]), table) + + result = self.objectLoader.loadObject(__name__, table) + if not result["OK"]: + return result + table_c = result["Value"] + primaryKeys = [key.name for key in class_mapper(table_c).primary_key] try: diff --git a/src/DIRAC/ResourceStatusSystem/PolicySystem/Actions/LogPolicyResultAction.py b/src/DIRAC/ResourceStatusSystem/PolicySystem/Actions/LogPolicyResultAction.py index 20ee4f36998..2bba0773c19 100644 --- a/src/DIRAC/ResourceStatusSystem/PolicySystem/Actions/LogPolicyResultAction.py +++ b/src/DIRAC/ResourceStatusSystem/PolicySystem/Actions/LogPolicyResultAction.py @@ -2,12 +2,8 @@ """ from DIRAC import S_OK, S_ERROR +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader from DIRAC.ResourceStatusSystem.PolicySystem.Actions.BaseAction import BaseAction -from DIRAC.ResourceStatusSystem.Utilities import Utils - -ResourceManagementClient = getattr( - Utils.voimport("DIRAC.ResourceStatusSystem.Client.ResourceManagementClient"), "ResourceManagementClient" -) class LogPolicyResultAction(BaseAction): @@ -23,7 +19,11 @@ def __init__(self, name, decisionParams, enforcementResult, singlePolicyResults, if clients is not None and "ResourceManagementClient" in clients: self.rmClient = clients["ResourceManagementClient"] else: - self.rmClient = ResourceManagementClient() + result = ObjectLoader().loadObject("DIRAC.ResourceStatusSystem.Client.ResourceManagementClient") + if not result["OK"]: + raise Exception(result["Message"]) + resourceManagementClientClass = result["Value"] + self.rmClient = resourceManagementClientClass() def run(self): """ diff --git a/src/DIRAC/ResourceStatusSystem/PolicySystem/PEP.py b/src/DIRAC/ResourceStatusSystem/PolicySystem/PEP.py index cff8206e670..1441598e23c 100644 --- a/src/DIRAC/ResourceStatusSystem/PolicySystem/PEP.py +++ b/src/DIRAC/ResourceStatusSystem/PolicySystem/PEP.py @@ -12,7 +12,6 @@ """ from DIRAC import gLogger, S_OK, S_ERROR from DIRAC.ResourceStatusSystem.PolicySystem.PDP import PDP -from DIRAC.ResourceStatusSystem.Utilities import Utils from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader @@ -37,21 +36,23 @@ def __init__(self, clients=dict()): self.clients = dict(clients) + self.objectLoader = ObjectLoader() + # Creating the client in the PEP is a convenience for the PDP, that uses internally the RSS clients - res = ObjectLoader().loadObject("DIRAC.ResourceStatusSystem.Client.ResourceStatusClient") + res = self.objectLoader.loadObject("DIRAC.ResourceStatusSystem.Client.ResourceStatusClient") if not res["OK"]: self.log.error("Failed to load ResourceStatusClient class: %s" % res["Message"]) raise ImportError(res["Message"]) rsClass = res["Value"] - res = ObjectLoader().loadObject("DIRAC.ResourceStatusSystem.Client.ResourceManagementClient") + res = self.objectLoader.loadObject("DIRAC.ResourceStatusSystem.Client.ResourceManagementClient") if not res["OK"]: self.log.error("Failed to load ResourceManagementClient class: %s" % res["Message"]) raise ImportError(res["Message"]) rmClass = res["Value"] - res = ObjectLoader().loadObject("DIRAC.ResourceStatusSystem.Client.SiteStatus") + res = self.objectLoader.loadObject("DIRAC.ResourceStatusSystem.Client.SiteStatus") if not res["OK"]: self.log.error("Failed to load SiteStatus class: %s" % res["Message"]) raise ImportError(res["Message"]) @@ -141,19 +142,13 @@ def enforce(self, decisionParams): for policyActionName, policyActionType in policyCombinedResult["PolicyAction"]: - try: - actionMod = Utils.voimport("DIRAC.ResourceStatusSystem.PolicySystem.Actions.%s" % policyActionType) - except ImportError: - self.log.error("Error importing %s action" % policyActionType) - continue - - try: - action = getattr(actionMod, policyActionType) - except AttributeError: - self.log.error("Error importing %s action class" % policyActionType) + result = self.objectLoader.loadObject(f"DIRAC.ResourceStatusSystem.PolicySystem.Actions.{policyActionType}") + if not result["OK"]: + self.log.error(result["Message"]) continue - actionObj = action( + actionClass = result["Value"] + actionObj = actionClass( policyActionName, decisionParams, policyCombinedResult, singlePolicyResults, self.clients ) diff --git a/src/DIRAC/ResourceStatusSystem/PolicySystem/PolicyCaller.py b/src/DIRAC/ResourceStatusSystem/PolicySystem/PolicyCaller.py index a2e7fbd725d..ce099a91baf 100644 --- a/src/DIRAC/ResourceStatusSystem/PolicySystem/PolicyCaller.py +++ b/src/DIRAC/ResourceStatusSystem/PolicySystem/PolicyCaller.py @@ -5,7 +5,7 @@ """ from DIRAC import S_ERROR -from DIRAC.ResourceStatusSystem.Utilities import Utils +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader from DIRAC.ResourceStatusSystem.Command import CommandCaller @@ -54,15 +54,11 @@ def policyInvocation(self, decisionParams, policyDict): return S_ERROR("Malformed policyDict %s" % policyDict) pArgs = policyDict["args"] - try: - policyModule = Utils.voimport("DIRAC.ResourceStatusSystem.Policy.%s" % pModuleName) - except ImportError: - return S_ERROR("Unable to import DIRAC.ResourceStatusSystem.Policy.%s" % pModuleName) - - if not hasattr(policyModule, pModuleName): - return S_ERROR(f"{policyModule} has no attibute {pModuleName}") - - policy = getattr(policyModule, pModuleName)() + result = ObjectLoader().loadObject(f"DIRAC.ResourceStatusSystem.Policy.{pModuleName}") + if not result["OK"]: + return result + policyClass = result["Value"] + policy = policyClass() command = self.cCaller.commandInvocation(pCommand, pArgs, decisionParams, self.clients) if not command["OK"]: diff --git a/src/DIRAC/ResourceStatusSystem/Service/PublisherHandler.py b/src/DIRAC/ResourceStatusSystem/Service/PublisherHandler.py index 11052725180..d5ae4bac943 100644 --- a/src/DIRAC/ResourceStatusSystem/Service/PublisherHandler.py +++ b/src/DIRAC/ResourceStatusSystem/Service/PublisherHandler.py @@ -1,8 +1,8 @@ -""" PublisherHandler +""" +PublisherHandler This service has been built to provide the RSS web views with all the information they need. NO OTHER COMPONENT THAN Web controllers should make use of it. - """ # pylint: disable=no-self-use from datetime import datetime, timedelta @@ -10,28 +10,33 @@ # DIRAC from DIRAC import S_OK, gConfig, S_ERROR from DIRAC.Core.DISET.RequestHandler import RequestHandler +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader from DIRAC.Core.Utilities.SiteSEMapping import getSEHosts, getStorageElementsHosts from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSites, getSiteCEMapping from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers -from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient import ResourceStatusClient -from DIRAC.ResourceStatusSystem.Utilities import Utils - -ResourceManagementClient = getattr( - Utils.voimport("DIRAC.ResourceStatusSystem.Client.ResourceManagementClient"), "ResourceManagementClient" -) class PublisherHandlerMixin: """ RPCServer used to deliver data to the web portal. - """ @classmethod def initializeHandler(cls, serviceInfoDict): """Initialization of client objects""" - cls.rsClient = ResourceStatusClient() - cls.rmClient = ResourceManagementClient() + objectLoader = ObjectLoader() + + result = objectLoader.loadObject("DIRAC.ResourceStatusSystem.Client.ResourceStatusClient") + if not result["OK"]: + return result + resourceStatusClientClass = result["Value"] + cls.rsClient = resourceStatusClientClass() + + result = objectLoader.loadObject("DIRAC.ResourceStatusSystem.Client.ResourceManagementClient") + if not result["OK"]: + return result + resourceManagementClientClass = result["Value"] + cls.rmClient = resourceManagementClientClass() return S_OK() diff --git a/src/DIRAC/ResourceStatusSystem/Utilities/InfoGetter.py b/src/DIRAC/ResourceStatusSystem/Utilities/InfoGetter.py index aa8ddc53bbc..6ce5c9574ab 100644 --- a/src/DIRAC/ResourceStatusSystem/Utilities/InfoGetter.py +++ b/src/DIRAC/ResourceStatusSystem/Utilities/InfoGetter.py @@ -6,6 +6,7 @@ import copy from DIRAC import S_OK, S_ERROR, gConfig, gLogger +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader from DIRAC.ResourceStatusSystem.Utilities import RssConfiguration, Utils @@ -72,12 +73,16 @@ def getPoliciesThatApply(decisionParams): policiesThatApply = postProcessingPolicyList(policiesThatApply) gLogger.debug("policies that apply (after post-processing): %s" % str(policiesThatApply)) + objectLoader = ObjectLoader() policiesToBeLoaded = [] # Gets policies parameters from code. for policyName, policyType, _policyConfigParams in policiesThatApply: try: - configModule = Utils.voimport("DIRAC.ResourceStatusSystem.Policy.Configurations") + result = objectLoader.loadModule("DIRAC.ResourceStatusSystem.Policy.Configurations") + if not result["OK"]: + return result + configModule = result["Value"] policies = copy.deepcopy(configModule.POLICIESMETA) policyMeta = policies[policyType] except KeyError: diff --git a/src/DIRAC/ResourceStatusSystem/Utilities/Synchronizer.py b/src/DIRAC/ResourceStatusSystem/Utilities/Synchronizer.py index 874588368a6..4c6fb48a682 100644 --- a/src/DIRAC/ResourceStatusSystem/Utilities/Synchronizer.py +++ b/src/DIRAC/ResourceStatusSystem/Utilities/Synchronizer.py @@ -9,6 +9,7 @@ """ from DIRAC import gLogger, S_OK from DIRAC.Core.Security.ProxyInfo import getProxyInfo +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader from DIRAC.Core.Utilities.SiteSEMapping import getStorageElementsHosts from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSites, getFTS3Servers, getCESiteMapping from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceURL @@ -16,11 +17,6 @@ from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient import ResourceStatusClient from DIRAC.ResourceStatusSystem.Utilities import CSHelpers from DIRAC.ResourceStatusSystem.Utilities.RssConfiguration import RssConfiguration -from DIRAC.ResourceStatusSystem.Utilities import Utils - -ResourceManagementClient = getattr( - Utils.voimport("DIRAC.ResourceStatusSystem.Client.ResourceManagementClient"), "ResourceManagementClient" -) class Synchronizer: @@ -38,7 +34,11 @@ def __init__(self, rStatus=None, rManagement=None, defaultStatus="Unknown"): if rStatus is None: self.rStatus = ResourceStatusClient() if rManagement is None: - self.rManagement = ResourceManagementClient() + result = ObjectLoader().loadObject("DIRAC.ResourceStatusSystem.Client.ResourceManagementClient") + if not result["OK"]: + raise Exception(result["Value"]) + ressourceManagementClientClass = result["Value"] + self.rManagement = ressourceManagementClientClass() self.defaultStatus = defaultStatus self.rssConfig = RssConfiguration() diff --git a/src/DIRAC/ResourceStatusSystem/Utilities/Utils.py b/src/DIRAC/ResourceStatusSystem/Utilities/Utils.py index cfbe805a1bd..80e3b764614 100644 --- a/src/DIRAC/ResourceStatusSystem/Utilities/Utils.py +++ b/src/DIRAC/ResourceStatusSystem/Utilities/Utils.py @@ -6,26 +6,11 @@ """ import fnmatch -from DIRAC import gConfig, S_OK +from DIRAC import S_OK from DIRAC.Core.Utilities import List from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations -def voimport(base_mod): - """ - Function to import from extensions, if not found, tries from DIRAC. - """ - - for ext in gConfig.getValue("DIRAC/Extensions", []): - - try: - return __import__(ext + base_mod, globals(), locals(), ["*"]) - except ImportError: - continue - # If not found in extensions, import it in DIRAC base. - return __import__(base_mod, globals(), locals(), ["*"]) - - def getCSTree(csPath=""): """ Gives the configuration rooted at path in a Python dict. The diff --git a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_dtcache.py b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_dtcache.py index 0a83aa05721..d49d20d56f7 100755 --- a/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_dtcache.py +++ b/src/DIRAC/ResourceStatusSystem/scripts/dirac_rss_query_dtcache.py @@ -7,8 +7,8 @@ from DIRAC import gLogger, exit as DIRACExit, version from DIRAC.Core.Base.Script import Script from DIRAC.Core.Utilities import TimeUtilities +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader from DIRAC.Core.Utilities.PrettyPrint import printTable -from DIRAC.ResourceStatusSystem.Utilities import Utils subLogger = None @@ -350,10 +350,10 @@ def main(): registerUsageMessage() args, switchDict = parseSwitches() - ResourceManagementClient = getattr( - Utils.voimport("DIRAC.ResourceStatusSystem.Client.ResourceManagementClient"), - "ResourceManagementClient", - ) + result = ObjectLoader().loadObject("DIRAC.ResourceStatusSystem.Client.ResourceManagementClient") + if not result["OK"]: + raise Exception(result["Message"]) + ResourceManagementClient = result["Value"] # Run script run(args, switchDict) diff --git a/src/DIRAC/Resources/Computing/BatchSystems/TimeLeft/TimeLeft.py b/src/DIRAC/Resources/Computing/BatchSystems/TimeLeft/TimeLeft.py index ffbb4fc6b5d..ba577c0f9e3 100644 --- a/src/DIRAC/Resources/Computing/BatchSystems/TimeLeft/TimeLeft.py +++ b/src/DIRAC/Resources/Computing/BatchSystems/TimeLeft/TimeLeft.py @@ -15,10 +15,11 @@ import DIRAC from DIRAC import gLogger, gConfig, S_OK, S_ERROR +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader from DIRAC.Core.Utilities.Subprocess import systemCall -class TimeLeft(object): +class TimeLeft: """This generally does not run alone""" def __init__(self): @@ -27,7 +28,7 @@ def __init__(self): self.cpuPower = gConfig.getValue("/LocalSite/CPUNormalizationFactor", 0.0) if not self.cpuPower: - self.log.warn("/LocalSite/CPUNormalizationFactor not defined for site %s" % DIRAC.siteName()) + self.log.warn(f"/LocalSite/CPUNormalizationFactor not defined for site {DIRAC.siteName()}") result = self.__getBatchSystemPlugin() if result["OK"]: @@ -53,7 +54,7 @@ def getScaledCPU(self, processors=1): if "Value" in resourceDict: if resourceDict["Value"].get("CPU"): return resourceDict["Value"]["CPU"] * self.cpuPower - elif resourceDict["Value"].get("WallClock"): + if resourceDict["Value"].get("WallClock"): # When CPU value missing, guess from WallClock and number of processors return resourceDict["Value"]["WallClock"] * self.cpuPower * processors @@ -65,18 +66,18 @@ def getTimeLeft(self, cpuConsumed=0.0, processors=1): """ # Quit if no norm factor available if not self.cpuPower: - return S_ERROR("/LocalSite/CPUNormalizationFactor not defined for site %s" % DIRAC.siteName()) + return S_ERROR(f"/LocalSite/CPUNormalizationFactor not defined for site {DIRAC.siteName()}") if not self.batchPlugin: return S_ERROR(self.batchError) resourceDict = self.batchPlugin.getResourceUsage() if not resourceDict["OK"]: - self.log.warn("Could not determine timeleft for batch system at site %s" % DIRAC.siteName()) + self.log.warn(f"Could not determine timeleft for batch system at site {DIRAC.siteName()}") return resourceDict resources = resourceDict["Value"] - self.log.debug("self.batchPlugin.getResourceUsage(): %s" % str(resources)) + self.log.debug(f"self.batchPlugin.getResourceUsage(): {str(resources)}") if not resources.get("CPULimit") and not resources.get("WallClockLimit"): # This should never happen return S_ERROR("No CPU or WallClock limit obtained") @@ -123,7 +124,7 @@ def getTimeLeft(self, cpuConsumed=0.0, processors=1): # In case the returned cpu and cpuLimit are not in real seconds, this is however rubbish cpuWorkLeft = (timeLimit - time) * self.cpuPower - self.log.verbose("Remaining CPU in normalized units is: %.02f" % cpuWorkLeft) + self.log.verbose(f"Remaining CPU in normalized units is: {cpuWorkLeft:.2f}") return S_OK(cpuWorkLeft) def __getBatchSystemPlugin(self): @@ -147,33 +148,16 @@ def __getBatchSystemPlugin(self): name = "MJF" if name is None: - self.log.warn("Batch system type for site %s is not currently supported" % DIRAC.siteName()) + self.log.warn(f"Batch system type for site {DIRAC.siteName()} is not currently supported") return S_ERROR("Current batch system is not supported") - self.log.debug("Creating plugin for %s batch system" % (name)) - try: - batchSystemName = "%sResourceUsage" % (name) - batchPlugin = __import__( - "DIRAC.Resources.Computing.BatchSystems.TimeLeft.%s" - % batchSystemName, # pylint: disable=unused-variable - globals(), - locals(), - [batchSystemName], - ) - except ImportError as x: - msg = "Could not import DIRAC.Resources.Computing.BatchSystems.TimeLeft.%s" % (batchSystemName) - self.log.warn(x) - self.log.warn(msg) - return S_ERROR(msg) - - try: - batchStr = "batchPlugin.%s()" % (batchSystemName) - batchInstance = eval(batchStr) - except Exception as x: # pylint: disable=broad-except - msg = "Could not instantiate %s()" % (batchSystemName) - self.log.warn(x) - self.log.warn(msg) - return S_ERROR(msg) + self.log.debug(f"Creating plugin for {name} batch system") + + result = ObjectLoader().loadObject(f"DIRAC.Resources.Computing.BatchSystems.TimeLeft.{name}ResourceUsage") + if not result["OK"]: + return result + batchClass = result["Value"] + batchInstance = batchClass() return S_OK(batchInstance) @@ -189,12 +173,12 @@ def runCommand(cmd, timeout=120): status, stdout, stderr = result["Value"][0:3] if status: - gLogger.warn("Status %s while executing %s" % (status, cmd)) + gLogger.warn(f"Status {status} while executing {cmd}") gLogger.warn(stderr) if stdout: return S_ERROR(stdout) if stderr: return S_ERROR(stderr) - return S_ERROR("Status %s while executing %s" % (status, cmd)) - else: - return S_OK(str(stdout)) + return S_ERROR(f"Status {status} while executing {cmd}") + + return S_OK(str(stdout)) diff --git a/src/DIRAC/Resources/Computing/BatchSystems/TimeLeft/test/Test_TimeLeft.py b/src/DIRAC/Resources/Computing/BatchSystems/TimeLeft/test/Test_TimeLeft.py index 011cfee25bb..bccc04f3b4f 100644 --- a/src/DIRAC/Resources/Computing/BatchSystems/TimeLeft/test/Test_TimeLeft.py +++ b/src/DIRAC/Resources/Computing/BatchSystems/TimeLeft/test/Test_TimeLeft.py @@ -2,12 +2,11 @@ (Partially) tested here are SGE and LSF, PBS is TO-DO """ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +# pylint: disable=invalid-name + +from importlib import import_module, reload import pytest -from importlib import reload from DIRAC import S_OK, gLogger from DIRAC.Resources.Computing.BatchSystems.TimeLeft.TimeLeft import TimeLeft @@ -94,14 +93,12 @@ def test_getScaledCPU(mocker, batch, requiredVariables, returnValue, expected): tl.cpuPower = 5.0 - batchSystemName = "%sResourceUsage" % batch - batchSystemPath = "DIRAC.Resources.Computing.BatchSystems.TimeLeft.%s" % batchSystemName - batchPlugin = __import__(batchSystemPath, globals(), locals(), [batchSystemName]) # pylint: disable=unused-variable + batchSystemName = f"{batch}ResourceUsage" + batchSystemPath = f"DIRAC.Resources.Computing.BatchSystems.TimeLeft.{batchSystemName}" + batchPlugin = import_module(batchSystemPath) # Need to be reloaded to update the mock within the module, else, it will reuse the one when loaded the first time reload(batchPlugin) - - batchStr = "batchPlugin.%s()" % (batchSystemName) - tl.batchPlugin = eval(batchStr) + tl.batchPlugin = getattr(batchPlugin, batchSystemName)() # Update attributes of the batch systems to get scaled CPU tl.batchPlugin.__dict__.update(requiredVariables) @@ -130,14 +127,13 @@ def test_getTimeLeft(mocker, batch, requiredVariables, returnValue, expected_1, mocker.patch("DIRAC.Resources.Computing.BatchSystems.TimeLeft.TimeLeft.runCommand", return_value=S_OK(returnValue)) tl = TimeLeft() - batchSystemName = "%sResourceUsage" % batch - batchSystemPath = "DIRAC.Resources.Computing.BatchSystems.TimeLeft.%s" % batchSystemName - batchPlugin = __import__(batchSystemPath, globals(), locals(), [batchSystemName]) + batchSystemName = f"{batch}ResourceUsage" + batchSystemPath = f"DIRAC.Resources.Computing.BatchSystems.TimeLeft.{batchSystemName}" + batchPlugin = import_module(batchSystemPath) # Need to be reloaded to update the mock within the module, else, it will reuse the one when loaded the first time reload(batchPlugin) + tl.batchPlugin = getattr(batchPlugin, batchSystemName)() - batchStr = "batchPlugin.%s()" % (batchSystemName) - tl.batchPlugin = eval(batchStr) tl.cpuPower = 10.0 # Update attributes of the batch systems to get scaled CPU diff --git a/src/DIRAC/TransformationSystem/Agent/TransformationAgent.py b/src/DIRAC/TransformationSystem/Agent/TransformationAgent.py index aac224b71a9..6600960e208 100644 --- a/src/DIRAC/TransformationSystem/Agent/TransformationAgent.py +++ b/src/DIRAC/TransformationSystem/Agent/TransformationAgent.py @@ -8,6 +8,7 @@ :dedent: 2 :caption: TransformationAgent options """ +from importlib import import_module import time import os import datetime @@ -623,10 +624,8 @@ def __removeFilesFromCache(self, transID, lfns): self.__writeCache(transID) def __removeFromCache(self, transID, lfns): - if transID not in self.replicaCache: - return removed = 0 - if self.replicaCache[transID] and lfns: + if transID in self.replicaCache and self.replicaCache[transID] and lfns: for lfn in lfns: for timeKey in self.replicaCache[transID]: if self.replicaCache[transID][timeKey].pop(lfn, None): @@ -699,22 +698,22 @@ def __writeCache(self, transID=None): def __generatePluginObject(self, plugin, clients): """This simply instantiates the TransformationPlugin class with the relevant plugin name""" try: - plugModule = __import__(self.pluginLocation, globals(), locals(), ["TransformationPlugin"]) - except ImportError as e: - self._logException( - "Failed to import 'TransformationPlugin' %s" % plugin, lException=e, method="__generatePluginObject" - ) + plugModule = import_module(self.pluginLocation) + except ImportError: + self._logException(f"Failed to import 'TransformationPlugin' {plugin}", method="__generatePluginObject") return S_ERROR() try: - plugin_o = getattr(plugModule, "TransformationPlugin")( - "%s" % plugin, transClient=clients["TransformationClient"], dataManager=clients["DataManager"] + pluginClass = getattr(plugModule, "TransformationPlugin") + except AttributeError: + self._logException( + f"Failed to import 'TransformationPlugin' class from module {self.pluginLocation}", + method="__generatePluginObject", ) - return S_OK(plugin_o) - except AttributeError as e: - self._logException("Failed to create %s()" % plugin, lException=e, method="__generatePluginObject") return S_ERROR() - plugin_o.setDirectory(self.workDirectory) - plugin_o.setCallback(self.pluginCallback) + + plugin_o = pluginClass(plugin, transClient=clients["TransformationClient"], dataManager=clients["DataManager"]) + + return S_OK(plugin_o) def pluginCallback(self, transID, invalidateCache=False): """Standard plugin callback""" diff --git a/src/DIRAC/TransformationSystem/Client/WorkflowTasks.py b/src/DIRAC/TransformationSystem/Client/WorkflowTasks.py index bb65a2df31a..519593ce58b 100644 --- a/src/DIRAC/TransformationSystem/Client/WorkflowTasks.py +++ b/src/DIRAC/TransformationSystem/Client/WorkflowTasks.py @@ -1,15 +1,15 @@ -import time -from io import StringIO import copy import os +import time +from io import StringIO from DIRAC import S_OK, S_ERROR, gLogger from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getDNForUsername from DIRAC.Core.Security.ProxyInfo import getProxyInfo from DIRAC.Core.Utilities.List import fromChar -from DIRAC.Core.Utilities.ModuleFactory import ModuleFactory from DIRAC.Core.Utilities.DErrno import ETSDATA, ETSUKN +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader from DIRAC.Interfaces.API.Job import Job from DIRAC.TransformationSystem.Client import TransformationFilesStatus from DIRAC.TransformationSystem.Client.TaskManager import TaskBase @@ -43,7 +43,7 @@ def __init__( super().__init__(transClient, logger) - useCertificates = True if (bool(ownerDN) and bool(ownerGroup)) else False + useCertificates = bool(bool(ownerDN) and bool(ownerGroup)) if not submissionClient: self.submissionClient = WMSClient( useCertificates=useCertificates, delegatedDN=ownerDN, delegatedGroup=ownerGroup @@ -79,6 +79,7 @@ def __init__( self.destinationPlugin_o = None self.outputDataModule_o = None + self.objectLoader = ObjectLoader() def prepareTransformationTasks( self, transBody, taskDict, owner="", ownerGroup="", ownerDN="", bulkSubmissionFlag=False @@ -187,9 +188,8 @@ def __prepareTasksBulk(self, transBody, taskDict, owner, ownerGroup, ownerDN): if not sites: self._logError("Could not get a list a sites", transID=transID, method=method) return S_ERROR(ETSUKN, "Can not evaluate destination site") - else: - self._logVerbose("Setting Site: ", str(sites), transID=transID, method=method) - seqDict["Site"] = sites + self._logVerbose("Setting Site: ", str(sites), transID=transID, method=method) + seqDict["Site"] = sites seqDict["JobName"] = self._transTaskName(transID, taskID) seqDict["JOB_ID"] = str(taskID).zfill(8) @@ -332,13 +332,13 @@ def __prepareTasks(self, transBody, taskDict, owner, ownerGroup, ownerDN): self._logError("Could not get a list a sites", transID=transID, method=method) paramsDict["TaskObject"] = "" continue - else: - self._logDebug("Setting Site: ", str(sites), transID=transID, method=method) - res = oJob.setDestination(sites) - if not res["OK"]: - self._logError("Could not set the site: %s" % res["Message"], transID=transID, method=method) - paramsDict["TaskObject"] = "" - continue + + self._logDebug("Setting Site: ", str(sites), transID=transID, method=method) + res = oJob.setDestination(sites) + if not res["OK"]: + self._logError("Could not set the site: %s" % res["Message"], transID=transID, method=method) + paramsDict["TaskObject"] = "" + continue self._handleInputs(oJob, paramsDict) self._handleRest(oJob, paramsDict) @@ -382,18 +382,16 @@ def _handleDestination(self, paramsDict): except KeyError: pass - if self.destinationPlugin_o: - destinationPlugin_o = self.destinationPlugin_o - else: - res = self.__generatePluginObject(self.destinationPlugin) - if not res["OK"]: + if not self.destinationPlugin_o: + result = self.objectLoader.loadObject(self.pluginLocation) + if not result["OK"]: self._logFatal("Could not generate a destination plugin object") - return res - destinationPlugin_o = res["Value"] - self.destinationPlugin_o = destinationPlugin_o + return result + taskManagerPluginClass = result["Value"] + self.destinationPlugin_o = taskManagerPluginClass("%s" % self.destinationPlugin, operationsHelper=self.opsH) - destinationPlugin_o.setParameters(paramsDict) - destSites = destinationPlugin_o.run() + self.destinationPlugin_o.setParameters(paramsDict) + destSites = self.destinationPlugin_o.run() if not destSites: return sites @@ -457,33 +455,18 @@ def _handleHospital(self, oJob, clinicPath): if hospitalCEs: oJob._addJDLParameter("GridCE", hospitalCEs) - def __generatePluginObject(self, plugin): - """This simply instantiates the TaskManagerPlugin class with the relevant plugin name""" - method = "__generatePluginObject" - try: - plugModule = __import__(self.pluginLocation, globals(), locals(), ["TaskManagerPlugin"]) - except ImportError as e: - self._logException(f"Failed to import 'TaskManagerPlugin' {plugin}: {e}", method=method) - return S_ERROR() - try: - plugin_o = getattr(plugModule, "TaskManagerPlugin")("%s" % plugin, operationsHelper=self.opsH) - return S_OK(plugin_o) - except AttributeError as e: - self._logException(f"Failed to create {plugin}(): {e}.", method=method) - return S_ERROR() - ############################################################################# def getOutputData(self, paramDict): """Get the list of job output LFNs from the provided plugin""" if not self.outputDataModule_o: - # Create the module object - moduleFactory = ModuleFactory() + result = self.objectLoader.loadObject(self.outputDataModule) + if not result["OK"]: + return result + objectClass = result["Value"] + + self.outputDataModule_o = objectClass() - moduleInstance = moduleFactory.getModule(self.outputDataModule, None) - if not moduleInstance["OK"]: - return moduleInstance - self.outputDataModule_o = moduleInstance["Value"] # This is the "argument" to the module, set it and then execute self.outputDataModule_o.paramDict = paramDict return self.outputDataModule_o.execute() diff --git a/src/DIRAC/TransformationSystem/scripts/dirac_production_runjoblocal.py b/src/DIRAC/TransformationSystem/scripts/dirac_production_runjoblocal.py index 3c07b9f4d31..484e2d41d8f 100755 --- a/src/DIRAC/TransformationSystem/scripts/dirac_production_runjoblocal.py +++ b/src/DIRAC/TransformationSystem/scripts/dirac_production_runjoblocal.py @@ -4,6 +4,7 @@ It creates the necessary environment, downloads the necessary files, modifies the necessary files and runs the job """ +from importlib import import_module import os import shutil import ssl @@ -103,7 +104,7 @@ def __runJobLocally(jobID, basepath, vo): """ Runs the job! """ - ipr = __import__(str(vo) + "DIRAC.Interfaces.API." + str(vo) + "Job", globals(), locals(), [str(vo) + "Job"], -1) + ipr = import_module(str(vo) + "DIRAC.Interfaces.API." + str(vo) + "Job") voJob = getattr(ipr, str(vo) + "Job") localJob = voJob(basepath + "InputSandbox" + str(jobID) + os.path.sep + "jobDescription.xml") localJob.setInputSandbox(os.getcwd() + os.path.sep + "pilot.cfg") diff --git a/src/DIRAC/Workflow/Utilities/test/Test_Utilities.py b/src/DIRAC/Workflow/Utilities/test/Test_Utilities.py index 5d9fb1441c6..fd19faa430c 100644 --- a/src/DIRAC/Workflow/Utilities/test/Test_Utilities.py +++ b/src/DIRAC/Workflow/Utilities/test/Test_Utilities.py @@ -1,3 +1,6 @@ +# pylint: disable=invalid-name,missing-docstring,protected-access + +from importlib import import_module import unittest from DIRAC.Core.Workflow.Module import ModuleDefinition @@ -5,16 +8,12 @@ from DIRAC.Interfaces.API.Job import Job from DIRAC.Workflow.Utilities.Utils import getStepDefinition, getStepCPUTimes -############################################################################# - class UtilitiesTestCase(unittest.TestCase): """Base class""" def setUp(self): - self.job = Job() - pass class UtilsSuccess(UtilitiesTestCase): @@ -25,21 +24,14 @@ def test__getStepDefinition(self): # modules gaudiApp = ModuleDefinition("Script") body = importLine.replace("", "Script") - gaudiApp.setDescription( - getattr( - __import__("{}.{}".format("DIRAC.Workflow.Modules", "Script"), globals(), locals(), ["__doc__"]), - "__doc__", - ) - ) + gaudiApp.setDescription(getattr(import_module("DIRAC.Workflow.Modules.Script"), "__doc__")) gaudiApp.setBody(body) genBKReport = ModuleDefinition("FailoverRequest") body = importLine.replace("", "FailoverRequest") genBKReport.setDescription( getattr( - __import__( - "{}.{}".format("DIRAC.Workflow.Modules", "FailoverRequest"), globals(), locals(), ["__doc__"] - ), + import_module("DIRAC.Workflow.Modules.FailoverRequest"), "__doc__", ) ) diff --git a/src/DIRAC/WorkloadManagementSystem/Agent/JobAgent.py b/src/DIRAC/WorkloadManagementSystem/Agent/JobAgent.py index 88e304a13b2..3dee03949db 100755 --- a/src/DIRAC/WorkloadManagementSystem/Agent/JobAgent.py +++ b/src/DIRAC/WorkloadManagementSystem/Agent/JobAgent.py @@ -17,12 +17,12 @@ from diraccfg import CFG from DIRAC import S_OK, S_ERROR, gConfig, rootPath, siteName -from DIRAC.Core.Utilities.ModuleFactory import ModuleFactory from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd from DIRAC.Core.Base.AgentModule import AgentModule from DIRAC.Core.Security.ProxyInfo import getProxyInfo from DIRAC.Core.Security import Properties from DIRAC.Core.Utilities import DErrno +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager from DIRAC.Resources.Computing.BatchSystems.TimeLeft.TimeLeft import TimeLeft from DIRAC.Resources.Computing.ComputingElementFactory import ComputingElementFactory @@ -522,12 +522,12 @@ def _checkInstallSoftware(self, jobID, jobParams, resourceParams, jobReport): softwareDist = jobParams["SoftwareDistModule"] self.log.verbose("Found VO Software Distribution module", ": %s" % (softwareDist)) argumentsDict = {"Job": jobParams, "CE": resourceParams} - moduleFactory = ModuleFactory() - moduleInstance = moduleFactory.getModule(softwareDist, argumentsDict) - if not moduleInstance["OK"]: - return moduleInstance - module = moduleInstance["Value"] + result = ObjectLoader().loadObject(softwareDist) + if not result["OK"]: + return result + module = result["Value"](argumentsDict) + return module.execute() ############################################################################# diff --git a/src/DIRAC/WorkloadManagementSystem/Client/InputDataResolution.py b/src/DIRAC/WorkloadManagementSystem/Client/InputDataResolution.py index 61d5a6b6226..37db0740b0a 100644 --- a/src/DIRAC/WorkloadManagementSystem/Client/InputDataResolution.py +++ b/src/DIRAC/WorkloadManagementSystem/Client/InputDataResolution.py @@ -7,8 +7,8 @@ for applications. """ import DIRAC -from DIRAC import S_OK, S_ERROR, gLogger, gConfig -from DIRAC.Core.Utilities.ModuleFactory import ModuleFactory +from DIRAC import S_OK, S_ERROR, gLogger +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader from DIRAC.WorkloadManagementSystem.Client.PoolXMLSlice import PoolXMLSlice from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations @@ -19,7 +19,6 @@ class InputDataResolution: """Defines the Input Data Policy""" - ############################################################################# def __init__(self, argumentsDict): """Standard constructor""" self.arguments = argumentsDict @@ -37,7 +36,6 @@ def __init__(self, argumentsDict): # By default put input data into the current directory self.arguments.setdefault("InputDataDirectory", gConfig.getValue("/LocalSite/InputDataDirectory", "CWD")) - ############################################################################# def execute(self): """Given the arguments from the Job Wrapper, this function calls existing utilities in DIRAC to resolve input data. @@ -66,8 +64,6 @@ def execute(self): return resolvedInputData - ############################################################################# - def _createCatalog(self, resolvedInputData, catalogName="pool_xml_catalog.xml", pfnType="ROOT_All"): """By default uses PoolXMLSlice, VO extensions can modify at will""" @@ -85,8 +81,6 @@ def _createCatalog(self, resolvedInputData, catalogName="pool_xml_catalog.xml", appCatalog = PoolXMLSlice(catalogName) return appCatalog.execute(resolvedData) - ############################################################################# - def __resolveInputData(self): """This method controls the execution of the DIRAC input data modules according to the VO policy defined in the configuration service. @@ -120,9 +114,15 @@ def __resolveInputData(self): dataToResolve = [] # if none, all supplied input data is resolved successful = {} for modulePath in policy: - result = self.__runModule(modulePath, dataToResolve) + self.log.info(f"Attempting to run {modulePath}") + + result = ObjectLoader().loadObject(modulePath) + if not result["OK"]: + return result + module = result["Value"](self.arguments) + + result = module.execute(dataToResolve) if not result["OK"]: - self.log.warn("Problem during %s execution" % modulePath) return result result = result["Value"] @@ -138,23 +138,3 @@ def __resolveInputData(self): self.log.verbose("Successfully resolved:", str(successful)) return S_OK({"Successful": successful, "Failed": dataToResolve}) - - ############################################################################# - def __runModule(self, modulePath, remainingReplicas): - """This method provides a way to run the modules specified by the VO that - govern the input data access policy for the current site. Using the - InputDataPolicy section from Operations different modules can be defined for - particular sites or for InputDataPolicy defined in the JDL of the jobs. - """ - self.log.info("Attempting to run %s" % (modulePath)) - moduleFactory = ModuleFactory() - moduleInstance = moduleFactory.getModule(modulePath, self.arguments) - if not moduleInstance["OK"]: - return moduleInstance - - module = moduleInstance["Value"] - result = module.execute(remainingReplicas) - return result - - -# EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF# diff --git a/src/DIRAC/WorkloadManagementSystem/DB/PilotsLoggingDB.py b/src/DIRAC/WorkloadManagementSystem/DB/PilotsLoggingDB.py index a5c5292b357..3635ac127d7 100644 --- a/src/DIRAC/WorkloadManagementSystem/DB/PilotsLoggingDB.py +++ b/src/DIRAC/WorkloadManagementSystem/DB/PilotsLoggingDB.py @@ -10,10 +10,6 @@ deletePilotsLoggin() """ -from DIRAC import gLogger, gConfig, S_OK, S_ERROR -from DIRAC.Core.Utilities import DErrno -from DIRAC.ConfigurationSystem.Client.Utilities import getDBParameters -from DIRAC.ResourceStatusSystem.Utilities import Utils from sqlalchemy.orm import sessionmaker, scoped_session from sqlalchemy.engine.reflection import Inspector @@ -21,6 +17,10 @@ from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.exc import SQLAlchemyError +from DIRAC import gLogger, S_OK, S_ERROR +from DIRAC.Core.Utilities import DErrno +from DIRAC.ConfigurationSystem.Client.Utilities import getDBParameters +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader TABLESLIST = ["PilotsLogging"] @@ -28,7 +28,6 @@ Base = declarative_base() -############################################################################# class PilotsLoggingDB: def __init__(self, parentLogger=None): @@ -47,9 +46,14 @@ def __init__(self, parentLogger=None): self.dbPass = dbParameters["Password"] self.dbName = dbParameters["DBName"] + self.objectLoader = ObjectLoader() + # These are the list of tables that will be created. # They can be extended in an extension module - self.tablesList = getattr(Utils.voimport("DIRAC.WorkloadManagementSystem.DB.PilotsLoggingDB"), "TABLESLIST") + result = self.objectLoader.loadObject(__name__, "TABLESLIST") + if not result["OK"]: + raise Exception(result["Message"]) + self.tablesList = result["Value"] self.__initializeConnection() resp = self.__initializeDB() @@ -82,24 +86,12 @@ def __initializeDB(self): for table in self.tablesList: if table not in tablesInDB: - found = False - # is it in the extension? (fully or extended) - for ext in gConfig.getValue("DIRAC/Extensions", []): - try: - getattr(__import__(ext + __name__, globals(), locals(), [table]), table).__table__.create( - self.engine - ) # pylint: disable=no-member - found = True - break - except (ImportError, AttributeError): - continue - # If not found in extensions, import it from DIRAC base. - if not found: - getattr(__import__(__name__, globals(), locals(), [table]), table).__table__.create( - self.engine - ) # pylint: disable=no-member + result = self.objectLoader.loadObject(__name__, table) + if not result["OK"]: + return result + result["Value"].__table__.create(self.engine) else: - gLogger.debug("Table %s already exists" % table) + gLogger.debug(f"Table {table} already exists") return S_OK() diff --git a/src/DIRAC/WorkloadManagementSystem/Executor/JobPath.py b/src/DIRAC/WorkloadManagementSystem/Executor/JobPath.py index b828f2a1198..0061fa5fc15 100755 --- a/src/DIRAC/WorkloadManagementSystem/Executor/JobPath.py +++ b/src/DIRAC/WorkloadManagementSystem/Executor/JobPath.py @@ -7,6 +7,7 @@ """ from DIRAC import S_OK, S_ERROR from DIRAC.Core.Utilities import List +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader from DIRAC.WorkloadManagementSystem.Executor.Base.OptimizerExecutor import OptimizerExecutor diff --git a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py index d0db3b57f60..09b9ab87108 100755 --- a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py +++ b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py @@ -30,8 +30,8 @@ from DIRAC.Core.Utilities import DErrno from DIRAC.Core.Utilities import List from DIRAC.Core.Utilities import DEncode +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader from DIRAC.Core.Utilities.SiteSEMapping import getSEsForSite -from DIRAC.Core.Utilities.ModuleFactory import ModuleFactory from DIRAC.Core.Utilities.Subprocess import systemCall from DIRAC.Core.Utilities.Subprocess import Subprocess from DIRAC.Core.Utilities.File import getGlobbedTotalSize, getGlobbedFiles @@ -646,13 +646,13 @@ def resolveInputData(self): "Job": self.jobArgs, } self.log.info(argumentsDict) - moduleFactory = ModuleFactory() + self.log.verbose("Now starting execution of input data policy module") - moduleInstance = moduleFactory.getModule(inputDataPolicy, argumentsDict) - if not moduleInstance["OK"]: - return moduleInstance + result = ObjectLoader().loadObject(inputDataPolicy) + if not result["OK"]: + return result + module = result["Value"](argumentsDict) - module = moduleInstance["Value"] result = module.execute() if not result["OK"]: self.log.warn("Input data resolution failed") diff --git a/src/DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapper.py b/src/DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapper.py index 18cd13594db..da41f362c2c 100644 --- a/src/DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapper.py +++ b/src/DIRAC/WorkloadManagementSystem/JobWrapper/test/Test_JobWrapper.py @@ -24,7 +24,7 @@ def test_InputData(mocker): mocker.patch( "DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper.getSystemSection", side_effect=getSystemSectionMock ) - mocker.patch("DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper.ModuleFactory", side_effect=MagicMock()) + mocker.patch("DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper.ObjectLoader", side_effect=MagicMock()) jw = JobWrapper() jw.jobArgs["InputData"] = "" @@ -119,7 +119,7 @@ def test_finalize(mocker, failedFlag, expectedRes, finalStates): mocker.patch( "DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper.getSystemSection", side_effect=getSystemSectionMock ) - mocker.patch("DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper.ModuleFactory", side_effect=MagicMock()) + mocker.patch("DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper.ObjectLoader", side_effect=MagicMock()) jw = JobWrapper() jw.jobArgs = {"Executable": "/bin/ls"} From 986c80cf0542a0b000a9be651be2c12cbe20abad Mon Sep 17 00:00:00 2001 From: fstagni Date: Wed, 28 Sep 2022 18:21:15 +0200 Subject: [PATCH 0015/1135] fix: no need to re-implement partialmethod --- src/DIRAC/Core/Base/Client.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/src/DIRAC/Core/Base/Client.py b/src/DIRAC/Core/Base/Client.py index fcdfd5b62c7..6027ee83bcc 100644 --- a/src/DIRAC/Core/Base/Client.py +++ b/src/DIRAC/Core/Base/Client.py @@ -3,19 +3,7 @@ This class exposes possible RPC calls, given a url of a service. """ import ast -from functools import partial -from importlib import resources - -try: - from functools import partialmethod -except ImportError: - - class partialmethod(partial): - def __get__(self, instance, owner): - if instance is None: - return self - return partial(self.func, instance, *(self.args or ()), **(self.keywords or {})) - +from functools import partial, partialmethod from DIRAC.Core.Tornado.Client.ClientSelector import RPCClientSelector from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient From 9cbf9f8d91be82391601f2c4eb1937a8d91e0bc1 Mon Sep 17 00:00:00 2001 From: fstagni Date: Wed, 28 Sep 2022 18:36:18 +0200 Subject: [PATCH 0016/1135] fix: sorting of several imports --- setup.cfg | 1 + src/DIRAC/Core/Base/Client.py | 1 + src/DIRAC/Core/Utilities/Extensions.py | 15 ++--- .../Client/ComponentInstaller.py | 59 +++++++++---------- .../private/RequestTask.py | 10 ++-- .../Client/InputDataResolution.py | 2 +- .../Executor/JobPath.py | 4 +- 7 files changed, 43 insertions(+), 49 deletions(-) diff --git a/setup.cfg b/setup.cfg index 74bebc90a94..3e58a8866d6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -84,6 +84,7 @@ server = suds tornado ~=5.1.1 tornado-m2crypto + importlib_resources testing = flaky hypothesis diff --git a/src/DIRAC/Core/Base/Client.py b/src/DIRAC/Core/Base/Client.py index 6027ee83bcc..6ce57def625 100644 --- a/src/DIRAC/Core/Base/Client.py +++ b/src/DIRAC/Core/Base/Client.py @@ -3,6 +3,7 @@ This class exposes possible RPC calls, given a url of a service. """ import ast +import importlib_resources from functools import partial, partialmethod from DIRAC.Core.Tornado.Client.ClientSelector import RPCClientSelector diff --git a/src/DIRAC/Core/Utilities/Extensions.py b/src/DIRAC/Core/Utilities/Extensions.py index 4364feac9c5..c737ef8821e 100644 --- a/src/DIRAC/Core/Utilities/Extensions.py +++ b/src/DIRAC/Core/Utilities/Extensions.py @@ -1,12 +1,13 @@ """Helpers for working with extensions to DIRAC""" import argparse -from collections import defaultdict import fnmatch -from importlib.machinery import PathFinder import functools +import importlib import os import pkgutil import sys +from collections import defaultdict +from importlib.machinery import PathFinder import importlib_metadata as metadata import importlib_resources @@ -28,7 +29,7 @@ def newFunc(modules, *args, **kwargs): results = set() for module in modules: if isinstance(module, str): - module = import_module(module) + module = importlib.import_module(module) results |= set(func(module, *args, **kwargs)) return sorted(results) @@ -138,12 +139,12 @@ def getExtensionMetadata(extensionName): def recurseImport(modName, parentModule=None, hideExceptions=False): - from DIRAC import S_OK, S_ERROR, gLogger + from DIRAC import S_ERROR, S_OK, gLogger if parentModule is not None: raise NotImplementedError(parentModule) try: - return S_OK(import_module(modName)) + return S_OK(importlib.import_module(modName)) except ImportError as excp: # name of the module reported as not found notFoundModule = excp.name @@ -168,7 +169,7 @@ def _findFile(module, submoduleName, pattern="*"): """Implementation of findDatabases""" for system in _findSystems(module): try: - dbModule = resources.files(".".join([module.__name__, system.name, submoduleName])) + dbModule = importlib_resources.files(".".join([module.__name__, system.name, submoduleName])) except ImportError: continue for file in dbModule.iterdir(): @@ -187,7 +188,7 @@ def parseArgs(): subparser.set_defaults(func=func) args = parser.parse_args() # Get the result and print it - extensions = [import_module(e) for e in args.extensions] + extensions = [importlib.import_module(e) for e in args.extensions] for result in args.func(extensions): if not isinstance(result, str): result = " ".join(result) diff --git a/src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py b/src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py index 0626927e03e..b0ea97a7ba4 100644 --- a/src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py +++ b/src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py @@ -47,69 +47,64 @@ /LocalInstallation/VirtualOrganization: Name of the main Virtual Organization (default: None) """ -from collections import defaultdict import glob -from importlib import metadata, resources, import_module +import importlib import inspect -import io import os import pkgutil import re import shutil import stat +import subprocess import time from collections import defaultdict import importlib_metadata as metadata import importlib_resources -import subprocess +import MySQLdb from diraccfg import CFG from prompt_toolkit import prompt import DIRAC -from DIRAC import rootPath -from DIRAC import gConfig -from DIRAC import gLogger -from DIRAC.Core.Utilities.Subprocess import systemCall -from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR - -from DIRAC.Core.Utilities.Version import getVersion -from DIRAC.Core.Utilities.File import mkDir, mkLink +from DIRAC import gConfig, gLogger, rootPath +from DIRAC.ConfigurationSystem.Client import PathFinder from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI from DIRAC.ConfigurationSystem.Client.Helpers import ( - cfgPath, - cfgPathToList, + CSGlobals, cfgInstallPath, cfgInstallSection, - CSGlobals, + cfgPath, + cfgPathToList, ) +from DIRAC.Core.Base.AgentModule import AgentModule +from DIRAC.Core.Base.ExecutorModule import ExecutorModule +from DIRAC.Core.Base.private.ModuleLoader import ModuleLoader +from DIRAC.Core.DISET.RequestHandler import RequestHandler from DIRAC.Core.Security.Properties import ( ALARMS_MANAGEMENT, - SERVICE_ADMINISTRATOR, CS_ADMINISTRATOR, - JOB_ADMINISTRATOR, FULL_DELEGATION, - PROXY_MANAGEMENT, - OPERATOR, + JOB_ADMINISTRATOR, NORMAL_USER, + OPERATOR, + PROXY_MANAGEMENT, + SERVICE_ADMINISTRATOR, TRUSTED_HOST, ) - -from DIRAC.ConfigurationSystem.Client import PathFinder -from DIRAC.Core.Utilities.MySQL import MySQL -from DIRAC.Core.Base.private.ModuleLoader import ModuleLoader -from DIRAC.Core.Base.AgentModule import AgentModule -from DIRAC.Core.Base.ExecutorModule import ExecutorModule -from DIRAC.Core.DISET.RequestHandler import RequestHandler -from DIRAC.Core.Utilities.PrettyPrint import printTable from DIRAC.Core.Utilities.Extensions import ( extensionsByPriority, + findAgents, findDatabases, + findExecutors, findModules, - findAgents, findServices, - findExecutors, ) +from DIRAC.Core.Utilities.File import mkDir, mkLink +from DIRAC.Core.Utilities.MySQL import MySQL +from DIRAC.Core.Utilities.PrettyPrint import printTable +from DIRAC.Core.Utilities.ReturnValues import S_ERROR, S_OK +from DIRAC.Core.Utilities.Subprocess import systemCall +from DIRAC.Core.Utilities.Version import getVersion from DIRAC.FrameworkSystem.Client.ComponentMonitoringClient import ComponentMonitoringClient @@ -798,7 +793,7 @@ def getComponentCfg( for ext in extensions: cfgTemplateModule = f"{ext}.{system}System" try: - cfgTemplate = resources.read_text(cfgTemplateModule, "ConfigTemplate.cfg") + cfgTemplate = importlib_resources.read_text(cfgTemplateModule, "ConfigTemplate.cfg") except (ImportError, OSError): continue gLogger.notice("Loading configuration template from", cfgTemplateModule) @@ -2066,7 +2061,7 @@ def getAvailableESDatabases(self, extensions): # Introspect all possible ones for a ElasticDB attribute try: - module = import_module(".".join([extension, systemName, "DB", dbName])) + module = importlib.import_module(".".join([extension, systemName, "DB", dbName])) dbClass = getattr(module, dbName) except (AttributeError, ImportError): continue @@ -2229,7 +2224,7 @@ def _createMySQLCMDLines(self, dbSql): sourcedDBbFileName = line.split(" ")[1].replace("\n", "") gLogger.info("Found file to source: %s" % sourcedDBbFileName) module, filename = sourcedDBbFileName.rsplit("/", 1) - dbSourced = resources.read_text(module.replace("/", "."), filename) + dbSourced = importlib_resources.read_text(module.replace("/", "."), filename) for lineSourced in dbSourced.split("\n"): if lineSourced.strip(): cmdLines.append(lineSourced.strip()) diff --git a/src/DIRAC/RequestManagementSystem/private/RequestTask.py b/src/DIRAC/RequestManagementSystem/private/RequestTask.py index 892ea005905..27ede694788 100644 --- a/src/DIRAC/RequestManagementSystem/private/RequestTask.py +++ b/src/DIRAC/RequestManagementSystem/private/RequestTask.py @@ -11,16 +11,15 @@ request processing task to be used inside ProcessTask created in RequestExecutingAgent """ -from importlib import import_module import os import time +from importlib import import_module -from DIRAC import gLogger, S_OK, S_ERROR, gConfig -from DIRAC.Core.Utilities import DErrno, TimeUtilities +from DIRAC import S_ERROR, S_OK, gConfig, gLogger from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData -from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.ConfigurationSystem.Client.Helpers import Registry -from DIRAC.Core.Utilities import Network +from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations +from DIRAC.Core.Utilities import DErrno, Network, TimeUtilities from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager from DIRAC.MonitoringSystem.Client.MonitoringReporter import MonitoringReporter from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient @@ -29,7 +28,6 @@ from DIRAC.WorkloadManagementSystem.Client.JobMonitoringClient import JobMonitoringClient - class RequestTask: """ .. class:: RequestTask diff --git a/src/DIRAC/WorkloadManagementSystem/Client/InputDataResolution.py b/src/DIRAC/WorkloadManagementSystem/Client/InputDataResolution.py index 37db0740b0a..f2d64787051 100644 --- a/src/DIRAC/WorkloadManagementSystem/Client/InputDataResolution.py +++ b/src/DIRAC/WorkloadManagementSystem/Client/InputDataResolution.py @@ -7,7 +7,7 @@ for applications. """ import DIRAC -from DIRAC import S_OK, S_ERROR, gLogger +from DIRAC import S_OK, S_ERROR, gLogger, gConfig from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader from DIRAC.WorkloadManagementSystem.Client.PoolXMLSlice import PoolXMLSlice from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations diff --git a/src/DIRAC/WorkloadManagementSystem/Executor/JobPath.py b/src/DIRAC/WorkloadManagementSystem/Executor/JobPath.py index 0061fa5fc15..546a9c73ffc 100755 --- a/src/DIRAC/WorkloadManagementSystem/Executor/JobPath.py +++ b/src/DIRAC/WorkloadManagementSystem/Executor/JobPath.py @@ -5,9 +5,7 @@ Initially this takes jobs in the received state and starts the jobs on the optimizer chain. """ -from DIRAC import S_OK, S_ERROR -from DIRAC.Core.Utilities import List -from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader +from DIRAC import S_OK from DIRAC.WorkloadManagementSystem.Executor.Base.OptimizerExecutor import OptimizerExecutor From fc22a084a8f6376e00357d6c1c4d97ae5636ed37 Mon Sep 17 00:00:00 2001 From: DIRACGrid CI Date: Thu, 29 Sep 2022 07:55:25 +0000 Subject: [PATCH 0017/1135] docs: Add release notes for v8.1.0a1 --- release.notes | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/release.notes b/release.notes index c9829330815..cc658f1fe7f 100644 --- a/release.notes +++ b/release.notes @@ -1,3 +1,24 @@ +[v8.1.0a1] + +*WorkloadManagementSystem + +FIX: (#6388) replace opPath by opChain in JobPath + +*DataManagementSystem + +FIX: (#6385) Monitoring does not update the dict using by the accounting, avoiding a crash +NEW: (#6385) FailedDataOperation monitoring now keeps track of interactive/job failures +NEW: (#6383) cancel the FTS transfers when an RMS request is found to be canceled + +*Core + +FIX: (#6379) Explicitly depend on db12 +CHANGE: (#6337) Use Python's default SSL ciphers by default + +*Resources + +CHANGE: (#6378) negative free space value is transformed to 0 + [v8.0.0] *Resources From b3603554ea8ddca6fc12e1aefaf26e34c9c63d88 Mon Sep 17 00:00:00 2001 From: Andre Sailer Date: Thu, 29 Sep 2022 11:52:21 +0200 Subject: [PATCH 0018/1135] sweep: #6391 Fix for FileCatalog DirectoryLevelTree --- src/DIRAC/Core/Utilities/MySQL.py | 4 ++-- .../DirectoryManager/DirectoryLevelTree.py | 24 +++++++++---------- .../DB/PilotAgentsDB.py | 2 +- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/DIRAC/Core/Utilities/MySQL.py b/src/DIRAC/Core/Utilities/MySQL.py index 310ab68dad7..f7054e65044 100755 --- a/src/DIRAC/Core/Utilities/MySQL.py +++ b/src/DIRAC/Core/Utilities/MySQL.py @@ -29,7 +29,7 @@ Returns S_OK or S_ERROR. - _query( cmd, [conn] ) + _query( cmd, [conn=conn] ) Executes SQL command "cmd". Gets a connection from the Queue (or open a new one if none is available), @@ -39,7 +39,7 @@ Returns S_OK with fetchall() out in Value or S_ERROR upon failure. - _update( cmd, [conn] ) + _update( cmd, [conn=conn] ) Executes SQL command "cmd" and issue a commit Gets a connection from the Queue (or open a new one if none is available), diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryLevelTree.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryLevelTree.py index 3116126a951..cec86cf0af5 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryLevelTree.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryLevelTree.py @@ -141,10 +141,10 @@ def makeDir(self, path): result = self.db._getConnection() conn = result["Value"] - # result = self.db._query("LOCK TABLES FC_DirectoryLevelTree WRITE; ",conn) - result = self.db.insertFields("FC_DirectoryLevelTree", names, values, conn) + # result = self.db._query("LOCK TABLES FC_DirectoryLevelTree WRITE; ", conn=conn) + result = self.db.insertFields("FC_DirectoryLevelTree", names, values, conn=conn) if not result["OK"]: - # resUnlock = self.db._query("UNLOCK TABLES;",conn) + # resUnlock = self.db._query("UNLOCK TABLES;", conn=conn) if result["Message"].find("Duplicate") != -1: # The directory is already added resFind = self.findDir(path) @@ -162,25 +162,25 @@ def makeDir(self, path): if parentDirID: # lPath = "LPATH%d" % (level) # req = " SELECT @tmpvar:=max(%s)+1 FROM FC_DirectoryLevelTree WHERE Parent=%d; " % (lPath,parentDirID) - # resultLock = self.db._query("LOCK TABLES FC_DirectoryLevelTree WRITE; ",conn) - # result = self.db._query(req,conn) + # resultLock = self.db._query("LOCK TABLES FC_DirectoryLevelTree WRITE; ", conn=conn) + # result = self.db._query(req, conn=conn) # req = "UPDATE FC_DirectoryLevelTree SET %s=@tmpvar WHERE DirID=%d; " % (lPath,dirID) - # result = self.db._update(req,conn) - # result = self.db._query("UNLOCK TABLES;",conn) + # result = self.db._update(req, conn=conn) + # result = self.db._query("UNLOCK TABLES;", conn=conn) lPath = "LPATH%d" % (level) req = " SELECT @tmpvar:=max(%s)+1 FROM FC_DirectoryLevelTree WHERE Parent=%d FOR UPDATE; " % ( lPath, parentDirID, ) - resultLock = self.db._query("START TRANSACTION; ", conn) - result = self.db._query(req, conn) + resultLock = self.db._query("START TRANSACTION; ", conn=conn) + result = self.db._query(req, conn=conn) req = "UPDATE FC_DirectoryLevelTree SET %s=@tmpvar WHERE DirID=%d; " % (lPath, dirID) - result = self.db._update(req, conn) - result = self.db._query("COMMIT;", conn) + result = self.db._update(req, conn=conn) + result = self.db._query("COMMIT;", conn=conn) if not result["OK"]: return result else: - result = self.db._query("ROLLBACK;", conn) + result = self.db._query("ROLLBACK;", conn=conn) result = S_OK(dirID) result["NewDirectory"] = True diff --git a/src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.py b/src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.py index 8efb897119d..33f6f28d344 100755 --- a/src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.py +++ b/src/DIRAC/WorkloadManagementSystem/DB/PilotAgentsDB.py @@ -136,7 +136,7 @@ def setPilotStatus( # req = reqBase + ','.join("('%s', '%s', '%s')" % (pilotJobReference, status, statusReason)) # req += " ON DUPLICATE KEY UPDATE Status=VALUES(Status),StatusReason=VALUES(StatusReason)" - # return self._update(req, conn) + # return self._update(req, conn=conn) ########################################################################################## def selectPilots( From a70221f8a71eedf2cd8c527dd97af8f839a22cb5 Mon Sep 17 00:00:00 2001 From: Andre Sailer Date: Mon, 3 Oct 2022 11:39:54 +0200 Subject: [PATCH 0019/1135] sweep: #6394 fixes for rescheduling --- src/DIRAC/Core/Utilities/TimeUtilities.py | 5 +++++ src/DIRAC/WorkloadManagementSystem/Executor/JobScheduling.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/DIRAC/Core/Utilities/TimeUtilities.py b/src/DIRAC/Core/Utilities/TimeUtilities.py index 2369dbe3ac3..8595ad1076f 100755 --- a/src/DIRAC/Core/Utilities/TimeUtilities.py +++ b/src/DIRAC/Core/Utilities/TimeUtilities.py @@ -152,7 +152,12 @@ def fromString(myDate=None): The format of the string it is assume to be that returned by toString method. See notice on toString method On Error, return None + + :param myDate: the date string to be converted + :type myDate: str or datetime.datetime """ + if isinstance(myDate, datetime.datetime): + return myDate if isinstance(myDate, str): if myDate.find(" ") > 0: dateTimeTuple = myDate.split(" ") diff --git a/src/DIRAC/WorkloadManagementSystem/Executor/JobScheduling.py b/src/DIRAC/WorkloadManagementSystem/Executor/JobScheduling.py index b6a47ad2163..1296aa6989b 100755 --- a/src/DIRAC/WorkloadManagementSystem/Executor/JobScheduling.py +++ b/src/DIRAC/WorkloadManagementSystem/Executor/JobScheduling.py @@ -60,7 +60,7 @@ def optimizeJob(self, jid, jobState): return S_ERROR("RescheduleCounter has to be an integer") if reschedules != 0: delays = self.ex_getOption("RescheduleDelays", [60, 180, 300, 600]) - delay = delays[min(reschedules, len(delays) - 1)] + delay = delays[min(reschedules - 1, len(delays) - 1)] waited = toEpoch() - toEpoch(fromString(attDict["RescheduleTime"])) if waited < delay: return self.__holdJob(jobState, "On Hold: after rescheduling %s" % reschedules, delay) From 28088a3beec6cdfc0b399b1379d2751d38674e9c Mon Sep 17 00:00:00 2001 From: Simon Fayer Date: Mon, 3 Oct 2022 11:44:09 +0200 Subject: [PATCH 0020/1135] sweep: #6396 Use safer mode for grid-security directories --- src/DIRAC/Core/Utilities/File.py | 13 ++++++++++--- src/DIRAC/Core/scripts/dirac_configure.py | 4 ++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/DIRAC/Core/Utilities/File.py b/src/DIRAC/Core/Utilities/File.py index 8dfacf4ae5e..0ab7c982eed 100755 --- a/src/DIRAC/Core/Utilities/File.py +++ b/src/DIRAC/Core/Utilities/File.py @@ -24,12 +24,19 @@ } -def mkDir(path): - """Emulate 'mkdir -p path' (if path exists already, don't raise an exception)""" +def mkDir(path, mode=None): + """Emulate 'mkdir -p path' (if path exists already, don't raise an exception) + + :param str path: directory hierarchy to create + :param int mode: Use this mode as the mode for new directories, use python default if None. + """ try: if os.path.isdir(path): return - os.makedirs(path) + if mode is None: + os.makedirs(path) + else: + os.makedirs(path, mode) except OSError as osError: if osError.errno == errno.EEXIST and os.path.isdir(path): pass diff --git a/src/DIRAC/Core/scripts/dirac_configure.py b/src/DIRAC/Core/scripts/dirac_configure.py index f64477ad190..a37bd23adbe 100755 --- a/src/DIRAC/Core/scripts/dirac_configure.py +++ b/src/DIRAC/Core/scripts/dirac_configure.py @@ -517,7 +517,7 @@ def runDiracConfigure(params): Script.enableCS() try: dirName = os.path.join(DIRAC.rootPath, "etc", "grid-security", "certificates") - mkDir(dirName) + mkDir(dirName, 0o755) except Exception: DIRAC.gLogger.exception() DIRAC.gLogger.fatal("Fail to create directory:", dirName) @@ -640,7 +640,7 @@ def runDiracConfigure(params): vomsDirPath = os.path.join(DIRAC.rootPath, "etc", "grid-security", "vomsdir", voName) vomsesDirPath = os.path.join(DIRAC.rootPath, "etc", "grid-security", "vomses") for path in (vomsDirPath, vomsesDirPath): - mkDir(path) + mkDir(path, 0o755) vomsesLines = [] for vomsHost in vomsDict[vo].get("Servers", {}): hostFilePath = os.path.join(vomsDirPath, "%s.lsc" % vomsHost) From d403d01df3117bc0cf74dde4d133729f2284ed59 Mon Sep 17 00:00:00 2001 From: fstagni Date: Wed, 12 Oct 2022 14:56:47 +0200 Subject: [PATCH 0021/1135] sweep: #6401 Add command to complete productions --- setup.cfg | 1 + src/DIRAC/ProductionSystem/DB/ProductionDB.py | 1 + .../Utilities/StateMachine.py | 11 ++++--- .../scripts/dirac_prod_complete.py | 33 +++++++++++++++++++ .../Client/TransformationClient.py | 10 ++++++ .../Service/TransformationManagerHandler.py | 8 +++++ 6 files changed, 59 insertions(+), 5 deletions(-) create mode 100644 src/DIRAC/ProductionSystem/scripts/dirac_prod_complete.py diff --git a/setup.cfg b/setup.cfg index 3e58a8866d6..30ba8409997 100644 --- a/setup.cfg +++ b/setup.cfg @@ -240,6 +240,7 @@ console_scripts = dirac-wms-select-jobs = DIRAC.Interfaces.scripts.dirac_wms_select_jobs:main # ProductionSystem dirac-prod-add-trans = DIRAC.ProductionSystem.scripts.dirac_prod_add_trans:main [admin] + dirac-prod-complete = DIRAC.ProductionSystem.scripts.dirac_prod_complete:main [admin] dirac-prod-clean = DIRAC.ProductionSystem.scripts.dirac_prod_clean:main [admin] dirac-prod-delete = DIRAC.ProductionSystem.scripts.dirac_prod_delete:main [admin] dirac-prod-get = DIRAC.ProductionSystem.scripts.dirac_prod_get:main [admin] diff --git a/src/DIRAC/ProductionSystem/DB/ProductionDB.py b/src/DIRAC/ProductionSystem/DB/ProductionDB.py index 5a4ae53e1b6..7757876edb0 100644 --- a/src/DIRAC/ProductionSystem/DB/ProductionDB.py +++ b/src/DIRAC/ProductionSystem/DB/ProductionDB.py @@ -73,6 +73,7 @@ def __init__(self, dbname=None, dbconfig=None, dbIn=None, parentLogger=None): "New": None, "Active": "startTransformation", "Stopped": "stopTransformation", + "Completed": "completeTransformation", "Cleaned": "cleanTransformation", } diff --git a/src/DIRAC/ProductionSystem/Utilities/StateMachine.py b/src/DIRAC/ProductionSystem/Utilities/StateMachine.py index 099962e927b..9ab8e21df8f 100644 --- a/src/DIRAC/ProductionSystem/Utilities/StateMachine.py +++ b/src/DIRAC/ProductionSystem/Utilities/StateMachine.py @@ -15,14 +15,15 @@ def __init__(self, state): # States transitions self.states = { - "Cleaned": State(4), # final state + "Cleaned": State(5), # final state # The Cleaning State should be added later. This State implies that there is an agent checking if # all the associated transformations are cleaned and which automatically updates the State to # Cleaned. For the moment the transition is directly from Stopped to Cleaned. # 'Cleaning' : State( 3, ['Cleaned'] ), - "Stopped": State(3, ["Active", "Flush", "Cleaned"], defState="Active"), - "Flush": State(2, ["Active", "Cleaned"], defState="Active"), - "Active": State(1, ["Flush", "Stopped", "Cleaned"], defState="Flush"), + "Completed": State(4, ["Cleaned"], defState="Completed"), + "Stopped": State(3, ["Active", "Flush", "Cleaned"], defState="Stopped"), + "Flush": State(2, ["Active", "Cleaned"], defState="Flush"), + "Active": State(1, ["Flush", "Stopped", "Cleaned"], defState="Active"), # initial state - "New": State(0, ["Active", "Cleaned"], defState="Active"), + "New": State(0, ["Active", "Cleaned"], defState="New"), } diff --git a/src/DIRAC/ProductionSystem/scripts/dirac_prod_complete.py b/src/DIRAC/ProductionSystem/scripts/dirac_prod_complete.py new file mode 100644 index 00000000000..8a1dbd02db7 --- /dev/null +++ b/src/DIRAC/ProductionSystem/scripts/dirac_prod_complete.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python +""" +Complete a given production + +Example: + $ dirac-prod-complete 312 +""" +import DIRAC +from DIRAC.Core.Base.Script import Script + + +@Script() +def main(): + # Registering arguments will automatically add their description to the help menu + Script.registerArgument("prodID: Production ID") + _, args = Script.parseCommandLine() + + from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient + + # get arguments + prodID = args[0] + + res = ProductionClient().setProductionStatus(prodID, "Completed") + if not res["OK"]: + DIRAC.gLogger.error(res["Message"]) + DIRAC.exit(1) + + DIRAC.gLogger.notice("Production %s successully completed" % prodID) + DIRAC.exit(0) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/TransformationSystem/Client/TransformationClient.py b/src/DIRAC/TransformationSystem/Client/TransformationClient.py index 7ed48187a33..05c9f348eaa 100644 --- a/src/DIRAC/TransformationSystem/Client/TransformationClient.py +++ b/src/DIRAC/TransformationSystem/Client/TransformationClient.py @@ -271,6 +271,16 @@ def getTransformationTasks( break return S_OK(transformationTasks) + def completeTransformation(self, transID): + """Complete the transformation, and set the status parameter (doing it here, for easier extensibility)""" + # Complete + rpcClient = self._getRPC() + res = rpcClient.completeTransformation(transID) + if not res["OK"]: + return res + # Setting the status + return self.setTransformationParameter(transID, "Status", "Completed") + def cleanTransformation(self, transID): """Clean the transformation, and set the status parameter (doing it here, for easier extensibility)""" # Cleaning diff --git a/src/DIRAC/TransformationSystem/Service/TransformationManagerHandler.py b/src/DIRAC/TransformationSystem/Service/TransformationManagerHandler.py index 863183edba1..965ae875b09 100644 --- a/src/DIRAC/TransformationSystem/Service/TransformationManagerHandler.py +++ b/src/DIRAC/TransformationSystem/Service/TransformationManagerHandler.py @@ -104,6 +104,14 @@ def export_deleteTransformation(self, transName): # authorDN = self._clientTransport.peerCredentials['DN'] return self.transformationDB.deleteTransformation(transName, author=authorDN) + types_completeTransformation = [[int, str]] + + def export_completeTransformation(self, transName): + credDict = self.getRemoteCredentials() + authorDN = credDict.get("DN", credDict.get("CN")) + # authorDN = self._clientTransport.peerCredentials['DN'] + return self.transformationDB.setTransformationParameter(transName, "Status", "Completed", author=authorDN) + types_cleanTransformation = [[int, str]] def export_cleanTransformation(self, transName): From 06726a8b691f85b7ca5ac034b2e62dbe3968f3e2 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Wed, 15 Jun 2022 16:46:30 +0200 Subject: [PATCH 0022/1135] refactor: removed setup parameter from WMS --- src/DIRAC/Interfaces/API/Dirac.py | 2 +- src/DIRAC/Interfaces/API/Job.py | 5 - .../scripts/dirac_wms_job_attributes.py | 1 - .../scripts/dirac_wms_job_get_jdl.py | 1 - .../TransformationSystem/test/Test_JobInfo.py | 4 - .../Client/JobState/JobManifest.py | 6 +- .../Client/SandboxStoreClient.py | 30 +- .../Test_Client_WorkloadManagementSystem.py | 2 - .../WorkloadManagementSystem/DB/JobDB.py | 28 +- .../WorkloadManagementSystem/DB/JobDB.sql | 2 - .../DB/SandboxMetadataDB.py | 68 ++- .../DB/TaskQueueDB.py | 38 +- .../Executor/JobSanity.py | 7 +- .../JobWrapper/Watchdog.py | 3 - .../Service/JobManagerHandler.py | 2 - .../Service/MatcherHandler.py | 1 - .../Service/SandboxStoreHandler.py | 20 +- .../Service/TornadoJobManagerHandler.py | 1 - .../Utilities/QueueUtilities.py | 2 - .../scripts/dirac_admin_show_task_queues.py | 1 - .../WorkloadManagementSystem/Test_JobDB.py | 30 +- .../Test_TaskQueueDB.py | 400 ++++++++---------- 22 files changed, 259 insertions(+), 395 deletions(-) diff --git a/src/DIRAC/Interfaces/API/Dirac.py b/src/DIRAC/Interfaces/API/Dirac.py index 812f4c5d0ed..0d6fce98381 100755 --- a/src/DIRAC/Interfaces/API/Dirac.py +++ b/src/DIRAC/Interfaces/API/Dirac.py @@ -2253,7 +2253,7 @@ def getJobAttributes(self, jobID, printOutput=False): >>> print dirac.getJobAttributes(79241) {'AccountedFlag': 'False','ApplicationNumStatus': '0', 'ApplicationStatus': 'Job Finished Successfully', - 'CPUTime': '0.0','DIRACSetup': 'LHCb-Production'} + 'CPUTime': '0.0'} :param jobID: JobID :type jobID: int, str or python:list diff --git a/src/DIRAC/Interfaces/API/Job.py b/src/DIRAC/Interfaces/API/Job.py index 353d0a25525..6c8798ae845 100755 --- a/src/DIRAC/Interfaces/API/Job.py +++ b/src/DIRAC/Interfaces/API/Job.py @@ -1090,11 +1090,6 @@ def _toJDL(self, xmlFile="", jobDescriptionObject=None): arguments.append("-o LogLevel=%s" % (paramsDict["LogLevel"]["value"])) else: self.log.warn("Job LogLevel defined with null value") - if "DIRACSetup" in paramsDict: - if paramsDict["DIRACSetup"]["value"]: - arguments.append("-o DIRAC/Setup=%s" % (paramsDict["DIRACSetup"]["value"])) - else: - self.log.warn("Job DIRACSetup defined with null value") if "JobConfigArgs" in paramsDict: if paramsDict["JobConfigArgs"]["value"]: arguments.append("--cfg %s" % (paramsDict["JobConfigArgs"]["value"])) diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_attributes.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_attributes.py index b68fbe1cafb..b349155187d 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_wms_job_attributes.py +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_attributes.py @@ -12,7 +12,6 @@ 'ApplicationNumStatus': '0', 'ApplicationStatus': 'Unknown', 'CPUTime': '0.0', - 'DIRACSetup': 'EELA-Production', 'DeletedFlag': 'False', 'EndExecTime': '2011-02-14 11:28:01', 'FailedFlag': 'False', diff --git a/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_jdl.py b/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_jdl.py index 57e06d9ae7e..fc2484f25d4 100755 --- a/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_jdl.py +++ b/src/DIRAC/Interfaces/scripts/dirac_wms_job_get_jdl.py @@ -16,7 +16,6 @@ $ dirac-wms-job-get-jdl 1 {'Arguments': '-ltrA', 'CPUTime': '86400', - 'DIRACSetup': 'EELA-Production', 'Executable': '/bin/ls', 'JobID': '1', 'JobName': 'DIRAC_vhamar_602138', diff --git a/src/DIRAC/TransformationSystem/test/Test_JobInfo.py b/src/DIRAC/TransformationSystem/test/Test_JobInfo.py index a097ee69874..03016590a2d 100644 --- a/src/DIRAC/TransformationSystem/test/Test_JobInfo.py +++ b/src/DIRAC/TransformationSystem/test/Test_JobInfo.py @@ -78,7 +78,6 @@ def setUp(self): "JobType": "MCReconstruction_Overlay", "GridEnv": "/cvmfs/grid.cern.ch/emi-ui-3.7.3-1_sl6v2/etc/profile.d/setup-emi3-ui-example", "TransformationID": 6326, - "DIRACSetup": "ILC-Production", "StdError": "std.err", "IS_PROD": "True", "OwnerDN": "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=sailer/CN=683529/CN=Andre Sailer", @@ -146,7 +145,6 @@ def setUp(self): "JobType": "MCReconstruction_Overlay", "GridEnv": "/cvmfs/grid.cern.ch/emi-ui-3.7.3-1_sl6v2/etc/profile.d/setup-emi3-ui-example", "TransformationID": 6326, - "DIRACSetup": "ILC-Production", "StdError": "std.err", "IS_PROD": "True", "OwnerDN": "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=sailer/CN=683529/CN=Andre Sailer", @@ -218,7 +216,6 @@ def setUp(self): "JobType": "MCSimulation", "GridEnv": "/cvmfs/grid.cern.ch/emi-ui-3.7.3-1_sl6v2/etc/profile.d/setup-emi3-ui-example", "TransformationID": 6301, - "DIRACSetup": "ILC-Production", "StdError": "std.err", "IS_PROD": "True", "OwnerDN": "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=sailer/CN=683529/CN=Andre Sailer", @@ -276,7 +273,6 @@ def setUp(self): "JobType": "MCGeneration", "GridEnv": "/cvmfs/grid.cern.ch/emi-ui-3.7.3-1_sl6v2/etc/profile.d/setup-emi3-ui-example", "TransformationID": 6498, - "DIRACSetup": "ILC-Production", "StdError": "std.err", "IS_PROD": "True", "OwnerDN": "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=sailer/CN=683529/CN=Andre Sailer", diff --git a/src/DIRAC/WorkloadManagementSystem/Client/JobState/JobManifest.py b/src/DIRAC/WorkloadManagementSystem/Client/JobState/JobManifest.py index d8827975bb0..ebd7acff409 100644 --- a/src/DIRAC/WorkloadManagementSystem/Client/JobState/JobManifest.py +++ b/src/DIRAC/WorkloadManagementSystem/Client/JobState/JobManifest.py @@ -67,7 +67,7 @@ def dumpAsJDL(self): def __getCSValue(self, varName, defaultVal=None): if not self.__ops: - self.__ops = Operations(group=self.__manifest["OwnerGroup"], setup=self.__manifest["DIRACSetup"]) + self.__ops = Operations(group=self.__manifest["OwnerGroup"]) if varName[0] != "/": varName = "JobDescription/%s" % varName return self.__ops.getValue(varName, defaultVal) @@ -154,9 +154,9 @@ def check(self): """ Check that the manifest is OK """ - for k in ["Owner", "OwnerDN", "OwnerGroup", "DIRACSetup"]: + for k in ["Owner", "OwnerDN", "OwnerGroup"]: if k not in self.__manifest: - return S_ERROR("Missing var %s in manifest" % k) + return S_ERROR(f"Missing var {k} in manifest") # Check CPUTime result = self.__checkNumericalVar("CPUTime", 86400, 100, 500000) diff --git a/src/DIRAC/WorkloadManagementSystem/Client/SandboxStoreClient.py b/src/DIRAC/WorkloadManagementSystem/Client/SandboxStoreClient.py index 557243c04cb..75709003446 100755 --- a/src/DIRAC/WorkloadManagementSystem/Client/SandboxStoreClient.py +++ b/src/DIRAC/WorkloadManagementSystem/Client/SandboxStoreClient.py @@ -9,7 +9,7 @@ import re from io import BytesIO, StringIO -from DIRAC import gLogger, S_OK, S_ERROR, gConfig +from DIRAC import gLogger, S_OK, S_ERROR from DIRAC.Core.Tornado.Client.ClientSelector import TransferClientSelector as TransferClient from DIRAC.Core.Base.Client import Client @@ -250,13 +250,13 @@ def getSandboxesForJob(self, jobId): """Download job sandbox""" return self.__getSandboxesForEntity("Job:%s" % jobId) - def assignSandboxesToJob(self, jobId, sbList, ownerName="", ownerGroup="", eSetup=""): + def assignSandboxesToJob(self, jobId, sbList, ownerName="", ownerGroup=""): """Assign SB to a job""" - return self.__assignSandboxesToEntity("Job:%s" % jobId, sbList, ownerName, ownerGroup, eSetup) + return self.__assignSandboxesToEntity("Job:%s" % jobId, sbList, ownerName, ownerGroup) - def assignSandboxToJob(self, jobId, sbLocation, sbType, ownerName="", ownerGroup="", eSetup=""): + def assignSandboxToJob(self, jobId, sbLocation, sbType, ownerName="", ownerGroup=""): """Assign SB to a job""" - return self.__assignSandboxToEntity("Job:%s" % jobId, sbLocation, sbType, ownerName, ownerGroup, eSetup) + return self.__assignSandboxToEntity("Job:%s" % jobId, sbLocation, sbType, ownerName, ownerGroup) def unassignJobs(self, jobIdList): """Unassign SB to a job""" @@ -300,13 +300,13 @@ def getSandboxesForPilot(self, pilotId): """Get SB for a pilot""" return self.__getSandboxesForEntity("Pilot:%s" % pilotId) - def assignSandboxesToPilot(self, pilotId, sbList, ownerName="", ownerGroup="", eSetup=""): + def assignSandboxesToPilot(self, pilotId, sbList, ownerName="", ownerGroup=""): """Assign SB to a pilot""" - return self.__assignSandboxesToEntity("Pilot:%s" % pilotId, sbList, ownerName, ownerGroup, eSetup) + return self.__assignSandboxesToEntity("Pilot:%s" % pilotId, sbList, ownerName, ownerGroup) - def assignSandboxToPilot(self, pilotId, sbLocation, sbType, ownerName="", ownerGroup="", eSetup=""): + def assignSandboxToPilot(self, pilotId, sbLocation, sbType, ownerName="", ownerGroup=""): """Assign SB to a pilot""" - return self.__assignSandboxToEntity("Pilot:%s" % pilotId, sbLocation, sbType, ownerName, ownerGroup, eSetup) + return self.__assignSandboxToEntity("Pilot:%s" % pilotId, sbLocation, sbType, ownerName, ownerGroup) def unassignPilots(self, pilotIdIdList): """Unassign SB to a pilot""" @@ -344,7 +344,7 @@ def __getSandboxesForEntity(self, eId): rpcClient = self.__getRPCClient() return rpcClient.getSandboxesAssignedToEntity(eId) - def __assignSandboxesToEntity(self, eId, sbList, ownerName="", ownerGroup="", eSetup=""): + def __assignSandboxesToEntity(self, eId, sbList, ownerName="", ownerGroup=""): """ Assign sandboxes to a job. sbList must be a list of sandboxes and relation types @@ -354,19 +354,17 @@ def __assignSandboxesToEntity(self, eId, sbList, ownerName="", ownerGroup="", eS if sbT[1] not in self.__validSandboxTypes: return S_ERROR("Invalid Sandbox type %s" % sbT[1]) if SandboxStoreClient.__smdb and ownerName and ownerGroup: - if not eSetup: - eSetup = gConfig.getValue("/DIRAC/Setup", "Production") - return SandboxStoreClient.__smdb.assignSandboxesToEntities({eId: sbList}, ownerName, ownerGroup, eSetup) + return SandboxStoreClient.__smdb.assignSandboxesToEntities({eId: sbList}, ownerName, ownerGroup) rpcClient = self.__getRPCClient() - return rpcClient.assignSandboxesToEntities({eId: sbList}, ownerName, ownerGroup, eSetup) + return rpcClient.assignSandboxesToEntities({eId: sbList}, ownerName, ownerGroup) - def __assignSandboxToEntity(self, eId, sbLocation, sbType, ownerName="", ownerGroup="", eSetup=""): + def __assignSandboxToEntity(self, eId, sbLocation, sbType, ownerName="", ownerGroup=""): """ Assign a sandbox to a job sbLocation is "SEName:SEPFN" sbType is Input or Output """ - return self.__assignSandboxesToEntity(eId, [(sbLocation, sbType)], ownerName, ownerGroup, eSetup) + return self.__assignSandboxesToEntity(eId, [(sbLocation, sbType)], ownerName, ownerGroup) def __unassignEntities(self, eIdList): """ diff --git a/src/DIRAC/WorkloadManagementSystem/Client/test/Test_Client_WorkloadManagementSystem.py b/src/DIRAC/WorkloadManagementSystem/Client/test/Test_Client_WorkloadManagementSystem.py index cc66eb29fa9..87a8bedf301 100644 --- a/src/DIRAC/WorkloadManagementSystem/Client/test/Test_Client_WorkloadManagementSystem.py +++ b/src/DIRAC/WorkloadManagementSystem/Client/test/Test_Client_WorkloadManagementSystem.py @@ -62,14 +62,12 @@ def test__processResourceDescription(setUp): "PilotReference": "somePilotReference", "Platform": "x86_64-slc6", "ReleaseVersion": "v8r0p1", - "Setup": "LHCb-Certification", "Site": "DIRAC.Jenkins.ch", "WaitingToRunningRatio": 0.05, } res = matcher._processResourceDescription(resourceDescription) resExpected = { - "Setup": "LHCb-Certification", "ReleaseVersion": "v8r0p1", "CPUTime": 1080000, "DIRACVersion": "v8r0p1", diff --git a/src/DIRAC/WorkloadManagementSystem/DB/JobDB.py b/src/DIRAC/WorkloadManagementSystem/DB/JobDB.py index 52947a80abd..743f0814261 100755 --- a/src/DIRAC/WorkloadManagementSystem/DB/JobDB.py +++ b/src/DIRAC/WorkloadManagementSystem/DB/JobDB.py @@ -910,7 +910,6 @@ def insertNewJobIntoDB( owner, ownerDN, ownerGroup, - diracSetup, initialStatus=JobStatus.RECEIVED, initialMinorStatus="Job accepted", ): @@ -922,7 +921,6 @@ def insertNewJobIntoDB( :param str owner: job owner user name :param str ownerDN: job owner DN :param str ownerGroup: job owner group - :param str diracSetup: setup in which context the job is submitted :param str initialStatus: optional initial job status (Received by default) :param str initialMinorStatus: optional initial minor job status :return: new job ID @@ -931,9 +929,7 @@ def insertNewJobIntoDB( result = jobManifest.load(jdl) if not result["OK"]: return result - jobManifest.setOptionsFromDict( - {"Owner": owner, "OwnerDN": ownerDN, "OwnerGroup": ownerGroup, "DIRACSetup": diracSetup} - ) + jobManifest.setOptionsFromDict({"Owner": owner, "OwnerDN": ownerDN, "OwnerGroup": ownerGroup}) result = jobManifest.check() if not result["OK"]: return result @@ -969,9 +965,6 @@ def insertNewJobIntoDB( jobAttrNames.append("OwnerGroup") jobAttrValues.append(ownerGroup) - jobAttrNames.append("DIRACSetup") - jobAttrValues.append(diracSetup) - # 2.- Check JDL and Prepare DIRAC JDL jobJDL = jobManifest.dumpAsJDL() @@ -1000,7 +993,7 @@ def insertNewJobIntoDB( classAdJob.insertAttributeInt("JobID", jobID) result = self.__checkAndPrepareJob( - jobID, classAdJob, classAdReq, owner, ownerDN, ownerGroup, diracSetup, jobAttrNames, jobAttrValues + jobID, classAdJob, classAdReq, owner, ownerDN, ownerGroup, jobAttrNames, jobAttrValues ) if not result["OK"]: return result @@ -1088,7 +1081,7 @@ def insertNewJobIntoDB( return retVal def __checkAndPrepareJob( - self, jobID, classAdJob, classAdReq, owner, ownerDN, ownerGroup, diracSetup, jobAttrNames, jobAttrValues + self, jobID, classAdJob, classAdReq, owner, ownerDN, ownerGroup, jobAttrNames, jobAttrValues ): """ Check Consistency of Submitted JDL and set some defaults @@ -1097,16 +1090,11 @@ def __checkAndPrepareJob( error = "" vo = getVOForGroup(ownerGroup) - jdlDiracSetup = classAdJob.getAttributeString("DIRACSetup") jdlOwner = classAdJob.getAttributeString("Owner") jdlOwnerDN = classAdJob.getAttributeString("OwnerDN") jdlOwnerGroup = classAdJob.getAttributeString("OwnerGroup") jdlVO = classAdJob.getAttributeString("VirtualOrganization") - # The below is commented out since this is always overwritten by the submitter IDs - # but the check allows to findout inconsistent client environments - if jdlDiracSetup and jdlDiracSetup != diracSetup: - error = "Wrong DIRAC Setup in JDL" if jdlOwner and jdlOwner != owner: error = "Wrong Owner in JDL" elif jdlOwnerDN and jdlOwnerDN != ownerDN: @@ -1123,7 +1111,6 @@ def __checkAndPrepareJob( if vo: classAdJob.insertAttributeString("VirtualOrganization", vo) - classAdReq.insertAttributeString("Setup", diracSetup) classAdReq.insertAttributeString("OwnerDN", ownerDN) classAdReq.insertAttributeString("OwnerGroup", ownerGroup) if vo: @@ -1142,7 +1129,7 @@ def __checkAndPrepareJob( # CPU time cpuTime = classAdJob.getAttributeInt("CPUTime") if cpuTime is None: - opsHelper = Operations(group=ownerGroup, setup=diracSetup) + opsHelper = Operations(group=ownerGroup) cpuTime = opsHelper.getValue("JobDescription/DefaultCPUTime", 86400) classAdReq.insertAttributeInt("CPUTime", cpuTime) @@ -1252,7 +1239,6 @@ def rescheduleJob(self, jobID): "Owner", "OwnerDN", "OwnerGroup", - "DIRACSetup", ], ) if result["OK"]: @@ -1333,7 +1319,6 @@ def rescheduleJob(self, jobID): resultDict["Owner"], resultDict["OwnerDN"], resultDict["OwnerGroup"], - resultDict["DIRACSetup"], jobAttrNames, jobAttrValues, ) @@ -2013,14 +1998,13 @@ def getSummarySnapshot(self, requestedFields=False): """Get the summary snapshot for a given combination""" if not requestedFields: requestedFields = ["Status", "MinorStatus", "Site", "Owner", "OwnerGroup", "JobGroup", "JobSplitType"] - defFields = ["DIRACSetup"] + requestedFields valueFields = ["COUNT(JobID)", "SUM(RescheduleCounter)"] - defString = ", ".join(defFields) + defString = ", ".join(requestedFields) valueString = ", ".join(valueFields) result = self._query(f"SELECT {defString}, {valueString} FROM Jobs GROUP BY {defString}") if not result["OK"]: return result - return S_OK(((defFields + valueFields), result["Value"])) + return S_OK(((requestedFields + valueFields), result["Value"])) def removeInfoFromHeartBeatLogging(self, status, delTime, maxLines): """Remove HeartBeatLoggingInfo from DB. diff --git a/src/DIRAC/WorkloadManagementSystem/DB/JobDB.sql b/src/DIRAC/WorkloadManagementSystem/DB/JobDB.sql index ab9fb397fc4..b5736313bad 100755 --- a/src/DIRAC/WorkloadManagementSystem/DB/JobDB.sql +++ b/src/DIRAC/WorkloadManagementSystem/DB/JobDB.sql @@ -35,7 +35,6 @@ DROP TABLE IF EXISTS `Jobs`; CREATE TABLE `Jobs` ( `JobID` INT(11) UNSIGNED NOT NULL DEFAULT 0, `JobType` VARCHAR(32) NOT NULL DEFAULT 'user', - `DIRACSetup` VARCHAR(32) NOT NULL DEFAULT 'test', `JobGroup` VARCHAR(32) NOT NULL DEFAULT '00000000', `JobSplitType` ENUM('Single','Master','Subjob','DAGNode') NOT NULL DEFAULT 'Single', `MasterJobID` INT(11) UNSIGNED NOT NULL DEFAULT 0, @@ -68,7 +67,6 @@ CREATE TABLE `Jobs` ( PRIMARY KEY (`JobID`), FOREIGN KEY (`JobID`) REFERENCES `JobJDLs`(`JobID`), KEY `JobType` (`JobType`), - KEY `DIRACSetup` (`DIRACSetup`), KEY `JobGroup` (`JobGroup`), KEY `JobSplitType` (`JobSplitType`), KEY `Site` (`Site`), diff --git a/src/DIRAC/WorkloadManagementSystem/DB/SandboxMetadataDB.py b/src/DIRAC/WorkloadManagementSystem/DB/SandboxMetadataDB.py index 7f5c6badc5d..70dd018de12 100644 --- a/src/DIRAC/WorkloadManagementSystem/DB/SandboxMetadataDB.py +++ b/src/DIRAC/WorkloadManagementSystem/DB/SandboxMetadataDB.py @@ -59,12 +59,11 @@ def __initializeDB(self): self.__tablesDesc["sb_EntityMapping"] = { "Fields": { "SBId": "INTEGER(10) UNSIGNED NOT NULL", - "EntitySetup": "VARCHAR(64) NOT NULL", "EntityId": "VARCHAR(128) NOT NULL", "Type": "VARCHAR(64) NOT NULL", }, - "Indexes": {"Entity": ["EntityId", "EntitySetup"], "SBIndex": ["SBId"]}, - "UniqueIndexes": {"Mapping": ["SBId", "EntitySetup", "EntityId", "Type"]}, + "Indexes": {"Entity": ["EntityId"], "SBIndex": ["SBId"]}, + "UniqueIndexes": {"Mapping": ["SBId", "EntityId", "Type"]}, } for tableName in self.__tablesDesc: @@ -135,7 +134,7 @@ def registerAndGetSandbox(self, owner, ownerDN, ownerGroup, sbSE, sbPFN, size=0) if not result["OK"]: return result if not result["Value"]: - return S_ERROR("SandBox already exists but doesn't belong to the user or setup") + return S_ERROR("SandBox already exists but doesn't belong to the user") sbId = result["Value"][0][0] self.accessedSandboxById(sbId) return S_OK((sbId, False)) @@ -157,7 +156,7 @@ def __accessedSandboxByCond(self, condDict): sqlCond = [f"{key}={condDict[key]}" for key in condDict] return self._update("UPDATE `sb_SandBoxes` SET LastAccessTime=UTC_TIMESTAMP() WHERE %s" % " AND ".join(sqlCond)) - def assignSandboxesToEntities(self, enDict, requesterName, requesterGroup, enSetup, ownerName="", ownerGroup=""): + def assignSandboxesToEntities(self, enDict, requesterName, requesterGroup, ownerName="", ownerGroup=""): """ Assign jobs to entities """ @@ -186,13 +185,13 @@ def assignSandboxesToEntities(self, enDict, requesterName, requesterGroup, enSet return S_ERROR("SB Location has to have SEName|SEPFN form") SEName = splitted[0] SEPFN = ":".join(splitted[1:]) - entitiesToSandboxList.append((entityId, enSetup, sbTuple[1], SEName, SEPFN)) + entitiesToSandboxList.append((entityId, sbTuple[1], SEName, SEPFN)) if not entitiesToSandboxList: return S_OK() sbIds = [] assigned = 0 - for entityId, entitySetup, SBType, SEName, SEPFN in entitiesToSandboxList: + for entityId, SBType, SEName, SEPFN in entitiesToSandboxList: result = self.getSandboxId(SEName, SEPFN, requesterName, requesterGroup) insertValues = [] if not result["OK"]: @@ -204,10 +203,9 @@ def assignSandboxesToEntities(self, enDict, requesterName, requesterGroup, enSet sbId = result["Value"] sbIds.append(str(sbId)) insertValues.append( - "( %s, %s, %s, %d )" + "( %s, %s, %d )" % ( self._escapeString(entityId)["Value"], - self._escapeString(entitySetup)["Value"], self._escapeString(SBType)["Value"], sbId, ) @@ -218,9 +216,7 @@ def assignSandboxesToEntities(self, enDict, requesterName, requesterGroup, enSet "Sandbox does not exist or you're not authorized to assign it being %s@%s" % (requesterName, requesterGroup) ) - sqlCmd = "INSERT INTO `sb_EntityMapping` ( entityId, entitySetup, Type, SBId ) VALUES %s" % ", ".join( - insertValues - ) + sqlCmd = "INSERT INTO `sb_EntityMapping` ( entityId, Type, SBId ) VALUES %s" % ", ".join(insertValues) result = self._update(sqlCmd) if not result["OK"]: if result["Message"].find("Duplicate entry") == -1: @@ -232,11 +228,11 @@ def assignSandboxesToEntities(self, enDict, requesterName, requesterGroup, enSet return result return S_OK(assigned) - def __filterEntitiesByRequester(self, entitiesList, entitiesSetup, requesterName, requesterGroup): + def __filterEntitiesByRequester(self, entitiesList, requesterName, requesterGroup): """ Given a list of entities and a requester, return the ones that the requester is allowed to modify """ - sqlCond = ["s.OwnerId=o.OwnerId", "s.SBId=e.SBId", "e.EntitySetup=%s" % entitiesSetup] + sqlCond = ["s.OwnerId=o.OwnerId", "s.SBId=e.SBId"] requesterProps = Registry.getPropertiesForEntity(requesterGroup, name=requesterName) if Properties.JOB_ADMINISTRATOR in requesterProps: # Do nothing, just ensure it doesn't fit in the other cases @@ -261,35 +257,32 @@ def __filterEntitiesByRequester(self, entitiesList, entitiesSetup, requesterName return result return S_OK([row[0] for row in result["Value"]]) - def unassignEntities(self, entitiesDict, requesterName, requesterGroup): + def unassignEntities(self, entities, requesterName, requesterGroup): """ Unassign jobs to sandboxes - entitiesDict = { 'setup' : [ 'entityId', 'entityId' ] } + entities = ['entityId_1', 'entityId_2'] """ updated = 0 - for entitySetup in entitiesDict: - entitiesIds = entitiesDict[entitySetup] - if not entitiesIds: - continue - escapedSetup = self._escapeString(entitySetup)["Value"] - result = self.__filterEntitiesByRequester(entitiesIds, escapedSetup, requesterName, requesterGroup) - if not result["OK"]: - gLogger.error("Cannot filter entities", result["Message"]) - continue - ids = result["Value"] - if not ids: - return S_OK(0) - sqlCond = ["EntitySetup = %s" % escapedSetup] - sqlCond.append("EntityId in ( %s )" % ", ".join(["'%s'" % str(eid) for eid in ids])) - sqlCmd = "DELETE FROM `sb_EntityMapping` WHERE %s" % " AND ".join(sqlCond) - result = self._update(sqlCmd) - if not result["OK"]: - gLogger.error("Cannot unassign entities", result["Message"]) - else: - updated += 1 + if not entities: + return S_OK() + result = self.__filterEntitiesByRequester(entities, requesterName, requesterGroup) + if not result["OK"]: + gLogger.error("Cannot filter entities", result["Message"]) + return result + ids = result["Value"] + if not ids: + return S_OK(0) + sqlCmd = "DELETE FROM `sb_EntityMapping` WHERE EntityId in ( %s )" % ", ".join( + ["'%s'" % str(eid) for eid in ids] + ) + result = self._update(sqlCmd) + if not result["OK"]: + gLogger.error("Cannot unassign entities", result["Message"]) + else: + updated += 1 return S_OK(updated) - def getSandboxesAssignedToEntity(self, entityId, entitySetup, requesterName, requesterGroup): + def getSandboxesAssignedToEntity(self, entityId, requesterName, requesterGroup): """ Get the sandboxes and the type of assignation to the jobId """ @@ -297,7 +290,6 @@ def getSandboxesAssignedToEntity(self, entityId, entitySetup, requesterName, req sqlCond = [ "s.SBId = e.SBId", "e.EntityId = %s" % self._escapeString(entityId)["Value"], - "e.EntitySetup = %s" % self._escapeString(entitySetup)["Value"], ] requesterProps = Registry.getPropertiesForEntity(requesterGroup, name=requesterName) if Properties.JOB_ADMINISTRATOR in requesterProps or Properties.JOB_MONITOR in requesterProps: diff --git a/src/DIRAC/WorkloadManagementSystem/DB/TaskQueueDB.py b/src/DIRAC/WorkloadManagementSystem/DB/TaskQueueDB.py index c025d5fd045..76c46285e9c 100755 --- a/src/DIRAC/WorkloadManagementSystem/DB/TaskQueueDB.py +++ b/src/DIRAC/WorkloadManagementSystem/DB/TaskQueueDB.py @@ -17,13 +17,13 @@ TQ_MIN_SHARE = 0.001 # For checks at insertion time, and not only -singleValueDefFields = ("OwnerDN", "OwnerGroup", "Setup", "CPUTime") +singleValueDefFields = ("OwnerDN", "OwnerGroup", "CPUTime") multiValueDefFields = ("Sites", "GridCEs", "BannedSites", "Platforms", "JobTypes", "Tags") # Used for matching multiValueMatchFields = ("GridCE", "Site", "Platform", "JobType", "Tag") bannedJobMatchFields = ("Site",) -mandatoryMatchFields = ("Setup", "CPUTime") +mandatoryMatchFields = ("CPUTime",) priorityIgnoredFields = ("Sites", "BannedSites") @@ -101,13 +101,12 @@ def __initializeDB(self): "TQId": "INTEGER(11) UNSIGNED AUTO_INCREMENT NOT NULL", "OwnerDN": "VARCHAR(255) NOT NULL", "OwnerGroup": "VARCHAR(32) NOT NULL", - "Setup": "VARCHAR(32) NOT NULL", "CPUTime": "BIGINT(20) UNSIGNED NOT NULL", "Priority": "FLOAT NOT NULL", "Enabled": "TINYINT(1) NOT NULL DEFAULT 0", }, "PrimaryKey": "TQId", - "Indexes": {"TQOwner": ["OwnerDN", "OwnerGroup", "Setup", "CPUTime"]}, + "Indexes": {"TQOwner": ["OwnerDN", "OwnerGroup", "CPUTime"]}, } self.__tablesDesc["tq_Jobs"] = { @@ -652,19 +651,14 @@ def __generateNotDictSQL(self, negativeCond): condList = [] for field in negativeCond: if field in multiValueMatchFields: - fullTableN = "`tq_TQTo%ss`" % field + fullTableN = f"`tq_TQTo{field}s`" valList = negativeCond[field] if not isinstance(valList, (list, tuple)): valList = (valList,) subList = [] for value in valList: value = self._escapeString(value)["Value"] - sql = "{} NOT IN ( SELECT {}.Value FROM {} WHERE {}.TQId = tq.TQId )".format( - value, - fullTableN, - fullTableN, - fullTableN, - ) + sql = f"{value} NOT IN ( SELECT {fullTableN}.Value FROM {fullTableN} WHERE {fullTableN}.TQId = tq.TQId )" subList.append(sql) condList.append("( %s )" % " AND ".join(subList)) elif field in singleValueDefFields: @@ -676,7 +670,7 @@ def __generateNotDictSQL(self, negativeCond): @staticmethod def __generateTablesName(sqlTables, field): - fullTableName = "tq_TQTo%ss" % field + fullTableName = f"tq_TQTo{field}s" if fullTableName not in sqlTables: tableN = field.lower() sqlTables[fullTableName] = tableN @@ -719,18 +713,14 @@ def __generateTQMatchSQL(self, tqMatchDict, numQueuesToGet=1, negativeCond=None) if field in tqMatchDict: sqlCondList.append(self.__generateSQLSubCond("tq.%s = %%s" % field, tqMatchDict[field])) # Type of single value conditions - for field in ("CPUTime", "Setup"): - if field in tqMatchDict: - if field == "CPUTime": - sqlCondList.append(self.__generateSQLSubCond("tq.%s <= %%s" % field, tqMatchDict[field])) - else: - sqlCondList.append(self.__generateSQLSubCond("tq.%s = %%s" % field, tqMatchDict[field])) + if "CPUTime" in tqMatchDict: + sqlCondList.append(self.__generateSQLSubCond("tq.%s <= %%s" % "CPUTime", tqMatchDict["CPUTime"])) tag_fv = [] # Match multi value fields for field in multiValueMatchFields: - self.log.debug("Evaluating field %s" % field) + self.log.debug(f"Evaluating field {field}") # It has to be %ss , with an 's' at the end because the columns names # are plural and match options are singular @@ -794,7 +784,7 @@ def __generateTQMatchSQL(self, tqMatchDict, numQueuesToGet=1, negativeCond=None) # In case of Site, check it's not in job banned sites if field in bannedJobMatchFields: - fullTableN = "`tq_TQToBanned%ss`" % field + fullTableN = f"`tq_TQToBanned{field}s`" csql = self.__generateSQLSubCond( "%%s not in ( SELECT %s.Value \ FROM %s \ @@ -821,7 +811,7 @@ def __generateTQMatchSQL(self, tqMatchDict, numQueuesToGet=1, negativeCond=None) # Add possibly Resource banning conditions for field in multiValueMatchFields: - bannedField = "Banned%s" % field + bannedField = f"Banned{field}" # Is there something to consider? b_fv = tqMatchDict.get(bannedField) @@ -834,13 +824,11 @@ def __generateTQMatchSQL(self, tqMatchDict, numQueuesToGet=1, negativeCond=None) ): continue - fullTableN = "`tq_TQTo%ss`" % field + fullTableN = f"`tq_TQTo{field}s`" sqlCondList.append( self.__generateSQLSubCond( - "%%s not in ( SELECT %s.Value \ - FROM %s \ - WHERE %s.TQId = tq.TQId )" + "%%s not in ( SELECT %s.Value FROM %s WHERE %s.TQId = tq.TQId )" % (fullTableN, fullTableN, fullTableN), b_fv, boolOp="OR", diff --git a/src/DIRAC/WorkloadManagementSystem/Executor/JobSanity.py b/src/DIRAC/WorkloadManagementSystem/Executor/JobSanity.py index 2b97706298a..629c5facb9f 100755 --- a/src/DIRAC/WorkloadManagementSystem/Executor/JobSanity.py +++ b/src/DIRAC/WorkloadManagementSystem/Executor/JobSanity.py @@ -109,7 +109,7 @@ def checkInputSandbox(self, jobState, manifest): """The number of input sandbox files, as specified in the job JDL are checked in the JobDB. """ - result = jobState.getAttributes(["Owner", "OwnerDN", "OwnerGroup", "DIRACSetup"]) + result = jobState.getAttributes(["Owner", "OwnerDN", "OwnerGroup"]) if not result["OK"]: self.jobLog.error("Failed to get job attributes", result["Message"]) return result @@ -127,9 +127,6 @@ def checkInputSandbox(self, jobState, manifest): ownerGroup = attDict["OwnerGroup"] if not ownerGroup: return S_ERROR("Missing OwnerGroup") - jobSetup = attDict["DIRACSetup"] - if not jobSetup: - return S_ERROR("Missing DIRACSetup") isbList = manifest.getOption("InputSandbox", []) sbsToAssign = [] @@ -145,7 +142,7 @@ def checkInputSandbox(self, jobState, manifest): if not numSBsToAssign: return S_OK(0) self.jobLog.info("Assigning sandboxes", f"({numSBsToAssign} on behalf of {ownerName}@{ownerGroup})") - result = self.sandboxClient.assignSandboxesToJob(jobState.jid, sbsToAssign, ownerName, ownerGroup, jobSetup) + result = self.sandboxClient.assignSandboxesToJob(jobState.jid, sbsToAssign, ownerName, ownerGroup) if not result["OK"]: self.jobLog.error("Could not assign sandboxes in the SandboxStore") return S_ERROR("Cannot assign sandbox to job") diff --git a/src/DIRAC/WorkloadManagementSystem/JobWrapper/Watchdog.py b/src/DIRAC/WorkloadManagementSystem/JobWrapper/Watchdog.py index 80b6ce1a1af..208ef7a117d 100755 --- a/src/DIRAC/WorkloadManagementSystem/JobWrapper/Watchdog.py +++ b/src/DIRAC/WorkloadManagementSystem/JobWrapper/Watchdog.py @@ -108,9 +108,6 @@ def initialize(self): else: self.initialized = True - setup = gConfig.getValue("/DIRAC/Setup", "") - if not setup: - return S_ERROR("Can not get the DIRAC Setup value") wms_instance = getSystemInstance("WorkloadManagement") if not wms_instance: return S_ERROR("Can not get the WorkloadManagement system instance") diff --git a/src/DIRAC/WorkloadManagementSystem/Service/JobManagerHandler.py b/src/DIRAC/WorkloadManagementSystem/Service/JobManagerHandler.py index ac9470b8f5b..9d9508e1c9c 100755 --- a/src/DIRAC/WorkloadManagementSystem/Service/JobManagerHandler.py +++ b/src/DIRAC/WorkloadManagementSystem/Service/JobManagerHandler.py @@ -190,7 +190,6 @@ def export_submitJob(self, jobDesc): self.owner, self.ownerDN, self.ownerGroup, - self.diracSetup, initialStatus=initialStatus, initialMinorStatus=initialMinorStatus, ) @@ -665,5 +664,4 @@ def export_resetJob(self, jobIDs): class JobManagerHandler(JobManagerHandlerMixin, RequestHandler): def initialize(self): - self.diracSetup = self.serviceInfoDict["clientSetup"] return self.initializeRequest() diff --git a/src/DIRAC/WorkloadManagementSystem/Service/MatcherHandler.py b/src/DIRAC/WorkloadManagementSystem/Service/MatcherHandler.py index 075584f7d74..a3163026108 100755 --- a/src/DIRAC/WorkloadManagementSystem/Service/MatcherHandler.py +++ b/src/DIRAC/WorkloadManagementSystem/Service/MatcherHandler.py @@ -54,7 +54,6 @@ def export_requestJob(self, resourceDescription): one matching the agent's site capacity """ - resourceDescription["Setup"] = self.serviceInfoDict["clientSetup"] credDict = self.getRemoteCredentials() pilotRef = resourceDescription.get("PilotReference", "Unknown") diff --git a/src/DIRAC/WorkloadManagementSystem/Service/SandboxStoreHandler.py b/src/DIRAC/WorkloadManagementSystem/Service/SandboxStoreHandler.py index 888630cbe79..9f9ff071107 100755 --- a/src/DIRAC/WorkloadManagementSystem/Service/SandboxStoreHandler.py +++ b/src/DIRAC/WorkloadManagementSystem/Service/SandboxStoreHandler.py @@ -325,16 +325,14 @@ def __copyToExternalSE(self, localFilePath, sbPath): types_assignSandboxesToEntities = [dict] - def export_assignSandboxesToEntities(self, enDict, ownerName="", ownerGroup="", entitySetup=False): + def export_assignSandboxesToEntities(self, enDict, ownerName="", ownerGroup=""): """ Assign sandboxes to jobs. Expects a dict of { entityId : [ ( SB, SBType ), ... ] } """ - if not entitySetup: - entitySetup = self.serviceInfoDict["clientSetup"] credDict = self.getRemoteCredentials() return self.sandboxDB.assignSandboxesToEntities( - enDict, credDict["username"], credDict["group"], entitySetup, ownerName, ownerGroup + enDict, credDict["username"], credDict["group"], ownerName, ownerGroup ) ################## @@ -342,30 +340,24 @@ def export_assignSandboxesToEntities(self, enDict, ownerName="", ownerGroup="", types_unassignEntities = [(list, tuple)] - def export_unassignEntities(self, entitiesList, entitiesSetup=False): + def export_unassignEntities(self, entitiesList): """ Unassign a list of jobs """ - if not entitiesSetup: - entitiesSetup = self.serviceInfoDict["clientSetup"] credDict = self.getRemoteCredentials() - return self.sandboxDB.unassignEntities({entitiesSetup: entitiesList}, credDict["username"], credDict["group"]) + return self.sandboxDB.unassignEntities(entitiesList, credDict["username"], credDict["group"]) ################## # Getting assigned sandboxes types_getSandboxesAssignedToEntity = [str] - def export_getSandboxesAssignedToEntity(self, entityId, entitySetup=False): + def export_getSandboxesAssignedToEntity(self, entityId): """ Get the sandboxes associated to a job and the association type """ - if not entitySetup: - entitySetup = self.serviceInfoDict["clientSetup"] credDict = self.getRemoteCredentials() - result = self.sandboxDB.getSandboxesAssignedToEntity( - entityId, entitySetup, credDict["username"], credDict["group"] - ) + result = self.sandboxDB.getSandboxesAssignedToEntity(entityId, credDict["username"], credDict["group"]) if not result["OK"]: return result sbDict = {} diff --git a/src/DIRAC/WorkloadManagementSystem/Service/TornadoJobManagerHandler.py b/src/DIRAC/WorkloadManagementSystem/Service/TornadoJobManagerHandler.py index 35362e36295..81289478879 100644 --- a/src/DIRAC/WorkloadManagementSystem/Service/TornadoJobManagerHandler.py +++ b/src/DIRAC/WorkloadManagementSystem/Service/TornadoJobManagerHandler.py @@ -13,5 +13,4 @@ class TornadoJobManagerHandler(JobManagerHandlerMixin, TornadoService): def initializeRequest(self): - self.diracSetup = self.get_argument("clientSetup") return JobManagerHandlerMixin.initializeRequest(self) diff --git a/src/DIRAC/WorkloadManagementSystem/Utilities/QueueUtilities.py b/src/DIRAC/WorkloadManagementSystem/Utilities/QueueUtilities.py index 5bede8411c3..9ec9d3f6105 100644 --- a/src/DIRAC/WorkloadManagementSystem/Utilities/QueueUtilities.py +++ b/src/DIRAC/WorkloadManagementSystem/Utilities/QueueUtilities.py @@ -1,6 +1,5 @@ """Utilities to help Computing Element Queues manipulation """ -import os import hashlib from DIRAC import S_OK, S_ERROR @@ -9,7 +8,6 @@ from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getDIRACPlatform from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.Core.Security.ProxyInfo import getProxyInfo -from DIRAC.Core.Utilities.File import mkDir from DIRAC.Resources.Computing.ComputingElementFactory import ComputingElementFactory diff --git a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_show_task_queues.py b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_show_task_queues.py index 5eab0e74acf..f2d75e464a5 100755 --- a/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_show_task_queues.py +++ b/src/DIRAC/WorkloadManagementSystem/scripts/dirac_admin_show_task_queues.py @@ -11,7 +11,6 @@ OwnerDN: /O=GRID-FR/C=FR/O=CNRS/OU=CPPM/CN=Vanessa Hamar OwnerGroup: dirac_user Priority: 1.0 - Setup: Dirac-Production """ import sys diff --git a/tests/Integration/WorkloadManagementSystem/Test_JobDB.py b/tests/Integration/WorkloadManagementSystem/Test_JobDB.py index 2f3d3ed5651..5b9748c7e21 100644 --- a/tests/Integration/WorkloadManagementSystem/Test_JobDB.py +++ b/tests/Integration/WorkloadManagementSystem/Test_JobDB.py @@ -24,7 +24,6 @@ jdl = """[ Arguments = "jobDescription.xml -o LogLevel=info"; CPUTime = 86400; - DIRACSetup = "someSetup"; Executable = "dirac-jobexec"; InputData = ""; InputSandbox = @@ -84,10 +83,11 @@ def test_insertNewJobIntoDB(jobDB): """Test the insertNewJobIntoDB method""" # Act - res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup", "someSetup") + res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup") # Assert assert res["OK"], res["Message"] + jobID = int(res["JobID"]) res = jobDB.getJobAttribute(jobID, "Status") assert res["OK"], res["Message"] @@ -109,7 +109,7 @@ def test_insertNewJobIntoDB(jobDB): def test_removeJobFromDB(jobDB): # Arrange - res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup", "someSetup") + res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup") assert res["OK"], res["Message"] jobID = int(res["JobID"]) @@ -124,7 +124,7 @@ def test_getJobJDL_original(jobDB: JobDB): """Test of the getJobJDL method with the original parameter set to True""" # Arrange - res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup", "someSetup") + res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup") assert res["OK"], res["Message"] jobID = int(res["JobID"]) @@ -141,7 +141,7 @@ def test_getJobJDL_nonOriginal(jobDB: JobDB): """Test of the getJobJDL method with the original parameter set to True""" # Arrange - res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup", "someSetup") + res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup") assert res["OK"], res["Message"] jobID = int(res["JobID"]) @@ -155,7 +155,6 @@ def test_getJobJDL_nonOriginal(jobDB: JobDB): f"""[ Arguments = "jobDescription.xml -o LogLevel=info"; CPUTime = 86400; - DIRACSetup = "someSetup"; Executable = "dirac-jobexec"; InputData = ""; InputSandbox = @@ -173,7 +172,6 @@ def test_getJobJDL_nonOriginal(jobDB: JobDB): CPUTime = 86400; OwnerDN = "/DN/OF/owner"; OwnerGroup = "ownerGroup"; - Setup = "someSetup"; UserPriority = 1; VirtualOrganization = "vo"; ]; @@ -199,11 +197,11 @@ def test_getJobJDL_nonOriginal(jobDB: JobDB): def test_getJobsAttributes(jobDB): # Arrange - res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup", "someSetup") + res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup") assert res["OK"], res["Message"] jobID_1 = int(res["JobID"]) - res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup", "someSetup") + res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup") assert res["OK"], res["Message"] jobID_2 = int(res["JobID"]) @@ -221,7 +219,7 @@ def test_getJobsAttributes(jobDB): def test_rescheduleJob(jobDB): # Arrange - res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup", "someSetup") + res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup") assert res["OK"], res["Message"] jobID = res["JobID"] @@ -270,7 +268,7 @@ def test_getCounters(jobDB): def test_heartBeatLogging(jobDB): - res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup", "someSetup") + res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup") assert res["OK"], res["Message"] jobID = res["JobID"] @@ -312,7 +310,7 @@ def test_heartBeatLogging(jobDB): def test_getJobParameters(jobDB): - res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup", "someSetup") + res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup") assert res["OK"], res["Message"] jobID = res["JobID"] @@ -330,10 +328,10 @@ def test_getJobParameters(jobDB): def test_setJobsMajorStatus(jobDB): - res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup", "someSetup") + res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup") assert res["OK"], res["Message"] jobID_1 = res["JobID"] - res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup", "someSetup") + res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup") assert res["OK"], res["Message"] jobID_2 = res["JobID"] @@ -377,10 +375,10 @@ def test_setJobsMajorStatus(jobDB): def test_attributes(jobDB): - res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup", "someSetup") + res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup") assert res["OK"], res["Message"] jobID_1 = res["JobID"] - res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup", "someSetup") + res = jobDB.insertNewJobIntoDB(jdl, "owner", "/DN/OF/owner", "ownerGroup") assert res["OK"], res["Message"] jobID_2 = res["JobID"] diff --git a/tests/Integration/WorkloadManagementSystem/Test_TaskQueueDB.py b/tests/Integration/WorkloadManagementSystem/Test_TaskQueueDB.py index b97602d066a..4fd7073fe56 100644 --- a/tests/Integration/WorkloadManagementSystem/Test_TaskQueueDB.py +++ b/tests/Integration/WorkloadManagementSystem/Test_TaskQueueDB.py @@ -25,68 +25,67 @@ def test_basicChain(): """a basic put - remove""" - tqDefDict = {"OwnerDN": "/my/DN", "OwnerGroup": "myGroup", "Setup": "aSetup", "CPUTime": 50000} + tqDefDict = {"OwnerDN": "/my/DN", "OwnerGroup": "myGroup", "CPUTime": 50000} result = tqDB.insertJob(123, tqDefDict, 10) - assert result["OK"] is True + assert result["OK"] result = tqDB.getTaskQueueForJobs([123]) - assert result["OK"] is True + assert result["OK"] assert 123 in result["Value"] tq = result["Value"][123] result = tqDB.deleteJob(123) - assert result["OK"] is True + assert result["OK"] result = tqDB.cleanOrphanedTaskQueues() - assert result["OK"] is True + assert result["OK"] result = tqDB.deleteTaskQueueIfEmpty(tq) - assert result["OK"] is True + assert result["OK"] def test_chainWithParameter(): """put - remove with parameters""" - tqDefDict = {"OwnerDN": "/my/DN", "OwnerGroup": "myGroup", "Setup": "aSetup", "CPUTime": 50000} + tqDefDict = {"OwnerDN": "/my/DN", "OwnerGroup": "myGroup", "CPUTime": 50000} # first job result = tqDB.insertJob(123, tqDefDict, 10) - assert result["OK"] is True + assert result["OK"] result = tqDB.getTaskQueueForJobs([123]) - assert result["OK"] is True + assert result["OK"] tq = result["Value"][123] result = tqDB.deleteTaskQueue(tq) assert result["OK"] is False # This will fail because of the foreign key result = tqDB.cleanOrphanedTaskQueues() - assert result["OK"] is True + assert result["OK"] result = tqDB.deleteTaskQueueIfEmpty(tq) # this won't delete anything - assert result["OK"] is True + assert result["OK"] # second job result = tqDB.insertJob(125, tqDefDict, 10) - assert result["OK"] is True + assert result["OK"] result = tqDB.getTaskQueueForJobs([125]) tq = result["Value"][125] result = tqDB.deleteTaskQueue(tq) assert result["OK"] is False # This will fail because of the foreign key result = tqDB.deleteTaskQueueIfEmpty(tq) # this won't delete anything, as both 123 and 125 are in - assert result["OK"] is True # but still it won't fail + assert result["OK"] # but still it won't fail assert result["Value"] is False result = tqDB.retrieveTaskQueues() - assert result["OK"] is True + assert result["OK"] assert list(result["Value"].values())[0] == { "OwnerDN": "/my/DN", "Jobs": 2, "OwnerGroup": "myGroup", - "Setup": "aSetup", "CPUTime": 86400, "Priority": 1.0, } # now we will try to delete result = tqDB.deleteJob(123) - assert result["OK"] is True + assert result["OK"] result = tqDB.deleteJob(125) - assert result["OK"] is True + assert result["OK"] result = tqDB.deleteTaskQueueIfEmpty(tq) # this should now delete tq - assert result["OK"] is True + assert result["OK"] result = tqDB.retrieveTaskQueues() - assert result["OK"] is True + assert result["OK"] assert result["Value"] == {} @@ -95,80 +94,73 @@ def test_chainWithSites(): tqDefDict = { "OwnerDN": "/my/DN", "OwnerGroup": "myGroup", - "Setup": "aSetup", "CPUTime": 5000, "Sites": ["LCG.CERN.ch"], } result = tqDB.insertJob(201, tqDefDict, 10) - assert result["OK"] is True + assert result["OK"] result = tqDB.getTaskQueueForJobs([201]) tq_job1 = result["Value"][201] result = tqDB.insertJob(2011, tqDefDict, 10) - assert result["OK"] is True + assert result["OK"] result = tqDB.getTaskQueueForJobs([2011]) tq_job11 = result["Value"][2011] tqDefDict = { "OwnerDN": "/my/DN", "OwnerGroup": "myGroup", - "Setup": "aSetup", "CPUTime": 5000, "Sites": ["CLOUD.IN2P3.fr"], } result = tqDB.insertJob(203, tqDefDict, 10) - assert result["OK"] is True + assert result["OK"] result = tqDB.getTaskQueueForJobs([203]) tq_job2 = result["Value"][203] tqDefDict = { "OwnerDN": "/my/DN", "OwnerGroup": "myGroup", - "Setup": "aSetup", "CPUTime": 5000, "Sites": ["LCG.CERN.ch", "CLOUD.IN2P3.fr"], } result = tqDB.insertJob(203, tqDefDict, 10) - assert result["OK"] is True + assert result["OK"] result = tqDB.getTaskQueueForJobs([203]) tq_job3 = result["Value"][203] # matching # this should match everything - result = tqDB.matchAndGetTaskQueue({"Setup": "aSetup", "CPUTime": 50000}, numQueuesToGet=5) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000}, numQueuesToGet=5) + assert result["OK"] res = {int(x[0]) for x in result["Value"]} assert res == {tq_job1, tq_job2, tq_job3, tq_job11} # this should match those for CERN - result = tqDB.matchAndGetTaskQueue({"Setup": "aSetup", "CPUTime": 50000, "Site": "LCG.CERN.ch"}, numQueuesToGet=4) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000, "Site": "LCG.CERN.ch"}, numQueuesToGet=4) + assert result["OK"] res = {int(x[0]) for x in result["Value"]} assert res == {tq_job1, tq_job3, tq_job11} # this should match those for IN2P3 - result = tqDB.matchAndGetTaskQueue( - {"Setup": "aSetup", "CPUTime": 50000, "Site": "CLOUD.IN2P3.fr"}, numQueuesToGet=4 - ) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000, "Site": "CLOUD.IN2P3.fr"}, numQueuesToGet=4) + assert result["OK"] res = {int(x[0]) for x in result["Value"]} assert res == {tq_job2, tq_job3} # this should match those for IN2P3 - result = tqDB.matchAndGetTaskQueue( - {"Setup": "aSetup", "CPUTime": 50000, "Site": "CLOUD.IN2P3.fr"}, numQueuesToGet=4 - ) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000, "Site": "CLOUD.IN2P3.fr"}, numQueuesToGet=4) + assert result["OK"] res = {int(x[0]) for x in result["Value"]} assert res == {tq_job2, tq_job3} # now we will try to delete for jobID in [201, 2011, 202, 203]: result = tqDB.deleteJob(jobID) - assert result["OK"] is True + assert result["OK"] for tqID in [tq_job1, tq_job2, tq_job3]: result = tqDB.deleteTaskQueueIfEmpty(tqID) - assert result["OK"] is True + assert result["OK"] def test_chainWithBannedSites(): @@ -176,78 +168,72 @@ def test_chainWithBannedSites(): tqDefDict = { "OwnerDN": "/my/DN", "OwnerGroup": "myGroup", - "Setup": "aSetup", "CPUTime": 5000, "BannedSites": ["LCG.CERN.ch", "CLOUD.IN2P3.fr"], } result = tqDB.insertJob(127, tqDefDict, 10) - assert result["OK"] is True + assert result["OK"] result = tqDB.getTaskQueueForJobs([127]) tq_job1 = result["Value"][127] tqDefDict = { "OwnerDN": "/my/DN", "OwnerGroup": "myGroup", - "Setup": "aSetup", "CPUTime": 5000, "BannedSites": ["CLOUD.IN2P3.fr", "DIRAC.Test.org"], } result = tqDB.insertJob(128, tqDefDict, 10) - assert result["OK"] is True + assert result["OK"] result = tqDB.getTaskQueueForJobs([128]) tq_job2 = result["Value"][128] # matching # this should match everything - result = tqDB.matchAndGetTaskQueue({"Setup": "aSetup", "CPUTime": 50000}, numQueuesToGet=4) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000}, numQueuesToGet=4) + assert result["OK"] res = {int(x[0]) for x in result["Value"]} assert res == {tq_job1, tq_job2} # this should match also everything - result = tqDB.matchAndGetTaskQueue({"Setup": "aSetup", "CPUTime": 50000, "Platform": "centos7"}, numQueuesToGet=4) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000, "Platform": "centos7"}, numQueuesToGet=4) + assert result["OK"] res = {int(x[0]) for x in result["Value"]} assert res == {tq_job1, tq_job2} # this should match the first - result = tqDB.matchAndGetTaskQueue( - {"Setup": "aSetup", "CPUTime": 50000, "Site": "DIRAC.Test.org"}, numQueuesToGet=4 - ) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000, "Site": "DIRAC.Test.org"}, numQueuesToGet=4) + assert result["OK"] res = {int(x[0]) for x in result["Value"]} assert res == {tq_job1} # this should match the second - result = tqDB.matchAndGetTaskQueue({"Setup": "aSetup", "CPUTime": 50000, "Site": "LCG.CERN.ch"}, numQueuesToGet=4) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000, "Site": "LCG.CERN.ch"}, numQueuesToGet=4) + assert result["OK"] res = {int(x[0]) for x in result["Value"]} assert res == {tq_job2} # this should not match anything because of the banned site CLOUD.IN2P3.fr - result = tqDB.matchAndGetTaskQueue( - {"Setup": "aSetup", "CPUTime": 50000, "Site": "CLOUD.IN2P3.fr"}, numQueuesToGet=4 - ) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000, "Site": "CLOUD.IN2P3.fr"}, numQueuesToGet=4) + assert result["OK"] assert result["Value"] == [] result = tqDB.deleteTaskQueueIfEmpty(tq_job1) # this won't delete anything, as 127 is in - assert result["OK"] is True # but still it won't fail + assert result["OK"] # but still it won't fail assert result["Value"] is False result = tqDB.deleteJob(127) - assert result["OK"] is True + assert result["OK"] result = tqDB.deleteTaskQueueIfEmpty(tq_job1) # this should now delete tq - assert result["OK"] is True + assert result["OK"] result = tqDB.deleteJob(128) - assert result["OK"] is True + assert result["OK"] for tqId in [tq_job1, tq_job2]: result = tqDB.deleteTaskQueueIfEmpty(tqId) - assert result["OK"] is True + assert result["OK"] result = tqDB.retrieveTaskQueues() - assert result["OK"] is True + assert result["OK"] assert result["Value"] == {} @@ -265,18 +251,17 @@ def test_chainWithPlatforms(): tqDefDict = { "OwnerDN": "/my/DN", "OwnerGroup": "myGroup", - "Setup": "aSetup", "CPUTime": 5000, "Platforms": ["centos7"], } result = tqDB.insertJob(1, tqDefDict, 10) - assert result["OK"] is True + assert result["OK"] result = tqDB.getTaskQueueForJobs([1]) tq_job1 = result["Value"][1] assert tq_job1 > 0 result = tqDB.insertJob(2, tqDefDict, 10) - assert result["OK"] is True + assert result["OK"] result = tqDB.getTaskQueueForJobs([2]) tq_job2 = result["Value"][2] assert tq_job1 == tq_job2 @@ -284,12 +269,11 @@ def test_chainWithPlatforms(): tqDefDict = { "OwnerDN": "/my/DN", "OwnerGroup": "myGroup", - "Setup": "aSetup", "CPUTime": 5000, "Platforms": ["ubuntu"], } result = tqDB.insertJob(3, tqDefDict, 10) - assert result["OK"] is True + assert result["OK"] result = tqDB.getTaskQueueForJobs([3]) tq_job3 = result["Value"][3] assert tq_job3 == tq_job1 + 1 @@ -297,12 +281,11 @@ def test_chainWithPlatforms(): tqDefDict = { "OwnerDN": "/my/DN", "OwnerGroup": "myGroup", - "Setup": "aSetup", "CPUTime": 5000, "Platforms": ["centos7", "slc6"], } result = tqDB.insertJob(4, tqDefDict, 10) - assert result["OK"] is True + assert result["OK"] result = tqDB.getTaskQueueForJobs([4]) tq_job4 = result["Value"][4] assert tq_job4 == tq_job3 + 1 @@ -310,12 +293,11 @@ def test_chainWithPlatforms(): tqDefDict = { "OwnerDN": "/my/DN", "OwnerGroup": "myGroup", - "Setup": "aSetup", "CPUTime": 5000, "Platforms": ["debian", "ubuntu"], } result = tqDB.insertJob(5, tqDefDict, 10) - assert result["OK"] is True + assert result["OK"] result = tqDB.getTaskQueueForJobs([5]) tq_job5 = result["Value"][5] assert tq_job5 == tq_job4 + 1 @@ -348,46 +330,46 @@ def test_chainWithPlatforms(): # strict matching # centos7 - result = tqDB.matchAndGetTaskQueue({"Setup": "aSetup", "CPUTime": 50000, "Platform": "centos7"}, numQueuesToGet=4) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000, "Platform": "centos7"}, numQueuesToGet=4) + assert result["OK"] # this should match one in [tq_job1, tq_job2, tq_job4] res = {int(x[0]) for x in result["Value"]} assert res == {tq_job1, tq_job2, tq_job4} # ubuntu - result = tqDB.matchAndGetTaskQueue({"Setup": "aSetup", "CPUTime": 50000, "Platform": "ubuntu"}, numQueuesToGet=4) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000, "Platform": "ubuntu"}, numQueuesToGet=4) + assert result["OK"] # this should match one in [tq_job3, tq_job5] res = {int(x[0]) for x in result["Value"]} assert res == {tq_job3, tq_job5} # slc6 - result = tqDB.matchAndGetTaskQueue({"Setup": "aSetup", "CPUTime": 50000, "Platform": "slc6"}, numQueuesToGet=4) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000, "Platform": "slc6"}, numQueuesToGet=4) + assert result["OK"] # this should match only tq_job4, as this is the only one that can run on slc6 res = {int(x[0]) for x in result["Value"]} assert res == {tq_job4} # slc5 - result = tqDB.matchAndGetTaskQueue({"Setup": "aSetup", "CPUTime": 50000, "Platform": "slc5"}, numQueuesToGet=4) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000, "Platform": "slc5"}, numQueuesToGet=4) + assert result["OK"] # this should not match anything assert result["Value"] == [] # compatibility matching # ANY platform - result = tqDB.matchAndGetTaskQueue({"Setup": "aSetup", "CPUTime": 50000, "Platform": "ANY"}, numQueuesToGet=5) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000, "Platform": "ANY"}, numQueuesToGet=5) + assert result["OK"] # this should match whatever res = {int(x[0]) for x in result["Value"]} assert res == {tq_job1, tq_job2, tq_job3, tq_job4, tq_job5} # Now we insert a TQ without platform - tqDefDict = {"OwnerDN": "/my/DN", "OwnerGroup": "myGroup", "Setup": "aSetup", "CPUTime": 5000} + tqDefDict = {"OwnerDN": "/my/DN", "OwnerGroup": "myGroup", "CPUTime": 5000} result = tqDB.insertJob(6, tqDefDict, 10) - assert result["OK"] is True + assert result["OK"] result = tqDB.getTaskQueueForJobs([6]) tq_job6 = result["Value"][6] assert tq_job6 == tq_job5 + 1 @@ -395,59 +377,55 @@ def test_chainWithPlatforms(): # matching for this one # ANY platform - result = tqDB.matchAndGetTaskQueue({"Setup": "aSetup", "CPUTime": 50000, "Platform": "ANY"}, numQueuesToGet=6) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000, "Platform": "ANY"}, numQueuesToGet=6) + assert result["OK"] # this should match whatever res = {int(x[0]) for x in result["Value"]} assert res == {tq_job1, tq_job2, tq_job3, tq_job4, tq_job5, tq_job6} # ANY platform within a list - result = tqDB.matchAndGetTaskQueue({"Setup": "aSetup", "CPUTime": 50000, "Platform": ["ANY"]}, numQueuesToGet=6) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000, "Platform": ["ANY"]}, numQueuesToGet=6) + assert result["OK"] # this should match whatever res = {int(x[0]) for x in result["Value"]} assert res == {tq_job1, tq_job2, tq_job3, tq_job4, tq_job5, tq_job6} # no platform at all - result = tqDB.matchAndGetTaskQueue({"Setup": "aSetup", "CPUTime": 50000}, numQueuesToGet=6) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000}, numQueuesToGet=6) + assert result["OK"] # this should match whatever res = {int(x[0]) for x in result["Value"]} assert res == {tq_job1, tq_job2, tq_job3, tq_job4, tq_job5, tq_job6} # slc5 -- this time it should match 1 (the one without specified platform) - result = tqDB.matchAndGetTaskQueue({"Setup": "aSetup", "CPUTime": 50000, "Platform": "slc5"}, numQueuesToGet=6) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000, "Platform": "slc5"}, numQueuesToGet=6) + assert result["OK"] res = {int(x[0]) for x in result["Value"]} assert res == {tq_job6} # slc6 - result = tqDB.matchAndGetTaskQueue({"Setup": "aSetup", "CPUTime": 50000, "Platform": "slc6"}, numQueuesToGet=6) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000, "Platform": "slc6"}, numQueuesToGet=6) + assert result["OK"] res = {int(x[0]) for x in result["Value"]} assert res == {tq_job4, tq_job6} # slc5, slc6 - result = tqDB.matchAndGetTaskQueue( - {"Setup": "aSetup", "CPUTime": 50000, "Platform": ["slc5", "slc6"]}, numQueuesToGet=6 - ) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000, "Platform": ["slc5", "slc6"]}, numQueuesToGet=6) + assert result["OK"] res = {int(x[0]) for x in result["Value"]} assert res == {tq_job4, tq_job6} # slc5, slc6, ubuntu - result = tqDB.matchAndGetTaskQueue( - {"Setup": "aSetup", "CPUTime": 50000, "Platform": ["slc5", "slc6", "ubuntu"]}, numQueuesToGet=6 - ) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000, "Platform": ["slc5", "slc6", "ubuntu"]}, numQueuesToGet=6) + assert result["OK"] res = {int(x[0]) for x in result["Value"]} assert res == {tq_job3, tq_job4, tq_job5, tq_job6} # Now we insert a TQ with platform "ANY" (same as no platform) - tqDefDict = {"OwnerDN": "/my/DN", "OwnerGroup": "myGroup", "Setup": "aSetup", "CPUTime": 5000, "Platform": "ANY"} + tqDefDict = {"OwnerDN": "/my/DN", "OwnerGroup": "myGroup", "CPUTime": 5000, "Platform": "ANY"} result = tqDB.insertJob(7, tqDefDict, 10) - assert result["OK"] is True + assert result["OK"] result = tqDB.getTaskQueueForJobs([7]) tq_job7 = result["Value"][7] assert tq_job7 == tq_job6 # would be inserted in the same TQ @@ -455,36 +433,36 @@ def test_chainWithPlatforms(): # matching for this one # ANY platform - result = tqDB.matchAndGetTaskQueue({"Setup": "aSetup", "CPUTime": 50000, "Platform": "ANY"}, numQueuesToGet=7) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000, "Platform": "ANY"}, numQueuesToGet=7) + assert result["OK"] # this should match whatever res = {int(x[0]) for x in result["Value"]} assert res == {tq_job1, tq_job2, tq_job3, tq_job4, tq_job5, tq_job6, tq_job7} # NO platform - result = tqDB.matchAndGetTaskQueue({"Setup": "aSetup", "CPUTime": 50000}, numQueuesToGet=7) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000}, numQueuesToGet=7) + assert result["OK"] # this should match whatever assert int(result["Value"][0][0]) in [tq_job1, tq_job2, tq_job3, tq_job4, tq_job5, tq_job6, tq_job7] res = {int(x[0]) for x in result["Value"]} assert res == {tq_job1, tq_job2, tq_job3, tq_job4, tq_job5, tq_job6, tq_job7} # slc5 -- this time it should match 2 - result = tqDB.matchAndGetTaskQueue({"Setup": "aSetup", "CPUTime": 50000, "Platform": "slc5"}, numQueuesToGet=7) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000, "Platform": "slc5"}, numQueuesToGet=7) + assert result["OK"] res = {int(x[0]) for x in result["Value"]} assert res == {tq_job6, tq_job7} # slc6 - result = tqDB.matchAndGetTaskQueue({"Setup": "aSetup", "CPUTime": 50000, "Platform": "slc6"}, numQueuesToGet=7) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000, "Platform": "slc6"}, numQueuesToGet=7) + assert result["OK"] res = {int(x[0]) for x in result["Value"]} assert res == {tq_job4, tq_job6, tq_job7} # new platform appears # centos8 (> centos7) - result = tqDB.matchAndGetTaskQueue({"Setup": "aSetup", "CPUTime": 50000, "Platform": "centos8"}, numQueuesToGet=7) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000, "Platform": "centos8"}, numQueuesToGet=7) + assert result["OK"] # here, I would like to see 3 TQs matched: those for slc6 + centos7 + ANY assert len(result["Value"]) == 1 # but here it returns only 1 (those for ANY), by construction @@ -492,11 +470,11 @@ def test_chainWithPlatforms(): for jobId in range(1, 8): result = tqDB.deleteJob(jobId) - assert result["OK"] is True + assert result["OK"] for tqId in [tq_job1, tq_job2, tq_job3, tq_job4, tq_job5, tq_job6, tq_job7]: result = tqDB.deleteTaskQueueIfEmpty(tqId) - assert result["OK"] is True + assert result["OK"] def test_chainWithTags(): @@ -517,12 +495,11 @@ def test_chainWithTags(): tqDefDict = { "OwnerDN": "/my/DN", "OwnerGroup": "myGroup", - "Setup": "aSetup", "CPUTime": 5000, "Tags": ["MultiProcessor"], } result = tqDB.insertJob(1, tqDefDict, 10) - assert result["OK"] is True + assert result["OK"] result = tqDB.getTaskQueueForJobs([1]) tq_job1 = result["Value"][1] assert tq_job1 > 0 @@ -530,12 +507,11 @@ def test_chainWithTags(): tqDefDict = { "OwnerDN": "/my/DN", "OwnerGroup": "myGroup", - "Setup": "aSetup", "CPUTime": 5000, "Tags": ["SingleProcessor"], } result = tqDB.insertJob(2, tqDefDict, 10) - assert result["OK"] is True + assert result["OK"] result = tqDB.getTaskQueueForJobs([2]) tq_job2 = result["Value"][2] assert tq_job2 > tq_job1 @@ -543,12 +519,11 @@ def test_chainWithTags(): tqDefDict = { "OwnerDN": "/my/DN", "OwnerGroup": "myGroup", - "Setup": "aSetup", "CPUTime": 5000, "Tags": ["SingleProcessor", "MultiProcessor"], } result = tqDB.insertJob(3, tqDefDict, 10) - assert result["OK"] is True + assert result["OK"] result = tqDB.getTaskQueueForJobs([3]) tq_job3 = result["Value"][3] assert tq_job3 > tq_job2 @@ -556,19 +531,18 @@ def test_chainWithTags(): tqDefDict = { "OwnerDN": "/my/DN", "OwnerGroup": "myGroup", - "Setup": "aSetup", "CPUTime": 5000, "Tags": ["MultiProcessor", "GPU"], } result = tqDB.insertJob(4, tqDefDict, 10) - assert result["OK"] is True + assert result["OK"] result = tqDB.getTaskQueueForJobs([4]) tq_job4 = result["Value"][4] assert tq_job4 > tq_job3 - tqDefDict = {"OwnerDN": "/my/DN", "OwnerGroup": "myGroup", "Setup": "aSetup", "CPUTime": 5000} + tqDefDict = {"OwnerDN": "/my/DN", "OwnerGroup": "myGroup", "CPUTime": 5000} result = tqDB.insertJob(5, tqDefDict, 10) - assert result["OK"] is True + assert result["OK"] result = tqDB.getTaskQueueForJobs([5]) tq_job5 = result["Value"][5] assert tq_job5 > tq_job4 @@ -576,12 +550,11 @@ def test_chainWithTags(): tqDefDict = { "OwnerDN": "/my/DN", "OwnerGroup": "myGroup", - "Setup": "aSetup", "CPUTime": 5000, "Tags": ["MultiProcessor", "17Processors"], } result = tqDB.insertJob(6, tqDefDict, 10) - assert result["OK"] is True + assert result["OK"] result = tqDB.getTaskQueueForJobs([6]) tq_job6 = result["Value"][6] assert tq_job6 > tq_job5 @@ -617,41 +590,39 @@ def test_chainWithTags(): # Matching # Matching Everything with Tag = "ANY" - result = tqDB.matchAndGetTaskQueue({"Setup": "aSetup", "CPUTime": 50000, "Tag": "ANY"}, numQueuesToGet=6) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000, "Tag": "ANY"}, numQueuesToGet=6) + assert result["OK"] # this should match whatever res = {int(x[0]) for x in result["Value"]} assert res == {tq_job1, tq_job2, tq_job3, tq_job4, tq_job5, tq_job6} # Matching Everything with Tag = "aNy" - result = tqDB.matchAndGetTaskQueue({"Setup": "aSetup", "CPUTime": 50000, "Tag": "aNy"}, numQueuesToGet=6) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000, "Tag": "aNy"}, numQueuesToGet=6) + assert result["OK"] res = {int(x[0]) for x in result["Value"]} assert res == {tq_job1, tq_job2, tq_job3, tq_job4, tq_job5, tq_job6} # Matching Everything with Tag contains "aNy" - result = tqDB.matchAndGetTaskQueue( - {"Setup": "aSetup", "CPUTime": 50000, "Tag": ["MultiProcessor", "aNy"]}, numQueuesToGet=6 - ) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000, "Tag": ["MultiProcessor", "aNy"]}, numQueuesToGet=6) + assert result["OK"] res = {int(x[0]) for x in result["Value"]} assert res == {tq_job1, tq_job2, tq_job3, tq_job4, tq_job5, tq_job6} # Matching only tq_job5 when no tag is specified - result = tqDB.matchAndGetTaskQueue({"Setup": "aSetup", "CPUTime": 50000}, numQueuesToGet=5) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000}, numQueuesToGet=5) + assert result["OK"] res = {int(x[0]) for x in result["Value"]} assert res == {tq_job5} # Matching only tq_job5 when Tag = "" - result = tqDB.matchAndGetTaskQueue({"Setup": "aSetup", "CPUTime": 50000, "Tag": ""}, numQueuesToGet=5) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000, "Tag": ""}, numQueuesToGet=5) + assert result["OK"] res = {int(x[0]) for x in result["Value"]} assert res == {tq_job5} # Matching only tq_job5 when Tag = [] - result = tqDB.matchAndGetTaskQueue({"Setup": "aSetup", "CPUTime": 50000, "Tag": []}, numQueuesToGet=5) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000, "Tag": []}, numQueuesToGet=5) + assert result["OK"] res = {int(x[0]) for x in result["Value"]} assert res == {tq_job5} @@ -659,8 +630,8 @@ def test_chainWithTags(): # Tag: 'MultiProcessor' # By doing this, we are basically saying that this CE is accepting ALSO MultiProcessor payloads - result = tqDB.matchAndGetTaskQueue({"Setup": "aSetup", "CPUTime": 50000, "Tag": "MultiProcessor"}, numQueuesToGet=4) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000, "Tag": "MultiProcessor"}, numQueuesToGet=4) + assert result["OK"] # this matches the tq_job1, as it is the only one that requires ONLY MultiProcessor, # AND the tq_job5, for which we have inserted no tags res = {int(x[0]) for x in result["Value"]} @@ -668,10 +639,8 @@ def test_chainWithTags(): # Tags: ['MultiProcessor', 'GPU'] # By doing this, we are basically saying that this CE is accepting ALSO payloads that require MultiProcessor or GPU - result = tqDB.matchAndGetTaskQueue( - {"Setup": "aSetup", "CPUTime": 50000, "Tag": ["MultiProcessor", "GPU"]}, numQueuesToGet=4 - ) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000, "Tag": ["MultiProcessor", "GPU"]}, numQueuesToGet=4) + assert result["OK"] # this matches the tq_job1, as it requires ONLY MultiProcessor # the tq_job4, as it is the only one that requires BOTH MultiProcessor and GPU, # AND the tq_job5, for which we have inserted no tags @@ -681,19 +650,17 @@ def test_chainWithTags(): # RequiredTag: 'MultiProcessor' (but no Tag) # By doing this, we would be saying that this CE is accepting ONLY MultiProcessor payloads, # BUT since there are no Tags, we can't know what's POSSIBLE to run, so nothing should be matched - result = tqDB.matchAndGetTaskQueue( - {"Setup": "aSetup", "CPUTime": 50000, "RequiredTag": "MultiProcessor"}, numQueuesToGet=4 - ) + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000, "RequiredTag": "MultiProcessor"}, numQueuesToGet=4) assert result["OK"] is False # Tag: 'MultiProcessor' + RequiredTag: 'MultiProcessor' # By doing this, we are basically saying that this CE is accepting ONLY MultiProcessor payloads # which have ONLY the 'MultiProcessor' tag result = tqDB.matchAndGetTaskQueue( - {"Setup": "aSetup", "CPUTime": 50000, "Tag": "MultiProcessor", "RequiredTag": "MultiProcessor"}, + {"CPUTime": 50000, "Tag": "MultiProcessor", "RequiredTag": "MultiProcessor"}, numQueuesToGet=4, ) - assert result["OK"] is True + assert result["OK"] # this matches the tq_job1 as it is the only one that exposes the MultiProcessor tag ONLY res = {int(x[0]) for x in result["Value"]} assert res == {tq_job1} @@ -702,10 +669,10 @@ def test_chainWithTags(): # By doing this, we are basically saying that this CE is accepting MultiProcessor and GPU payloads # but requires to have the MultiProcessor tag result = tqDB.matchAndGetTaskQueue( - {"Setup": "aSetup", "CPUTime": 50000, "Tag": ["MultiProcessor", "GPU"], "RequiredTag": "MultiProcessor"}, + {"CPUTime": 50000, "Tag": ["MultiProcessor", "GPU"], "RequiredTag": "MultiProcessor"}, numQueuesToGet=4, ) - assert result["OK"] is True + assert result["OK"] # this matches the tq_job1 as it is the only one that exposes the MultiProcessor tag ONLY # and tq_job4 because it has GPU and MultiProcessor tags res = {int(x[0]) for x in result["Value"]} @@ -715,14 +682,13 @@ def test_chainWithTags(): # We only want to have MultiProcessor payloads result = tqDB.matchAndGetTaskQueue( { - "Setup": "aSetup", "CPUTime": 50000, "Tag": ["MultiProcessor", "17Processors", "20Processors", "4Processors"], "RequiredTag": "MultiProcessor", }, numQueuesToGet=4, ) - assert result["OK"] is True + assert result["OK"] # this matches the tq_job1 as it is the only one that exposes the MultiProcessor tag ONLY # and tq_job6 because it has 17Processors and MultiProcessor tags res = {int(x[0]) for x in result["Value"]} @@ -730,8 +696,8 @@ def test_chainWithTags(): # NumberOfProcessors and MaxRAM # This is translated to "#Processors" by the SiteDirector - result = tqDB.matchAndGetTaskQueue({"Setup": "aSetup", "CPUTime": 50000, "Tag": "4Processors"}, numQueuesToGet=4) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000, "Tag": "4Processors"}, numQueuesToGet=4) + assert result["OK"] # FIXME: this is not interpreted in any different way --- is it correct? # I believe it should be instead interpreted in a way similar to CPUTime # FIXME: the MaxRam parameter has a similar fate, and becomes "#GB", @@ -739,11 +705,11 @@ def test_chainWithTags(): for jobId in range(1, 8): result = tqDB.deleteJob(jobId) - assert result["OK"] is True + assert result["OK"] for tqId in [tq_job1, tq_job2, tq_job3, tq_job4, tq_job5, tq_job6]: result = tqDB.deleteTaskQueueIfEmpty(tqId) - assert result["OK"] is True + assert result["OK"] def test_chainWithTagsAndPlatforms(): @@ -753,12 +719,11 @@ def test_chainWithTagsAndPlatforms(): tqDefDict = { "OwnerDN": "/my/DN", "OwnerGroup": "myGroup", - "Setup": "aSetup", "CPUTime": 5000, "Platforms": ["centos7"], } result = tqDB.insertJob(1, tqDefDict, 10) - assert result["OK"] is True + assert result["OK"] result = tqDB.getTaskQueueForJobs([1]) tq_job1 = result["Value"][1] assert tq_job1 > 0 @@ -767,12 +732,11 @@ def test_chainWithTagsAndPlatforms(): tqDefDict = { "OwnerDN": "/my/DN", "OwnerGroup": "myGroup", - "Setup": "aSetup", "CPUTime": 5000, "Tags": ["MultiProcessor"], } result = tqDB.insertJob(2, tqDefDict, 10) - assert result["OK"] is True + assert result["OK"] result = tqDB.getTaskQueueForJobs([2]) tq_job2 = result["Value"][2] assert tq_job2 > tq_job1 @@ -781,13 +745,12 @@ def test_chainWithTagsAndPlatforms(): tqDefDict = { "OwnerDN": "/my/DN", "OwnerGroup": "myGroup", - "Setup": "aSetup", "CPUTime": 5000, "Platforms": ["centos7"], "Tags": ["MultiProcessor"], } result = tqDB.insertJob(3, tqDefDict, 10) - assert result["OK"] is True + assert result["OK"] result = tqDB.getTaskQueueForJobs([3]) tq_job3 = result["Value"][3] assert tq_job3 > tq_job2 @@ -796,13 +759,12 @@ def test_chainWithTagsAndPlatforms(): tqDefDict = { "OwnerDN": "/my/DN", "OwnerGroup": "myGroup", - "Setup": "aSetup", "CPUTime": 5000, "Platforms": ["slc6"], "Tags": ["MultiProcessor"], } result = tqDB.insertJob(4, tqDefDict, 10) - assert result["OK"] is True + assert result["OK"] result = tqDB.getTaskQueueForJobs([4]) tq_job4 = result["Value"][4] assert tq_job4 > tq_job3 @@ -844,26 +806,24 @@ def test_chainWithTagsAndPlatforms(): # Matching Everything # No Tag, Platform = "ANY" - result = tqDB.matchAndGetTaskQueue({"Setup": "aSetup", "CPUTime": 50000, "Platform": "ANY"}, numQueuesToGet=4) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000, "Platform": "ANY"}, numQueuesToGet=4) + assert result["OK"] # this should match whatever that does not have tags required, so only tq_job1 res = {int(x[0]) for x in result["Value"]} assert res == {tq_job1} # Tag = "ANY", Platform = "ANY" - result = tqDB.matchAndGetTaskQueue( - {"Setup": "aSetup", "CPUTime": 50000, "Platform": "ANY", "Tag": "ANY"}, numQueuesToGet=4 - ) - assert result["OK"] is True + result = tqDB.matchAndGetTaskQueue({"CPUTime": 50000, "Platform": "ANY", "Tag": "ANY"}, numQueuesToGet=4) + assert result["OK"] # this should match whatever res = {int(x[0]) for x in result["Value"]} assert res == {tq_job1, tq_job2, tq_job3, tq_job4} # Tag = "ANY", Platform = "centos7" result = tqDB.matchAndGetTaskQueue( - {"Setup": "aSetup", "CPUTime": 50000, "Platform": "centos7", "Tag": "MultiProcessor"}, numQueuesToGet=4 + {"CPUTime": 50000, "Platform": "centos7", "Tag": "MultiProcessor"}, numQueuesToGet=4 ) - assert result["OK"] is True + assert result["OK"] # this should match whatever has platform == centos7, or no platform # and either no tags or the MultiProcessor tag res = {int(x[0]) for x in result["Value"]} @@ -871,11 +831,11 @@ def test_chainWithTagsAndPlatforms(): for jobId in range(1, 8): result = tqDB.deleteJob(jobId) - assert result["OK"] is True + assert result["OK"] for tqId in [tq_job1, tq_job2, tq_job3, tq_job4]: result = tqDB.deleteTaskQueueIfEmpty(tqId) - assert result["OK"] is True + assert result["OK"] def test_ComplexMatching(): @@ -883,7 +843,6 @@ def test_ComplexMatching(): {'NumberOfProcessors': 1, 'MaxRAM': 128000, - 'Setup': 'aSetup', 'Site': ['Site_1', 'Site_2'], 'Community': 'vo', 'OwnerGroup': ['admin', 'prod', 'user'], @@ -897,52 +856,48 @@ def test_ComplexMatching(): tqDefDict = { "OwnerDN": "/my/DN", "OwnerGroup": "admin", - "Setup": "aSetup", "CPUTime": 5000, "Sites": ["Site_1", "Site_2"], "Platforms": ["centos7"], } result = tqDB.insertJob(1, tqDefDict, 10) - assert result["OK"] is True + assert result["OK"] result = tqDB.getTaskQueueForJobs([1]) tq_job1 = result["Value"][1] tqDefDict = { "OwnerDN": "/my/DN", "OwnerGroup": "prod", - "Setup": "aSetup", "CPUTime": 5000, "Sites": ["Site_1"], "Platforms": ["slc6", "centos7"], } result = tqDB.insertJob(2, tqDefDict, 10) - assert result["OK"] is True + assert result["OK"] result = tqDB.getTaskQueueForJobs([2]) tq_job2 = result["Value"][2] tqDefDict = { "OwnerDN": "/my/DN", "OwnerGroup": "user", - "Setup": "aSetup", "CPUTime": 5000, "Sites": ["Site_2"], "Platforms": ["slc6", "centos7"], } result = tqDB.insertJob(3, tqDefDict, 10) - assert result["OK"] is True + assert result["OK"] result = tqDB.getTaskQueueForJobs([3]) tq_job3 = result["Value"][3] tqDefDict = { "OwnerDN": "/my/DN", "OwnerGroup": "user", - "Setup": "aSetup", "CPUTime": 5000, "Sites": ["Site_1", "Site_2"], "Platforms": ["ubuntu"], } result = tqDB.insertJob(4, tqDefDict, 10) - assert result["OK"] is True + assert result["OK"] result = tqDB.getTaskQueueForJobs([4]) tq_job4 = result["Value"][4] @@ -950,7 +905,6 @@ def test_ComplexMatching(): result = tqDB.matchAndGetTaskQueue( { - "Setup": "aSetup", "CPUTime": 9999999, "Platform": ["slc6", "centos7"], "OwnerGroup": ["admin", "prod", "user"], @@ -958,13 +912,12 @@ def test_ComplexMatching(): }, numQueuesToGet=4, ) - assert result["OK"] is True + assert result["OK"] res = {int(x[0]) for x in result["Value"]} assert res == {tq_job1, tq_job2, tq_job3} result = tqDB.matchAndGetTaskQueue( { - "Setup": "aSetup", "CPUTime": 9999999, "Platform": ["ubuntu"], "Tag": [], @@ -973,13 +926,12 @@ def test_ComplexMatching(): }, numQueuesToGet=4, ) - assert result["OK"] is True + assert result["OK"] res = {int(x[0]) for x in result["Value"]} assert res == {tq_job4} result = tqDB.matchAndGetTaskQueue( { - "Setup": "aSetup", "CPUTime": 9999999, "Platform": ["slc6", "centos7", "ubuntu"], "Tag": [], @@ -988,13 +940,12 @@ def test_ComplexMatching(): }, numQueuesToGet=4, ) - assert result["OK"] is True + assert result["OK"] res = {int(x[0]) for x in result["Value"]} assert res == {tq_job2, tq_job3, tq_job4} result = tqDB.matchAndGetTaskQueue( { - "Setup": "aSetup", "CPUTime": 9999999, "Platform": ["slc6", "centos7"], "Tag": [], @@ -1003,21 +954,20 @@ def test_ComplexMatching(): }, numQueuesToGet=4, ) - assert result["OK"] is True + assert result["OK"] res = {int(x[0]) for x in result["Value"]} assert res == {tq_job2, tq_job3} result = tqDB.matchAndGetTaskQueue( - {"Setup": "aSetup", "CPUTime": 9999999, "Platform": ["slc6", "centos7"], "OwnerGroup": ["prod", "user"]}, + {"CPUTime": 9999999, "Platform": ["slc6", "centos7"], "OwnerGroup": ["prod", "user"]}, numQueuesToGet=4, ) - assert result["OK"] is True + assert result["OK"] res = {int(x[0]) for x in result["Value"]} assert res == {tq_job2, tq_job3} result = tqDB.matchAndGetTaskQueue( { - "Setup": "aSetup", "CPUTime": 9999999, "Platform": ["slc6", "centos7"], "OwnerGroup": ["prod", "user"], @@ -1025,13 +975,12 @@ def test_ComplexMatching(): }, numQueuesToGet=4, ) - assert result["OK"] is True + assert result["OK"] res = {int(x[0]) for x in result["Value"]} assert res == {tq_job2, tq_job3} result = tqDB.matchAndGetTaskQueue( { - "Setup": "aSetup", "CPUTime": 9999999, "Platform": ["slc6", "centos7"], "OwnerGroup": ["prod", "user"], @@ -1039,13 +988,12 @@ def test_ComplexMatching(): }, numQueuesToGet=4, ) - assert result["OK"] is True + assert result["OK"] res = {int(x[0]) for x in result["Value"]} assert res == {tq_job2} result = tqDB.matchAndGetTaskQueue( { - "Setup": "aSetup", "CPUTime": 10, "Platform": ["slc6", "centos7"], "OwnerGroup": ["prod", "user"], @@ -1053,12 +1001,11 @@ def test_ComplexMatching(): }, numQueuesToGet=4, ) - assert result["OK"] is True + assert result["OK"] assert len(result["Value"]) == 0 result = tqDB.matchAndGetTaskQueue( { - "Setup": "aSetup", "CPUTime": 9999999, "Platform": "ANY", "OwnerGroup": ["admin", "prod", "user"], @@ -1066,7 +1013,7 @@ def test_ComplexMatching(): }, numQueuesToGet=4, ) - assert result["OK"] is True + assert result["OK"] assert len(result["Value"]) == 4 # now inserting one without platform, and try again @@ -1074,18 +1021,16 @@ def test_ComplexMatching(): tqDefDict = { "OwnerDN": "/my/DN", "OwnerGroup": "user", - "Setup": "aSetup", "CPUTime": 5000, "Sites": ["Site_1", "Site_2"], } result = tqDB.insertJob(5, tqDefDict, 10) - assert result["OK"] is True + assert result["OK"] result = tqDB.getTaskQueueForJobs([5]) tq_job5 = result["Value"][5] result = tqDB.matchAndGetTaskQueue( { - "Setup": "aSetup", "CPUTime": 9999999, "Platform": ["slc6", "centos7"], "OwnerGroup": ["admin", "prod", "user"], @@ -1093,13 +1038,12 @@ def test_ComplexMatching(): }, numQueuesToGet=5, ) - assert result["OK"] is True + assert result["OK"] res = {int(x[0]) for x in result["Value"]} assert res == {tq_job1, tq_job2, tq_job3, tq_job5} result = tqDB.matchAndGetTaskQueue( { - "Setup": "aSetup", "CPUTime": 9999999, "Platform": ["ubuntu"], "OwnerGroup": ["admin", "prod", "user"], @@ -1107,13 +1051,12 @@ def test_ComplexMatching(): }, numQueuesToGet=5, ) - assert result["OK"] is True + assert result["OK"] res = {int(x[0]) for x in result["Value"]} assert res == {tq_job4, tq_job5} result = tqDB.matchAndGetTaskQueue( { - "Setup": "aSetup", "CPUTime": 9999999, "Platform": ["ubuntu"], "OwnerGroup": ["admin", "prod", "user"], @@ -1122,13 +1065,12 @@ def test_ComplexMatching(): }, numQueuesToGet=5, ) - assert result["OK"] is True + assert result["OK"] res = {int(x[0]) for x in result["Value"]} assert res == {tq_job4, tq_job5} result = tqDB.matchAndGetTaskQueue( { - "Setup": "aSetup", "CPUTime": 9999999, "Platform": ["ubuntu"], "OwnerGroup": ["admin", "prod", "user"], @@ -1137,13 +1079,12 @@ def test_ComplexMatching(): }, numQueuesToGet=5, ) - assert result["OK"] is True + assert result["OK"] res = {int(x[0]) for x in result["Value"]} assert res == {tq_job4, tq_job5} result = tqDB.matchAndGetTaskQueue( { - "Setup": "aSetup", "CPUTime": 9999999, "Platform": ["ubuntu"], "OwnerGroup": ["admin", "prod", "user"], @@ -1152,48 +1093,47 @@ def test_ComplexMatching(): }, numQueuesToGet=5, ) - assert result["OK"] is True + assert result["OK"] res = {int(x[0]) for x in result["Value"]} assert res == {tq_job4, tq_job5} for jobId in range(1, 8): result = tqDB.deleteJob(jobId) - assert result["OK"] is True + assert result["OK"] for tqId in [tq_job1, tq_job2, tq_job3, tq_job4, tq_job5]: result = tqDB.deleteTaskQueueIfEmpty(tqId) - assert result["OK"] is True + assert result["OK"] def test_TQ(): """test of various functions""" - tqDefDict = {"OwnerDN": "/my/DN", "OwnerGroup": "myGroup", "Setup": "aSetup", "CPUTime": 50000} + tqDefDict = {"OwnerDN": "/my/DN", "OwnerGroup": "myGroup", "CPUTime": 50000} tqDB.insertJob(123, tqDefDict, 10) result = tqDB.getNumTaskQueues() - assert result["OK"] is True + assert result["OK"] assert result["Value"] == 1 result = tqDB.retrieveTaskQueues() - assert result["OK"] is True + assert result["OK"] assert list(result["Value"].values())[0] == { "OwnerDN": "/my/DN", "Jobs": 1, "OwnerGroup": "myGroup", - "Setup": "aSetup", "CPUTime": 86400, "Priority": 1.0, } result = tqDB.findOrphanJobs() - assert result["OK"] is True + assert result["OK"] result = tqDB.recalculateTQSharesForAll() - assert result["OK"] is True + assert result["OK"] # this will also remove the job - result = tqDB.matchAndGetJob({"Setup": "aSetup", "CPUTime": 300000}) - assert result["OK"] is True + result = tqDB.matchAndGetJob({"CPUTime": 300000}) + assert result["OK"] assert result["Value"]["matchFound"] is True assert result["Value"]["jobId"] in [123, 125] tq = result["Value"]["taskQueueId"] result = tqDB.deleteTaskQueueIfEmpty(tq) - assert result["OK"] is True + assert result["OK"] From b738f8aaef30294744ccc6053ca027edf53e4b77 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Wed, 12 Oct 2022 16:43:27 +0200 Subject: [PATCH 0023/1135] fix: removed unused Sandboxes for pilot functionality --- src/DIRAC/Interfaces/API/Dirac.py | 2 +- .../Client/JobState/JobManifest.py | 6 +- .../Client/SandboxStoreClient.py | 72 ++----------------- .../WorkloadManagementSystem/DB/JobDB.py | 12 ++-- .../DB/SandboxMetadataDB.py | 53 +++++++------- .../DB/TaskQueueDB.py | 23 +++--- .../Executor/JobSanity.py | 4 +- .../Service/SandboxStoreHandler.py | 6 +- .../Test_TaskQueueDB.py | 30 ++++---- 9 files changed, 72 insertions(+), 136 deletions(-) diff --git a/src/DIRAC/Interfaces/API/Dirac.py b/src/DIRAC/Interfaces/API/Dirac.py index 0d6fce98381..d61c54af51e 100755 --- a/src/DIRAC/Interfaces/API/Dirac.py +++ b/src/DIRAC/Interfaces/API/Dirac.py @@ -2253,7 +2253,7 @@ def getJobAttributes(self, jobID, printOutput=False): >>> print dirac.getJobAttributes(79241) {'AccountedFlag': 'False','ApplicationNumStatus': '0', 'ApplicationStatus': 'Job Finished Successfully', - 'CPUTime': '0.0'} + 'CPUTime': '0.0'} :param jobID: JobID :type jobID: int, str or python:list diff --git a/src/DIRAC/WorkloadManagementSystem/Client/JobState/JobManifest.py b/src/DIRAC/WorkloadManagementSystem/Client/JobState/JobManifest.py index ebd7acff409..7cd89883f57 100644 --- a/src/DIRAC/WorkloadManagementSystem/Client/JobState/JobManifest.py +++ b/src/DIRAC/WorkloadManagementSystem/Client/JobState/JobManifest.py @@ -67,7 +67,7 @@ def dumpAsJDL(self): def __getCSValue(self, varName, defaultVal=None): if not self.__ops: - self.__ops = Operations(group=self.__manifest["OwnerGroup"]) + self.__ops = Operations(group=self.__manifest["OwnerGroup"]) if varName[0] != "/": varName = "JobDescription/%s" % varName return self.__ops.getValue(varName, defaultVal) @@ -154,9 +154,9 @@ def check(self): """ Check that the manifest is OK """ - for k in ["Owner", "OwnerDN", "OwnerGroup"]: + for k in ["Owner", "OwnerDN", "OwnerGroup"]: if k not in self.__manifest: - return S_ERROR(f"Missing var {k} in manifest") + return S_ERROR(f"Missing var {k} in manifest") # Check CPUTime result = self.__checkNumericalVar("CPUTime", 86400, 100, 500000) diff --git a/src/DIRAC/WorkloadManagementSystem/Client/SandboxStoreClient.py b/src/DIRAC/WorkloadManagementSystem/Client/SandboxStoreClient.py index 75709003446..0889498e93b 100755 --- a/src/DIRAC/WorkloadManagementSystem/Client/SandboxStoreClient.py +++ b/src/DIRAC/WorkloadManagementSystem/Client/SandboxStoreClient.py @@ -60,8 +60,7 @@ def __getRPCClient(self): """Get an RPC client for SB service""" if self.__rpcClient: return self.__rpcClient - else: - return Client(url=self.__serviceName, **self.__kwargs) + return Client(url=self.__serviceName, **self.__kwargs) def __getTransferClient(self): """Get RPC client for TransferClient""" @@ -78,12 +77,6 @@ def uploadFilesAsSandboxForJob(self, fileList, jobId, sbType, sizeLimit=0): return S_ERROR("Invalid Sandbox type %s" % sbType) return self.uploadFilesAsSandbox(fileList, sizeLimit, assignTo={"Job:%s" % jobId: sbType}) - def uploadFilesAsSandboxForPilot(self, fileList, jobId, sbType, sizeLimit=0): - """Upload SB for a pilot""" - if sbType not in self.__validSandboxTypes: - return S_ERROR("Invalid Sandbox type %s" % sbType) - return self.uploadFilesAsSandbox(fileList, sizeLimit, assignTo={"Pilot:%s" % jobId: sbType}) - # Upload generic sandbox def uploadFilesAsSandbox(self, fileList, sizeLimit=0, assignTo=None): @@ -246,17 +239,9 @@ def downloadSandbox(self, sbLocation, destinationDir="", inMemory=False, unpack= ############## # Jobs - def getSandboxesForJob(self, jobId): - """Download job sandbox""" - return self.__getSandboxesForEntity("Job:%s" % jobId) - def assignSandboxesToJob(self, jobId, sbList, ownerName="", ownerGroup=""): """Assign SB to a job""" - return self.__assignSandboxesToEntity("Job:%s" % jobId, sbList, ownerName, ownerGroup) - - def assignSandboxToJob(self, jobId, sbLocation, sbType, ownerName="", ownerGroup=""): - """Assign SB to a job""" - return self.__assignSandboxToEntity("Job:%s" % jobId, sbLocation, sbType, ownerName, ownerGroup) + return self.__assignSandboxesToEntity("Job:%s" % jobId, sbList, ownerName, ownerGroup) def unassignJobs(self, jobIdList): """Unassign SB to a job""" @@ -293,47 +278,6 @@ def downloadSandboxForJob(self, jobId, sbType, destinationPath="", inMemory=Fals downloadedSandboxesLoc.append(result["Value"]) return S_OK(downloadedSandboxesLoc) - ############## - # Pilots - - def getSandboxesForPilot(self, pilotId): - """Get SB for a pilot""" - return self.__getSandboxesForEntity("Pilot:%s" % pilotId) - - def assignSandboxesToPilot(self, pilotId, sbList, ownerName="", ownerGroup=""): - """Assign SB to a pilot""" - return self.__assignSandboxesToEntity("Pilot:%s" % pilotId, sbList, ownerName, ownerGroup) - - def assignSandboxToPilot(self, pilotId, sbLocation, sbType, ownerName="", ownerGroup=""): - """Assign SB to a pilot""" - return self.__assignSandboxToEntity("Pilot:%s" % pilotId, sbLocation, sbType, ownerName, ownerGroup) - - def unassignPilots(self, pilotIdIdList): - """Unassign SB to a pilot""" - if isinstance(pilotIdIdList, int): - pilotIdIdList = [pilotIdIdList] - entitiesList = [] - for pilotId in pilotIdIdList: - entitiesList.append("Pilot:%s" % pilotId) - return self.__unassignEntities(entitiesList) - - def downloadSandboxForPilot(self, jobId, sbType, destinationPath=""): - """Download SB for a pilot""" - result = self.__getSandboxesForEntity("Pilot:%s" % jobId) - if not result["OK"]: - return result - sbDict = result["Value"] - if sbType not in sbDict: - return S_ERROR(f"No {sbType} sandbox registered for pilot {jobId}") - - downloadedSandboxesLoc = [] - for sbLocation in sbDict[sbType]: - result = self.downloadSandbox(sbLocation, destinationPath) - if not result["OK"]: - return result - downloadedSandboxesLoc.append(result["Value"]) - return S_OK(downloadedSandboxesLoc) - ############## # Entities @@ -354,17 +298,9 @@ def __assignSandboxesToEntity(self, eId, sbList, ownerName="", ownerGroup=""): if sbT[1] not in self.__validSandboxTypes: return S_ERROR("Invalid Sandbox type %s" % sbT[1]) if SandboxStoreClient.__smdb and ownerName and ownerGroup: - return SandboxStoreClient.__smdb.assignSandboxesToEntities({eId: sbList}, ownerName, ownerGroup) + return SandboxStoreClient.__smdb.assignSandboxesToEntities({eId: sbList}, ownerName, ownerGroup) rpcClient = self.__getRPCClient() - return rpcClient.assignSandboxesToEntities({eId: sbList}, ownerName, ownerGroup) - - def __assignSandboxToEntity(self, eId, sbLocation, sbType, ownerName="", ownerGroup=""): - """ - Assign a sandbox to a job - sbLocation is "SEName:SEPFN" - sbType is Input or Output - """ - return self.__assignSandboxesToEntity(eId, [(sbLocation, sbType)], ownerName, ownerGroup) + return rpcClient.assignSandboxesToEntities({eId: sbList}, ownerName, ownerGroup) def __unassignEntities(self, eIdList): """ diff --git a/src/DIRAC/WorkloadManagementSystem/DB/JobDB.py b/src/DIRAC/WorkloadManagementSystem/DB/JobDB.py index 743f0814261..6a43eb31634 100755 --- a/src/DIRAC/WorkloadManagementSystem/DB/JobDB.py +++ b/src/DIRAC/WorkloadManagementSystem/DB/JobDB.py @@ -929,7 +929,7 @@ def insertNewJobIntoDB( result = jobManifest.load(jdl) if not result["OK"]: return result - jobManifest.setOptionsFromDict({"Owner": owner, "OwnerDN": ownerDN, "OwnerGroup": ownerGroup}) + jobManifest.setOptionsFromDict({"Owner": owner, "OwnerDN": ownerDN, "OwnerGroup": ownerGroup}) result = jobManifest.check() if not result["OK"]: return result @@ -993,7 +993,7 @@ def insertNewJobIntoDB( classAdJob.insertAttributeInt("JobID", jobID) result = self.__checkAndPrepareJob( - jobID, classAdJob, classAdReq, owner, ownerDN, ownerGroup, jobAttrNames, jobAttrValues + jobID, classAdJob, classAdReq, owner, ownerDN, ownerGroup, jobAttrNames, jobAttrValues ) if not result["OK"]: return result @@ -1081,7 +1081,7 @@ def insertNewJobIntoDB( return retVal def __checkAndPrepareJob( - self, jobID, classAdJob, classAdReq, owner, ownerDN, ownerGroup, jobAttrNames, jobAttrValues + self, jobID, classAdJob, classAdReq, owner, ownerDN, ownerGroup, jobAttrNames, jobAttrValues ): """ Check Consistency of Submitted JDL and set some defaults @@ -1129,7 +1129,7 @@ def __checkAndPrepareJob( # CPU time cpuTime = classAdJob.getAttributeInt("CPUTime") if cpuTime is None: - opsHelper = Operations(group=ownerGroup) + opsHelper = Operations(group=ownerGroup) cpuTime = opsHelper.getValue("JobDescription/DefaultCPUTime", 86400) classAdReq.insertAttributeInt("CPUTime", cpuTime) @@ -1999,12 +1999,12 @@ def getSummarySnapshot(self, requestedFields=False): if not requestedFields: requestedFields = ["Status", "MinorStatus", "Site", "Owner", "OwnerGroup", "JobGroup", "JobSplitType"] valueFields = ["COUNT(JobID)", "SUM(RescheduleCounter)"] - defString = ", ".join(requestedFields) + defString = ", ".join(requestedFields) valueString = ", ".join(valueFields) result = self._query(f"SELECT {defString}, {valueString} FROM Jobs GROUP BY {defString}") if not result["OK"]: return result - return S_OK(((requestedFields + valueFields), result["Value"])) + return S_OK(((requestedFields + valueFields), result["Value"])) def removeInfoFromHeartBeatLogging(self, status, delTime, maxLines): """Remove HeartBeatLoggingInfo from DB. diff --git a/src/DIRAC/WorkloadManagementSystem/DB/SandboxMetadataDB.py b/src/DIRAC/WorkloadManagementSystem/DB/SandboxMetadataDB.py index 70dd018de12..164f6e55230 100644 --- a/src/DIRAC/WorkloadManagementSystem/DB/SandboxMetadataDB.py +++ b/src/DIRAC/WorkloadManagementSystem/DB/SandboxMetadataDB.py @@ -62,8 +62,8 @@ def __initializeDB(self): "EntityId": "VARCHAR(128) NOT NULL", "Type": "VARCHAR(64) NOT NULL", }, - "Indexes": {"Entity": ["EntityId"], "SBIndex": ["SBId"]}, - "UniqueIndexes": {"Mapping": ["SBId", "EntityId", "Type"]}, + "Indexes": {"Entity": ["EntityId"], "SBIndex": ["SBId"]}, + "UniqueIndexes": {"Mapping": ["SBId", "EntityId", "Type"]}, } for tableName in self.__tablesDesc: @@ -134,7 +134,7 @@ def registerAndGetSandbox(self, owner, ownerDN, ownerGroup, sbSE, sbPFN, size=0) if not result["OK"]: return result if not result["Value"]: - return S_ERROR("SandBox already exists but doesn't belong to the user") + return S_ERROR("SandBox already exists but doesn't belong to the user") sbId = result["Value"][0][0] self.accessedSandboxById(sbId) return S_OK((sbId, False)) @@ -185,13 +185,13 @@ def assignSandboxesToEntities(self, enDict, requesterName, requesterGroup, owner return S_ERROR("SB Location has to have SEName|SEPFN form") SEName = splitted[0] SEPFN = ":".join(splitted[1:]) - entitiesToSandboxList.append((entityId, sbTuple[1], SEName, SEPFN)) + entitiesToSandboxList.append((entityId, sbTuple[1], SEName, SEPFN)) if not entitiesToSandboxList: return S_OK() sbIds = [] assigned = 0 - for entityId, SBType, SEName, SEPFN in entitiesToSandboxList: + for entityId, SBType, SEName, SEPFN in entitiesToSandboxList: result = self.getSandboxId(SEName, SEPFN, requesterName, requesterGroup) insertValues = [] if not result["OK"]: @@ -203,7 +203,7 @@ def assignSandboxesToEntities(self, enDict, requesterName, requesterGroup, owner sbId = result["Value"] sbIds.append(str(sbId)) insertValues.append( - "( %s, %s, %d )" + "( %s, %s, %d )" % ( self._escapeString(entityId)["Value"], self._escapeString(SBType)["Value"], @@ -216,7 +216,7 @@ def assignSandboxesToEntities(self, enDict, requesterName, requesterGroup, owner "Sandbox does not exist or you're not authorized to assign it being %s@%s" % (requesterName, requesterGroup) ) - sqlCmd = "INSERT INTO `sb_EntityMapping` ( entityId, Type, SBId ) VALUES %s" % ", ".join(insertValues) + sqlCmd = "INSERT INTO `sb_EntityMapping` ( entityId, Type, SBId ) VALUES %s" % ", ".join(insertValues) result = self._update(sqlCmd) if not result["OK"]: if result["Message"].find("Duplicate entry") == -1: @@ -232,7 +232,7 @@ def __filterEntitiesByRequester(self, entitiesList, requesterName, requesterGrou """ Given a list of entities and a requester, return the ones that the requester is allowed to modify """ - sqlCond = ["s.OwnerId=o.OwnerId", "s.SBId=e.SBId"] + sqlCond = ["s.OwnerId=o.OwnerId", "s.SBId=e.SBId"] requesterProps = Registry.getPropertiesForEntity(requesterGroup, name=requesterName) if Properties.JOB_ADMINISTRATOR in requesterProps: # Do nothing, just ensure it doesn't fit in the other cases @@ -260,26 +260,27 @@ def __filterEntitiesByRequester(self, entitiesList, requesterName, requesterGrou def unassignEntities(self, entities, requesterName, requesterGroup): """ Unassign jobs to sandboxes - entities = ['entityId_1', 'entityId_2'] + + :param list entities: list of entities to unassign """ updated = 0 - if not entities: - return S_OK() - result = self.__filterEntitiesByRequester(entities, requesterName, requesterGroup) - if not result["OK"]: - gLogger.error("Cannot filter entities", result["Message"]) - return result - ids = result["Value"] - if not ids: - return S_OK(0) - sqlCmd = "DELETE FROM `sb_EntityMapping` WHERE EntityId in ( %s )" % ", ".join( - ["'%s'" % str(eid) for eid in ids] - ) - result = self._update(sqlCmd) - if not result["OK"]: - gLogger.error("Cannot unassign entities", result["Message"]) - else: - updated += 1 + if not entities: + return S_OK() + result = self.__filterEntitiesByRequester(entities, requesterName, requesterGroup) + if not result["OK"]: + gLogger.error("Cannot filter entities", result["Message"]) + return result + ids = result["Value"] + if not ids: + return S_OK(0) + sqlCmd = "DELETE FROM `sb_EntityMapping` WHERE EntityId in ( %s )" % ", ".join( + ["'%s'" % str(eid) for eid in ids] + ) + result = self._update(sqlCmd) + if not result["OK"]: + gLogger.error("Cannot unassign entities", result["Message"]) + else: + updated += 1 return S_OK(updated) def getSandboxesAssignedToEntity(self, entityId, requesterName, requesterGroup): diff --git a/src/DIRAC/WorkloadManagementSystem/DB/TaskQueueDB.py b/src/DIRAC/WorkloadManagementSystem/DB/TaskQueueDB.py index 76c46285e9c..7b0b5edd9da 100755 --- a/src/DIRAC/WorkloadManagementSystem/DB/TaskQueueDB.py +++ b/src/DIRAC/WorkloadManagementSystem/DB/TaskQueueDB.py @@ -106,7 +106,7 @@ def __initializeDB(self): "Enabled": "TINYINT(1) NOT NULL DEFAULT 0", }, "PrimaryKey": "TQId", - "Indexes": {"TQOwner": ["OwnerDN", "OwnerGroup", "CPUTime"]}, + "Indexes": {"TQOwner": ["OwnerDN", "OwnerGroup", "CPUTime"]}, } self.__tablesDesc["tq_Jobs"] = { @@ -651,14 +651,14 @@ def __generateNotDictSQL(self, negativeCond): condList = [] for field in negativeCond: if field in multiValueMatchFields: - fullTableN = f"`tq_TQTo{field}s`" + fullTableN = f"`tq_TQTo{field}s`" valList = negativeCond[field] if not isinstance(valList, (list, tuple)): valList = (valList,) subList = [] for value in valList: value = self._escapeString(value)["Value"] - sql = f"{value} NOT IN ( SELECT {fullTableN}.Value FROM {fullTableN} WHERE {fullTableN}.TQId = tq.TQId )" + sql = f"{value} NOT IN ( SELECT {fullTableN}.Value FROM {fullTableN} WHERE {fullTableN}.TQId = tq.TQId )" subList.append(sql) condList.append("( %s )" % " AND ".join(subList)) elif field in singleValueDefFields: @@ -670,7 +670,7 @@ def __generateNotDictSQL(self, negativeCond): @staticmethod def __generateTablesName(sqlTables, field): - fullTableName = f"tq_TQTo{field}s" + fullTableName = f"tq_TQTo{field}s" if fullTableName not in sqlTables: tableN = field.lower() sqlTables[fullTableName] = tableN @@ -713,14 +713,14 @@ def __generateTQMatchSQL(self, tqMatchDict, numQueuesToGet=1, negativeCond=None) if field in tqMatchDict: sqlCondList.append(self.__generateSQLSubCond("tq.%s = %%s" % field, tqMatchDict[field])) # Type of single value conditions - if "CPUTime" in tqMatchDict: - sqlCondList.append(self.__generateSQLSubCond("tq.%s <= %%s" % "CPUTime", tqMatchDict["CPUTime"])) + if "CPUTime" in tqMatchDict: + sqlCondList.append(self.__generateSQLSubCond("tq.%s <= %%s" % "CPUTime", tqMatchDict["CPUTime"])) tag_fv = [] # Match multi value fields for field in multiValueMatchFields: - self.log.debug(f"Evaluating field {field}") + self.log.debug(f"Evaluating field {field}") # It has to be %ss , with an 's' at the end because the columns names # are plural and match options are singular @@ -784,7 +784,7 @@ def __generateTQMatchSQL(self, tqMatchDict, numQueuesToGet=1, negativeCond=None) # In case of Site, check it's not in job banned sites if field in bannedJobMatchFields: - fullTableN = f"`tq_TQToBanned{field}s`" + fullTableN = f"`tq_TQToBanned{field}s`" csql = self.__generateSQLSubCond( "%%s not in ( SELECT %s.Value \ FROM %s \ @@ -811,7 +811,7 @@ def __generateTQMatchSQL(self, tqMatchDict, numQueuesToGet=1, negativeCond=None) # Add possibly Resource banning conditions for field in multiValueMatchFields: - bannedField = f"Banned{field}" + bannedField = f"Banned{field}" # Is there something to consider? b_fv = tqMatchDict.get(bannedField) @@ -824,12 +824,11 @@ def __generateTQMatchSQL(self, tqMatchDict, numQueuesToGet=1, negativeCond=None) ): continue - fullTableN = f"`tq_TQTo{field}s`" + fullTableN = f"`tq_TQTo{field}s`" sqlCondList.append( self.__generateSQLSubCond( - "%%s not in ( SELECT %s.Value FROM %s WHERE %s.TQId = tq.TQId )" - % (fullTableN, fullTableN, fullTableN), + f"%%s not in ( SELECT {fullTableN}.Value FROM {fullTableN} WHERE {fullTableN}.TQId = tq.TQId )", b_fv, boolOp="OR", ) diff --git a/src/DIRAC/WorkloadManagementSystem/Executor/JobSanity.py b/src/DIRAC/WorkloadManagementSystem/Executor/JobSanity.py index 629c5facb9f..289b3377dcd 100755 --- a/src/DIRAC/WorkloadManagementSystem/Executor/JobSanity.py +++ b/src/DIRAC/WorkloadManagementSystem/Executor/JobSanity.py @@ -109,7 +109,7 @@ def checkInputSandbox(self, jobState, manifest): """The number of input sandbox files, as specified in the job JDL are checked in the JobDB. """ - result = jobState.getAttributes(["Owner", "OwnerDN", "OwnerGroup"]) + result = jobState.getAttributes(["Owner", "OwnerDN", "OwnerGroup"]) if not result["OK"]: self.jobLog.error("Failed to get job attributes", result["Message"]) return result @@ -142,7 +142,7 @@ def checkInputSandbox(self, jobState, manifest): if not numSBsToAssign: return S_OK(0) self.jobLog.info("Assigning sandboxes", f"({numSBsToAssign} on behalf of {ownerName}@{ownerGroup})") - result = self.sandboxClient.assignSandboxesToJob(jobState.jid, sbsToAssign, ownerName, ownerGroup) + result = self.sandboxClient.assignSandboxesToJob(jobState.jid, sbsToAssign, ownerName, ownerGroup) if not result["OK"]: self.jobLog.error("Could not assign sandboxes in the SandboxStore") return S_ERROR("Cannot assign sandbox to job") diff --git a/src/DIRAC/WorkloadManagementSystem/Service/SandboxStoreHandler.py b/src/DIRAC/WorkloadManagementSystem/Service/SandboxStoreHandler.py index 9f9ff071107..1d6f41ef88e 100755 --- a/src/DIRAC/WorkloadManagementSystem/Service/SandboxStoreHandler.py +++ b/src/DIRAC/WorkloadManagementSystem/Service/SandboxStoreHandler.py @@ -332,7 +332,7 @@ def export_assignSandboxesToEntities(self, enDict, ownerName="", ownerGroup=""): """ credDict = self.getRemoteCredentials() return self.sandboxDB.assignSandboxesToEntities( - enDict, credDict["username"], credDict["group"], ownerName, ownerGroup + enDict, credDict["username"], credDict["group"], ownerName, ownerGroup ) ################## @@ -345,7 +345,7 @@ def export_unassignEntities(self, entitiesList): Unassign a list of jobs """ credDict = self.getRemoteCredentials() - return self.sandboxDB.unassignEntities(entitiesList, credDict["username"], credDict["group"]) + return self.sandboxDB.unassignEntities(entitiesList, credDict["username"], credDict["group"]) ################## # Getting assigned sandboxes @@ -357,7 +357,7 @@ def export_getSandboxesAssignedToEntity(self, entityId): Get the sandboxes associated to a job and the association type """ credDict = self.getRemoteCredentials() - result = self.sandboxDB.getSandboxesAssignedToEntity(entityId, credDict["username"], credDict["group"]) + result = self.sandboxDB.getSandboxesAssignedToEntity(entityId, credDict["username"], credDict["group"]) if not result["OK"]: return result sbDict = {} diff --git a/tests/Integration/WorkloadManagementSystem/Test_TaskQueueDB.py b/tests/Integration/WorkloadManagementSystem/Test_TaskQueueDB.py index 4fd7073fe56..634bf3a08b8 100644 --- a/tests/Integration/WorkloadManagementSystem/Test_TaskQueueDB.py +++ b/tests/Integration/WorkloadManagementSystem/Test_TaskQueueDB.py @@ -157,10 +157,10 @@ def test_chainWithSites(): # now we will try to delete for jobID in [201, 2011, 202, 203]: result = tqDB.deleteJob(jobID) - assert result["OK"] + assert result["OK"] for tqID in [tq_job1, tq_job2, tq_job3]: result = tqDB.deleteTaskQueueIfEmpty(tqID) - assert result["OK"] + assert result["OK"] def test_chainWithBannedSites(): @@ -230,7 +230,7 @@ def test_chainWithBannedSites(): for tqId in [tq_job1, tq_job2]: result = tqDB.deleteTaskQueueIfEmpty(tqId) - assert result["OK"] + assert result["OK"] result = tqDB.retrieveTaskQueues() assert result["OK"] @@ -470,11 +470,11 @@ def test_chainWithPlatforms(): for jobId in range(1, 8): result = tqDB.deleteJob(jobId) - assert result["OK"] + assert result["OK"] for tqId in [tq_job1, tq_job2, tq_job3, tq_job4, tq_job5, tq_job6, tq_job7]: result = tqDB.deleteTaskQueueIfEmpty(tqId) - assert result["OK"] + assert result["OK"] def test_chainWithTags(): @@ -657,7 +657,7 @@ def test_chainWithTags(): # By doing this, we are basically saying that this CE is accepting ONLY MultiProcessor payloads # which have ONLY the 'MultiProcessor' tag result = tqDB.matchAndGetTaskQueue( - {"CPUTime": 50000, "Tag": "MultiProcessor", "RequiredTag": "MultiProcessor"}, + {"CPUTime": 50000, "Tag": "MultiProcessor", "RequiredTag": "MultiProcessor"}, numQueuesToGet=4, ) assert result["OK"] @@ -669,7 +669,7 @@ def test_chainWithTags(): # By doing this, we are basically saying that this CE is accepting MultiProcessor and GPU payloads # but requires to have the MultiProcessor tag result = tqDB.matchAndGetTaskQueue( - {"CPUTime": 50000, "Tag": ["MultiProcessor", "GPU"], "RequiredTag": "MultiProcessor"}, + {"CPUTime": 50000, "Tag": ["MultiProcessor", "GPU"], "RequiredTag": "MultiProcessor"}, numQueuesToGet=4, ) assert result["OK"] @@ -705,11 +705,11 @@ def test_chainWithTags(): for jobId in range(1, 8): result = tqDB.deleteJob(jobId) - assert result["OK"] + assert result["OK"] for tqId in [tq_job1, tq_job2, tq_job3, tq_job4, tq_job5, tq_job6]: result = tqDB.deleteTaskQueueIfEmpty(tqId) - assert result["OK"] + assert result["OK"] def test_chainWithTagsAndPlatforms(): @@ -821,7 +821,7 @@ def test_chainWithTagsAndPlatforms(): # Tag = "ANY", Platform = "centos7" result = tqDB.matchAndGetTaskQueue( - {"CPUTime": 50000, "Platform": "centos7", "Tag": "MultiProcessor"}, numQueuesToGet=4 + {"CPUTime": 50000, "Platform": "centos7", "Tag": "MultiProcessor"}, numQueuesToGet=4 ) assert result["OK"] # this should match whatever has platform == centos7, or no platform @@ -831,11 +831,11 @@ def test_chainWithTagsAndPlatforms(): for jobId in range(1, 8): result = tqDB.deleteJob(jobId) - assert result["OK"] + assert result["OK"] for tqId in [tq_job1, tq_job2, tq_job3, tq_job4]: result = tqDB.deleteTaskQueueIfEmpty(tqId) - assert result["OK"] + assert result["OK"] def test_ComplexMatching(): @@ -959,7 +959,7 @@ def test_ComplexMatching(): assert res == {tq_job2, tq_job3} result = tqDB.matchAndGetTaskQueue( - {"CPUTime": 9999999, "Platform": ["slc6", "centos7"], "OwnerGroup": ["prod", "user"]}, + {"CPUTime": 9999999, "Platform": ["slc6", "centos7"], "OwnerGroup": ["prod", "user"]}, numQueuesToGet=4, ) assert result["OK"] @@ -1099,11 +1099,11 @@ def test_ComplexMatching(): for jobId in range(1, 8): result = tqDB.deleteJob(jobId) - assert result["OK"] + assert result["OK"] for tqId in [tq_job1, tq_job2, tq_job3, tq_job4, tq_job5]: result = tqDB.deleteTaskQueueIfEmpty(tqId) - assert result["OK"] + assert result["OK"] def test_TQ(): From 3ce389434caa36bb9d7cb03f981112585fde8af2 Mon Sep 17 00:00:00 2001 From: fstagni Date: Wed, 12 Oct 2022 16:59:48 +0200 Subject: [PATCH 0024/1135] sweep: #6399 ElasticJobParametersDB: create a new index every 1M jobs --- src/DIRAC/Core/Utilities/ElasticSearchDB.py | 27 +++++-- .../Agent/OptimizerModule.py | 2 +- .../DB/ElasticJobParametersDB.py | 77 ++++++++++++------- .../Service/JobStateUpdateHandler.py | 6 +- .../Test_ElasticJobParametersDB.py | 25 +++++- 5 files changed, 96 insertions(+), 41 deletions(-) diff --git a/src/DIRAC/Core/Utilities/ElasticSearchDB.py b/src/DIRAC/Core/Utilities/ElasticSearchDB.py index 4d9c32f814b..675afea78d5 100644 --- a/src/DIRAC/Core/Utilities/ElasticSearchDB.py +++ b/src/DIRAC/Core/Utilities/ElasticSearchDB.py @@ -6,20 +6,31 @@ from datetime import datetime from datetime import timedelta -import certifi import copy import functools import json +import certifi + try: - from opensearchpy import OpenSearch as Elasticsearch from opensearch_dsl import Search, Q, A - from opensearchpy.exceptions import ConnectionError, TransportError, NotFoundError, RequestError + from opensearchpy import OpenSearch as Elasticsearch + from opensearchpy.exceptions import ( + ConnectionError as ElasticConnectionError, + TransportError, + NotFoundError, + RequestError, + ) from opensearchpy.helpers import BulkIndexError, bulk except ImportError: - from elasticsearch import Elasticsearch from elasticsearch_dsl import Search, Q, A - from elasticsearch.exceptions import ConnectionError, TransportError, NotFoundError, RequestError + from elasticsearch import Elasticsearch + from elasticsearch.exceptions import ( + ConnectionError as ElasticConnectionError, + TransportError, + NotFoundError, + RequestError, + ) from elasticsearch.helpers import BulkIndexError, bulk from DIRAC import gLogger, S_OK, S_ERROR @@ -182,7 +193,7 @@ def __init__( self._connected = True else: sLog.error("Cannot ping ElasticsearchDB!") - except ConnectionError as e: + except ElasticConnectionError as e: sLog.error(repr(e)) def getIndexPrefix(self): @@ -404,7 +415,7 @@ def deleteIndex(self, indexName): retVal = self.client.indices.delete(indexName) except NotFoundError: sLog.warn("Index does not exist", indexName) - return S_OK("Noting to delete") + return S_OK("Nothing to delete") except ValueError as e: return S_ERROR(DErrno.EVALUE, e) @@ -535,7 +546,7 @@ def pingDB(self): connected = False try: connected = self.client.ping() - except ConnectionError as e: + except ElasticConnectionError as e: sLog.error("Cannot connect to the db", repr(e)) return S_OK(connected) diff --git a/src/DIRAC/WorkloadManagementSystem/Agent/OptimizerModule.py b/src/DIRAC/WorkloadManagementSystem/Agent/OptimizerModule.py index e6cfc3f7336..ea5621f9060 100755 --- a/src/DIRAC/WorkloadManagementSystem/Agent/OptimizerModule.py +++ b/src/DIRAC/WorkloadManagementSystem/Agent/OptimizerModule.py @@ -247,7 +247,7 @@ def setJobParam(self, job, reportName, value): self.log.debug(f"setJobParameter({job},'{reportName}','{value}')") if self.elasticJobParametersDB: - return self.elasticJobParametersDB.setJobParameter(job, reportName, value) + return self.elasticJobParametersDB.setJobParameter(int(job), reportName, value) return self.jobDB.setJobParameter(job, reportName, value) ############################################################################# diff --git a/src/DIRAC/WorkloadManagementSystem/DB/ElasticJobParametersDB.py b/src/DIRAC/WorkloadManagementSystem/DB/ElasticJobParametersDB.py index cd0eeaad5d1..ae9f48f9b75 100644 --- a/src/DIRAC/WorkloadManagementSystem/DB/ElasticJobParametersDB.py +++ b/src/DIRAC/WorkloadManagementSystem/DB/ElasticJobParametersDB.py @@ -38,7 +38,8 @@ except ImportError: from elasticsearch.exceptions import NotFoundError, RequestError -name = "ElasticJobParametersDB" +name = "ElasticJobParametersDB" # (base for) old index name + mapping = { "properties": { @@ -73,18 +74,32 @@ def __init__(self, parentLogger=None): raise RuntimeError("Can't connect to ElasticJobParametersDB") self.oldIndexName = f"{self.getIndexPrefix()}_{name.lower()}" - self.indexName = f"{self.getIndexPrefix()}_elasticjobparameters_index" + self.indexName_base = f"{self.getIndexPrefix()}_elasticjobparameters_index" + + self.dslSearch = self._Search(self.oldIndexName) + self.dslSearch.extra(track_total_hits=True) + + def _indexName(self, jobID: int) -> str: + """construct the index name + + :param jobID: Job ID + """ + indexSplit = int(jobID / 1e6) + return f"{self.indexName_base}_{indexSplit}m" + + def _createIndex(self, indexName: str) -> None: + """Create a new index if needed + + :param indexName: index name + """ # Verifying if the index is there, and if not create it - res = self.existingIndex(self.indexName) + res = self.existingIndex(indexName) if not res["OK"] or not res["Value"]: - result = self.createIndex(self.indexName, mapping, period=None) + result = self.createIndex(indexName, mapping, period=None) if not result["OK"]: self.log.error(result["Message"]) raise RuntimeError(result["Message"]) - self.log.always("Index created:", self.indexName) - - self.dslSearch = self._Search(self.oldIndexName) - self.dslSearch.extra(track_total_hits=True) + self.log.always("Index created:", indexName) def getJobParameters(self, jobID: int, paramList=None) -> dict: """Get Job Parameters defined for jobID. @@ -100,18 +115,18 @@ def getJobParameters(self, jobID: int, paramList=None) -> dict: paramList = paramList.replace(" ", "").split(",") self.log.debug(f"JobDB.getParameters: Getting Parameters for job {jobID}") resultDict = {} - inNewIndex = self.existsDoc(self.indexName, str(jobID)) + inNewIndex = self.existsDoc(self._indexName(jobID), jobID) inOldIndex = self._isInOldIndex(self.oldIndexName, jobID) # Case 1: the parameters are stored in both indices if inNewIndex and inOldIndex: # First we get the parameters from the old index self.log.debug( - f"A document with JobID {jobID} was found in the old index {self.oldIndexName} and in the new index {self.indexName}" + f"A document with JobID {jobID} was found in the old index {self.oldIndexName} and in the new index {self._indexName(jobID)}" ) resultDict = self._searchInOldIndex(jobID, paramList) # Now we get the parameters from the new index - res = self.getDoc(self.indexName, str(jobID)) + res = self.getDoc(self._indexName(jobID), jobID) if not res["OK"]: self.log.error("Could not retrieve the data from the new index!", res["Message"]) else: @@ -126,8 +141,10 @@ def getJobParameters(self, jobID: int, paramList=None) -> dict: # Case 3: only in the new index else: - self.log.debug(f"The searched parameters with JobID {jobID} exists in the new index {self.indexName}") - res = self.getDoc(self.indexName, str(jobID)) + self.log.debug( + f"The searched parameters with JobID {jobID} exists in the new index {self._indexName(jobID)}" + ) + res = self.getDoc(self._indexName(jobID), jobID) if not res["OK"]: return res resultDict = res["Value"] @@ -155,12 +172,13 @@ def setJobParameter(self, jobID: int, key: str, value: str) -> dict: # The _id in ES can't exceed 512 bytes, this is a ES hard-coded limitation. # If a record with this jobID update and add parameter, otherwise create a new record - if self.existsDoc(self.indexName, id=str(jobID)): + if self.existsDoc(self._indexName(jobID), id=str(jobID)): self.log.debug("A document for this job already exists, it will now be updated") - result = self.updateDoc(index=self.indexName, id=str(jobID), body={"doc": data}) + result = self.updateDoc(index=self._indexName(jobID), id=str(jobID), body={"doc": data}) else: self.log.debug("No document has this job id, creating a new document for this job") - result = self.index(self.indexName, body=data, docID=str(jobID)) + self._createIndex(self._indexName(jobID)) + result = self.index(indexName=self._indexName(jobID), body=data, docID=str(jobID)) if not result["OK"]: self.log.error("Couldn't insert or update data", result["Message"]) return result @@ -174,18 +192,19 @@ def setJobParameters(self, jobID: int, parameters: list) -> dict: :param parameters: list of tuples (name, value) pairs :returns: S_OK/S_ERROR as result of indexing """ - self.log.debug(f"Inserting parameters", "in {self.indexName}: for job {jobID}: {parameters}") + self.log.debug("Inserting parameters", f"in {self._indexName(jobID)}: for job {jobID}: {parameters}") parametersDict = dict(parameters) parametersDict["JobID"] = jobID parametersDict["timestamp"] = int(TimeUtilities.toEpochMilliSeconds()) - if self.existsDoc(self.indexName, id=str(jobID)): + if self.existsDoc(self._indexName(jobID), id=str(jobID)): self.log.debug("A document for this job already exists, it will now be updated") - result = self.updateDoc(index=self.indexName, id=str(jobID), body={"doc": parametersDict}) + result = self.updateDoc(index=self._indexName(jobID), id=str(jobID), body={"doc": parametersDict}) else: self.log.debug("Creating a new document for this job") - result = self.index(self.indexName, body=parametersDict, docID=str(jobID)) + self._createIndex(self._indexName(jobID)) + result = self.index(self._indexName(jobID), body=parametersDict, docID=str(jobID)) if not result["OK"]: self.log.error("Couldn't insert or update data", result["Message"]) return result @@ -203,7 +222,7 @@ def deleteJobParameters(self, jobID: int, paramList=None) -> dict: if isinstance(paramList, str): paramList = paramList.replace(" ", "").split(",") - inNewIndex = self.existsDoc(self.indexName, str(jobID)) + inNewIndex = self.existsDoc(self._indexName(jobID), str(jobID)) inOldIndex = self._isInOldIndex(self.oldIndexName, jobID) # 3 Cases as in GetJobParameters @@ -218,7 +237,9 @@ def deleteJobParameters(self, jobID: int, paramList=None) -> dict: self._deleteInOldIndex(jobID, paramList) else: - self.log.debug(f"The searched parameters with JobID {jobID} exists in the new index {self.indexName}") + self.log.debug( + f"The searched parameters with JobID {jobID} exists in the new index {self._indexName(jobID)}" + ) res = self._deleteInNewIndex(jobID, paramList) if not res["OK"]: return S_ERROR(res) @@ -269,10 +290,10 @@ def _searchInOldIndex(self, jobID: int, paramList: list) -> bool: res = s.scan() for hit in res: - name = hit.Name - if paramList and name not in paramList: + pname = hit.Name + if paramList and pname not in paramList: continue - resultDict[name] = hit.Value + resultDict[pname] = hit.Value return resultDict def _deleteInOldIndex(self, jobID: int, paramList: list) -> dict: @@ -309,13 +330,15 @@ def _deleteInNewIndex(self, jobID: int, paramList=None) -> dict: if not paramList: # Deleting the whole record self.log.debug("Deleting record of job {jobID}") - result = self.deleteDoc(self.indexName, id=str(jobID)) + result = self.deleteDoc(self._indexName(jobID), id=str(jobID)) else: # Deleting the specific parameters self.log.debug(f"JobDB.getParameters: Deleting Parameters {paramList} for job {jobID}") for paramName in paramList: result = self.updateDoc( - index=self.indexName, id=str(jobID), body={"script": "ctx._source.remove('" + paramName + "')"} + index=self._indexName(jobID), + id=str(jobID), + body={"script": "ctx._source.remove('" + paramName + "')"}, ) self.log.debug(f"Deleted parameter {paramName}") if not result["OK"]: diff --git a/src/DIRAC/WorkloadManagementSystem/Service/JobStateUpdateHandler.py b/src/DIRAC/WorkloadManagementSystem/Service/JobStateUpdateHandler.py index fee4b142fc6..f293960d8e3 100755 --- a/src/DIRAC/WorkloadManagementSystem/Service/JobStateUpdateHandler.py +++ b/src/DIRAC/WorkloadManagementSystem/Service/JobStateUpdateHandler.py @@ -254,7 +254,7 @@ def _setJobStatusBulk(cls, jobID, statusDict, force=False): if not result["OK"]: return result if cls.elasticJobParametersDB: - result = cls.elasticJobParametersDB.setJobParameter(jobID, "Status", status) + result = cls.elasticJobParametersDB.setJobParameter(int(jobID), "Status", status) if not result["OK"]: return result # Update start and end time if needed @@ -361,7 +361,7 @@ def export_setJobsParameter(cls, jobsParameterDict): if cls.elasticJobParametersDB: res = cls.elasticJobParametersDB.setJobParameter( - jobID, str(jobsParameterDict[jobID][0]), str(jobsParameterDict[jobID][1]) + int(jobID), str(jobsParameterDict[jobID][0]), str(jobsParameterDict[jobID][1]) ) if not res["OK"]: cls.log.error("Failed to add Job Parameter to elasticJobParametersDB", res["Message"]) @@ -391,7 +391,7 @@ def export_setJobParameters(cls, jobID, parameters): for job specified by its JobId """ if cls.elasticJobParametersDB: - result = cls.elasticJobParametersDB.setJobParameters(jobID, parameters) + result = cls.elasticJobParametersDB.setJobParameters(int(jobID), parameters) if not result["OK"]: cls.log.error("Failed to add Job Parameters to ElasticJobParametersDB", result["Message"]) else: diff --git a/tests/Integration/WorkloadManagementSystem/Test_ElasticJobParametersDB.py b/tests/Integration/WorkloadManagementSystem/Test_ElasticJobParametersDB.py index 91c9ee51781..10f8372ff40 100644 --- a/tests/Integration/WorkloadManagementSystem/Test_ElasticJobParametersDB.py +++ b/tests/Integration/WorkloadManagementSystem/Test_ElasticJobParametersDB.py @@ -107,6 +107,14 @@ def test_setAndGetJobFromDB(): assert res["Value"][101]["k"] == "v" assert res["Value"][101]["k2"] == "v2" + # another job with jobID > 1000000 + res = elasticJobParametersDB.setJobParameters(1010000, [("k", "v"), ("k1", "v1"), ("k2", "v2")]) + assert res["OK"] + time.sleep(SLEEP_DELAY) + res = elasticJobParametersDB.getJobParameters(1010000) + assert res["Value"][1010000]["k"] == "v" + assert res["Value"][1010000]["k2"] == "v2" + # deleting res = elasticJobParametersDB.deleteJobParameters(100) assert res["OK"] @@ -134,5 +142,18 @@ def test_setAndGetJobFromDB(): assert res["OK"] assert len(res["Value"][101]) == 6 - # delete the index - res = elasticJobParametersDB.deleteIndex(elasticJobParametersDB.indexName) + res = elasticJobParametersDB.deleteJobParameters(1010000) + assert res["OK"] + time.sleep(SLEEP_DELAY) + res = elasticJobParametersDB.getJobParameters(1010000) + assert res["OK"] + assert len(res["Value"][1010000]) == 0 + + # delete the indexes + res = elasticJobParametersDB.deleteIndex(elasticJobParametersDB.indexName_base) + assert res["OK"] + assert res["Value"] == "Nothing to delete" + res = elasticJobParametersDB.deleteIndex(elasticJobParametersDB._indexName(100)) + assert res["OK"] + res = elasticJobParametersDB.deleteIndex(elasticJobParametersDB._indexName(1010000)) + assert res["OK"] From 840dc3db20545d9c22acd6bc92a44932fbea825a Mon Sep 17 00:00:00 2001 From: chaen Date: Wed, 12 Oct 2022 17:05:17 +0200 Subject: [PATCH 0025/1135] sweep: #6386 Remove Monitoring DataOperationPlotter --- .../DataOperation/grafana/DMSOverview.json | 4855 +++++++++++++++++ .../DataOperation/grafana/Tier1DMS.json | 1905 +++++++ .../private/Plotters/DataOperationPlotter.py | 227 - 3 files changed, 6760 insertions(+), 227 deletions(-) create mode 100644 dashboards/DataOperation/grafana/DMSOverview.json create mode 100644 dashboards/DataOperation/grafana/Tier1DMS.json delete mode 100644 src/DIRAC/MonitoringSystem/private/Plotters/DataOperationPlotter.py diff --git a/dashboards/DataOperation/grafana/DMSOverview.json b/dashboards/DataOperation/grafana/DMSOverview.json new file mode 100644 index 00000000000..d1a24a57453 --- /dev/null +++ b/dashboards/DataOperation/grafana/DMSOverview.json @@ -0,0 +1,4855 @@ +{ + "__inputs": [ + { + "name": "DS_OPENSEARCH", + "label": "OpenSearch", + "description": "", + "type": "datasource", + "pluginId": "grafana-opensearch-datasource", + "pluginName": "OpenSearch" + }, + { + "name": "DS_EXPRESSION", + "label": "Expression", + "description": "", + "type": "datasource", + "pluginId": "__expr__" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "datasource", + "id": "__expr__", + "version": "1.0.0" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.1.6" + }, + { + "type": "datasource", + "id": "grafana-opensearch-datasource", + "name": "OpenSearch", + "version": "2.0.1" + }, + { + "type": "panel", + "id": "piechart", + "name": "Pie chart", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Overview of the ongoing Data Management operations", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 50, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Destination", + "id": "3", + "settings": { + "min_doc_count": "0", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": true, + "metrics": [ + { + "field": "TransferSize", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:Successful AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user} AND Destination:${destination}", + "queryType": "lucene", + "refId": "TransferSize", + "timeField": "timestamp" + }, + { + "alias": "", + "bucketAggs": [ + { + "field": "Destination", + "id": "3", + "settings": { + "min_doc_count": "0", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": true, + "metrics": [ + { + "field": "TransferTime", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:Successful AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user} AND Destination:${destination}", + "queryType": "lucene", + "refId": "TransferTime", + "timeField": "timestamp" + }, + { + "datasource": { + "type": "__expr__", + "uid": "${DS_EXPRESSION}" + }, + "expression": "$TransferSize/$TransferTime", + "hide": false, + "refId": "A", + "type": "math" + } + ], + "title": "Panel Title", + "transformations": [], + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 12, + "panels": [], + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "Final status of se.putFile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 0, + "y": 9 + }, + "id": 4, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "FinalStatus", + "id": "2", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "20" + }, + "type": "terms" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND Destination:${destination} AND ExecutionSite:${executionSite} AND FinalStatus:${finalStatus} AND Protocol:${protocol} AND User:${user}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Data upload final status", + "type": "piechart" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "Final status of se.putFile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 4, + "y": 9 + }, + "id": 16, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Protocol", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:Failed AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Failed Data upload by Protocol", + "type": "piechart" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "Final status of se.putFile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 8, + "y": 9 + }, + "id": 19, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Protocol", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:Successful AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Successful Data upload by Protocol", + "type": "piechart" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "Final status of se.putFile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 12, + "y": 9 + }, + "id": 22, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "User", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:Successful AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Successful Data upload by User", + "type": "piechart" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 16, + "y": 9 + }, + "id": 23, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "User", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:Failed AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Failed Data upload by User", + "type": "piechart" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 52, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Destination", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "time_series", + "hide": false, + "metrics": [ + { + "field": "TransferSize", + "hide": true, + "id": "1", + "type": "sum" + }, + { + "field": "1", + "id": "4", + "type": "cumulative_sum" + } + ], + "query": "OperationType:se.putFile AND Destination:${destination} AND ExecutionSite:${executionSite} AND FinalStatus:${finalStatus} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Cumulative uploaded data", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 53, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "ExecutionSite", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "time_series", + "hide": false, + "metrics": [ + { + "field": "TransferSize", + "hide": true, + "id": "1", + "type": "sum" + }, + { + "field": "1", + "id": "4", + "type": "cumulative_sum" + } + ], + "query": "OperationType:se.putFile AND Destination:${destination} AND ExecutionSite:${executionSite} AND FinalStatus:${finalStatus} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Cumulative uploaded data", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "Final status of se.putFile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 0, + "y": 24 + }, + "id": 39, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "FinalStatus", + "id": "2", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "20" + }, + "type": "terms" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.getFile AND Destination:${destination} AND ExecutionSite:${executionSite} AND FinalStatus:${finalStatus} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Data download final status", + "type": "piechart" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 4, + "y": 24 + }, + "id": 40, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Protocol", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.getFile AND FinalStatus:Failed AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Failed Data Download by Protocol", + "type": "piechart" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 8, + "y": 24 + }, + "id": 45, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Protocol", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.getFile AND FinalStatus:Successful AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Successful Data download by Protocol", + "type": "piechart" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 12, + "y": 24 + }, + "id": 42, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "User", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.getFile AND FinalStatus:Successful AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user}AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Successful Data download by User", + "type": "piechart" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 16, + "y": 24 + }, + "id": 43, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "User", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.getFile AND FinalStatus:Failed AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Failed Data download by User", + "type": "piechart" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 54, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Source", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "time_series", + "hide": false, + "metrics": [ + { + "field": "TransferSize", + "hide": true, + "id": "1", + "type": "sum" + }, + { + "field": "1", + "id": "4", + "type": "cumulative_sum" + } + ], + "query": "OperationType:se.getFile AND ExecutionSite:${executionSite} AND FinalStatus:${finalStatus} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Cumulative downloaded data (source SE)", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 55, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "ExecutionSite", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "time_series", + "hide": false, + "metrics": [ + { + "field": "TransferSize", + "hide": true, + "id": "1", + "type": "sum" + }, + { + "field": "1", + "id": "4", + "type": "cumulative_sum" + } + ], + "query": "OperationType:se.getFile AND ExecutionSite:${executionSite} AND FinalStatus:${finalStatus} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Cumulative downloaded data (execution site)", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "Final status of se.putFile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 0, + "y": 39 + }, + "id": 44, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "FinalStatus", + "id": "2", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "20" + }, + "type": "terms" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.removeFile AND Destination:${destination} AND ExecutionSite:${executionSite} AND FinalStatus:${finalStatus} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Removal final status", + "type": "piechart" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 4, + "y": 39 + }, + "id": 46, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Protocol", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.removeFile AND FinalStatus:Failed AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Failed removal by Protocol", + "type": "piechart" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 8, + "y": 39 + }, + "id": 41, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Protocol", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.removeFile AND FinalStatus:Successful AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Successful removal by Protocol", + "type": "piechart" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 12, + "y": 39 + }, + "id": 47, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "User", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.removeFile AND FinalStatus:Successful AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user}AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Successful removal by User", + "type": "piechart" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 16, + "y": 39 + }, + "id": 48, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "User", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.removeFile AND FinalStatus:Failed AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Failed removal by User", + "type": "piechart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 46 + }, + "id": 25, + "panels": [], + "title": "Data download", + "type": "row" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "asc" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "FinalStatus", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.getFile AND Destination:${destination} AND ExecutionSite:${executionSite} AND FinalStatus:${finalStatus} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": " Data download by status", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 29, + "options": { + "legend": { + "calcs": [ + "max", + "min", + "mean" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "id": "1", + "settings": { + "size": "500" + }, + "type": "raw_data" + } + ], + "query": "OperationType:se.getFile AND FinalStatus:Successful AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Successful download throughput", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "Throughput", + "binary": { + "left": "TransferSize", + "operator": "/", + "reducer": "sum", + "right": "TransferTime" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 56 + }, + "id": 31, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Source", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.getFile AND FinalStatus:Successful AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Successful download by source", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 56 + }, + "id": 33, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Source", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.getFile AND FinalStatus:Failed AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Failed download by source", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 64 + }, + "id": 34, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Protocol", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.getFile AND FinalStatus:Successful AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Successful download by protocol", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 64 + }, + "id": 36, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Protocol", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.getFile AND FinalStatus:Failed AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Failed download by Protocol", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 72 + }, + "id": 32, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "User", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.getFile AND FinalStatus:Successful AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Successful download by User", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 72 + }, + "id": 35, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "User", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.getFile AND FinalStatus:Failed AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Failed download by user", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 80 + }, + "id": 37, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "ExecutionSite", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.getFile AND FinalStatus:Successful AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Successful download by ExecutionSite", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 80 + }, + "id": 38, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "ExecutionSite", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.getFile AND FinalStatus:Failed AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Failed download by ExecutionSite", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 88 + }, + "id": 6, + "panels": [], + "title": "Data Upload", + "type": "row" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 89 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "asc" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "FinalStatus", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND Destination:${destination} AND ExecutionSite:${executionSite} AND FinalStatus:${finalStatus} AND Protocol:${protocol} AND User:${user}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Data Upload by Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 89 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "max", + "min", + "mean" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "id": "1", + "settings": { + "size": "500" + }, + "type": "raw_data" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:Successful AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Successful upload throughput", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "Throughput", + "binary": { + "left": "TransferSize", + "operator": "/", + "reducer": "sum", + "right": "TransferTime" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 98 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Destination", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:Successful AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Successful upload by destination", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 98 + }, + "id": 20, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Destination", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:Failed AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Failed upload by destination", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 106 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Protocol", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "20" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:Successful AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Successful upload by protocol", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 106 + }, + "id": 15, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Protocol", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "20" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:Failed AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Failed upload by protocol", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 114 + }, + "id": 17, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "User", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "20" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:Successful AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Successful upload by User", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 114 + }, + "id": 18, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "User", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "20" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:Failed AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Failed upload by User", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 122 + }, + "id": 21, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "ExecutionSite", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "20" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:Successful AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Failed upload by ExecutionSite", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 122 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "ExecutionSite", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "20" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:Failed AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Failed upload by ExecutionSite", + "type": "timeseries" + } + ], + "refresh": false, + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": "*", + "current": {}, + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "definition": "{\"find\": \"terms\", \"field\": \"Destination\" , \"size\":10000}", + "hide": 0, + "includeAll": true, + "label": "Destination", + "multi": true, + "name": "destination", + "options": [], + "query": "{\"find\": \"terms\", \"field\": \"Destination\" , \"size\":10000}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": "*", + "current": {}, + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "definition": "{\"find\": \"terms\", \"field\": \"ExecutionSite\" , \"size\":10000}", + "hide": 0, + "includeAll": true, + "label": "ExecutionSite", + "multi": true, + "name": "executionSite", + "options": [], + "query": "{\"find\": \"terms\", \"field\": \"ExecutionSite\" , \"size\":10000}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": "*", + "current": {}, + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "definition": "{\"find\": \"terms\", \"field\": \"User\" , \"size\":10000}", + "hide": 0, + "includeAll": true, + "label": "User", + "multi": true, + "name": "user", + "options": [], + "query": "{\"find\": \"terms\", \"field\": \"User\" , \"size\":10000}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": "*", + "current": {}, + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "definition": "{\"find\": \"terms\", \"field\": \"Protocol\" , \"size\":10000 , \"query\":\"OperationType:se*\"}", + "description": "All the protocols used by the StorageElement", + "hide": 0, + "includeAll": true, + "label": "Protocol", + "multi": true, + "name": "protocol", + "options": [], + "query": "{\"find\": \"terms\", \"field\": \"Protocol\" , \"size\":10000 , \"query\":\"OperationType:se*\"}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": "*", + "current": {}, + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "definition": "{\"find\": \"terms\", \"field\": \"Source\" , \"size\":10000}", + "hide": 0, + "includeAll": true, + "label": "Source", + "multi": true, + "name": "source", + "options": [], + "query": "{\"find\": \"terms\", \"field\": \"Source\" , \"size\":10000}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": "*", + "current": {}, + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "definition": "{\"find\": \"terms\", \"field\": \"FinalStatus\" , \"size\":10000}", + "hide": 0, + "includeAll": true, + "label": "Status", + "multi": true, + "name": "finalStatus", + "options": [], + "query": "{\"find\": \"terms\", \"field\": \"FinalStatus\" , \"size\":10000}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": "*", + "current": {}, + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "definition": "{\"find\": \"terms\", \"field\": \"OperationType\" , \"size\":10000}", + "hide": 0, + "includeAll": true, + "label": "Operation", + "multi": true, + "name": "operationType", + "options": [], + "query": "{\"find\": \"terms\", \"field\": \"OperationType\" , \"size\":10000}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Data Management overview", + "uid": "Ix04HM4Vz", + "version": 32, + "weekStart": "" +} diff --git a/dashboards/DataOperation/grafana/Tier1DMS.json b/dashboards/DataOperation/grafana/Tier1DMS.json new file mode 100644 index 00000000000..ccb8b378cbc --- /dev/null +++ b/dashboards/DataOperation/grafana/Tier1DMS.json @@ -0,0 +1,1905 @@ +{ + "__inputs": [ + { + "name": "DS_OPENSEARCH", + "label": "OpenSearch", + "description": "", + "type": "datasource", + "pluginId": "grafana-opensearch-datasource", + "pluginName": "OpenSearch" + }, + { + "name": "DS_EXPRESSION", + "label": "Expression", + "description": "", + "type": "datasource", + "pluginId": "__expr__" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "datasource", + "id": "__expr__", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "flant-statusmap-panel", + "name": "Statusmap", + "version": "0.5.1" + }, + { + "type": "panel", + "id": "gauge", + "name": "Gauge", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.1.6" + }, + { + "type": "datasource", + "id": "grafana-opensearch-datasource", + "name": "OpenSearch", + "version": "2.0.1" + }, + { + "type": "panel", + "id": "heatmap", + "name": "Heatmap", + "version": "" + }, + { + "type": "panel", + "id": "piechart", + "name": "Pie chart", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + }, + { + "type": "panel", + "id": "vonage-status-panel", + "name": "Status Panel", + "version": "1.0.11" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 451, + "panels": [], + "title": "Upload from site overwiew", + "type": "row" + }, + { + "clusterName": "", + "colorMode": "Panel", + "colors": { + "crit": "rgba(245, 54, 54, 0.9)", + "disable": "rgba(128, 128, 128, 0.9)", + "ok": "rgba(50, 128, 45, 0.9)", + "warn": "rgba(237, 129, 40, 0.9)" + }, + "cornerRadius": 0, + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "displayName": "", + "flipCard": true, + "flipTime": 5, + "fontFormat": "Regular", + "gridPos": { + "h": 8, + "w": 3.4285714285714284, + "x": 0, + "y": 1 + }, + "id": 525, + "isAutoScrollOnOverflow": false, + "isGrayOnNoData": false, + "isHideAlertsOnDisable": false, + "isIgnoreOKColors": false, + "maxPerRow": 12, + "pluginVersion": "9.1.6", + "repeat": "site", + "repeatDirection": "h", + "targets": [ + { + "$$hashKey": "object:24", + "aggregation": "Avg", + "alias": "Status", + "bucketAggs": [ + { + "field": "timestamp", + "id": "5", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "crit": 0.8, + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "decimals": 2, + "displayAliasType": "Always", + "displayType": "Regular", + "displayValueWithAlias": "When Alias Displayed", + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "hide": true, + "id": "1", + "type": "sum" + }, + { + "field": "TransferOK", + "hide": true, + "id": "6", + "type": "sum" + }, + { + "id": "7", + "pipelineVariables": [ + { + "name": "all", + "pipelineAgg": "1" + }, + { + "name": "ok", + "pipelineAgg": "6" + } + ], + "settings": { + "script": "params.ok / params.all" + }, + "type": "bucket_script" + } + ], + "query": "OperationType:se.putFile AND Destination:${destination} AND ExecutionSite:${site} AND FinalStatus:${finalStatus} AND Protocol:${protocol} AND User:${user}", + "queryType": "lucene", + "refId": "TransferTotal", + "timeField": "timestamp", + "units": "percentunit", + "valueHandler": "Number Threshold", + "warn": 0.95 + } + ], + "title": "Upload status from ${site}", + "type": "vonage-status-panel" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 2, + "panels": [], + "repeat": "site", + "repeatDirection": "h", + "title": "${site}", + "type": "row" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Destination", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:Successful AND ExecutionSite:${site} AND Protocol:${protocol} AND User:${user} AND Destination:${destination}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Successful upload from ${site}", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Destination", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:Failed AND Destination:${destination} AND ExecutionSite:${site} AND Protocol:${protocol} AND User:${user}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Failed upload from ${site}", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 24, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Source", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.getFile AND FinalStatus:Successful AND ExecutionSite:${site} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Successful download from ${site}", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 25, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Source", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.getFile AND FinalStatus:Failed AND ExecutionSite:${site} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Failed download from ${site}", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-RdYlGr" + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 70 + }, + { + "color": "red", + "value": 85 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 100, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Protocol", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "FinalStatus", + "id": "4", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.getFile AND FinalStatus:Successful AND ExecutionSite:${site} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Download by protocol at ${site}", + "type": "gauge" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-RdYlGr" + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 70 + }, + { + "color": "red", + "value": 85 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 101, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Protocol", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "FinalStatus", + "id": "4", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND ExecutionSite:${site} AND Protocol:${protocol} AND User:${user} AND Destination:${destination} AND FinalStatus:${finalStatus}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Upload by protocol at ${site}", + "type": "gauge" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 4.8, + "x": 0, + "y": 34 + }, + "id": 156, + "maxPerRow": 8, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "percent" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "repeat": "protocol", + "repeatDirection": "h", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "FinalStatus", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND ExecutionSite:${site} AND Protocol:${protocol} AND User:${user} AND Destination:${destination} AND FinalStatus:${finalStatus}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "${protocol} upload performance at ${site}", + "type": "piechart" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 4.8, + "x": 0, + "y": 42 + }, + "id": 245, + "maxPerRow": 8, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "percent" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "repeat": "protocol", + "repeatDirection": "h", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "FinalStatus", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.getFile AND ExecutionSite:${site} AND Protocol:${protocol} AND User:${user} AND Source:${source} AND FinalStatus:${finalStatus}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "${protocol} download performance at ${site}", + "type": "piechart" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 50 + }, + "id": 57, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "hide": false, + "id": "1", + "settings": { + "size": "500" + }, + "type": "raw_data" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:Successful AND ExecutionSite:${site} AND Protocol:${protocol} AND User:${user} AND Destination:${destination}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Upload throughput from ${site}", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "Throughput", + "binary": { + "left": "TransferSize", + "operator": "/", + "reducer": "sum", + "right": "TransferTime" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": false + } + }, + { + "id": "groupBy", + "options": { + "fields": { + "Destination": { + "aggregations": [], + "operation": "groupby" + }, + "Source": { + "aggregations": [] + }, + "Throughput": { + "aggregations": [ + "last" + ], + "operation": "aggregate" + } + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 50 + }, + "id": 56, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "hide": false, + "id": "1", + "settings": { + "size": "500" + }, + "type": "raw_data" + } + ], + "query": "OperationType:se.getFile AND FinalStatus:Successful AND ExecutionSite:${site} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Download throughput from ${site}", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "Throughput", + "binary": { + "left": "TransferSize", + "operator": "/", + "reducer": "sum", + "right": "TransferTime" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": false + } + }, + { + "id": "groupBy", + "options": { + "fields": { + "Source": { + "aggregations": [], + "operation": "groupby" + }, + "Throughput": { + "aggregations": [ + "last" + ], + "operation": "aggregate" + } + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 58 + }, + "id": 377, + "options": { + "calculate": false, + "cellGap": 1, + "cellValues": { + "unit": "percentunit" + }, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "max": 1, + "min": 0, + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlGn", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": -1 + }, + "legend": { + "show": false + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "show": true, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": true + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Destination", + "id": "6", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "1", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": true, + "metrics": [ + { + "field": "TransferOK", + "id": "4", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:${finalStatus} AND ExecutionSite:${site} AND Protocol:${protocol} AND User:${user} AND Destination:${destination}", + "queryType": "lucene", + "refId": "TransferOK", + "timeField": "timestamp" + }, + { + "alias": "", + "bucketAggs": [ + { + "field": "Destination", + "id": "6", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "1", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": true, + "metrics": [ + { + "field": "TransferTotal", + "id": "5", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:${finalStatus} AND ExecutionSite:${site} AND Protocol:${protocol} AND User:${user}", + "queryType": "lucene", + "refId": "TransferTotal", + "timeField": "timestamp" + }, + { + "datasource": { + "type": "__expr__", + "uid": "${DS_EXPRESSION}" + }, + "expression": "1-$TransferOK/$TransferTotal", + "hide": false, + "refId": "To", + "type": "math" + } + ], + "title": "Upload from ${site}", + "transformations": [], + "type": "heatmap" + }, + { + "cards": { + "cardHSpacing": 2, + "cardMinWidth": 5, + "cardVSpacing": 2 + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateGnYlRd", + "defaultColor": "#757575", + "exponent": 0.5, + "mode": "spectrum", + "thresholds": [] + }, + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 58 + }, + "hideBranding": false, + "highlightCards": true, + "id": 612, + "legend": { + "show": true + }, + "nullPointMode": "as empty", + "pageSize": 15, + "pluginVersion": "9.1.6", + "seriesFilterIndex": -1, + "statusmap": { + "ConfigVersion": "v1" + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Destination", + "id": "6", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "9", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferOK", + "hide": true, + "id": "4", + "settings": { + "missing": "-1" + }, + "type": "sum" + }, + { + "field": "TransferTotal", + "hide": true, + "id": "7", + "type": "sum" + }, + { + "id": "8", + "pipelineVariables": [ + { + "name": "ok", + "pipelineAgg": "4" + }, + { + "name": "all", + "pipelineAgg": "7" + } + ], + "settings": { + "script": "params.ok / params.all" + }, + "type": "bucket_script" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:${finalStatus} AND ExecutionSite:${site} AND Protocol:${protocol} AND User:${user} AND Destination:${destination}", + "queryType": "lucene", + "refId": "TransferOK", + "timeField": "timestamp" + } + ], + "title": "Upload from ${site}", + "tooltip": { + "extraInfo": "", + "freezeOnClick": true, + "items": [], + "show": true, + "showExtraInfo": false, + "showItems": false + }, + "transformations": [], + "type": "flant-statusmap-panel", + "useMax": true, + "usingPagination": false, + "xAxis": { + "show": true + }, + "yAxis": { + "maxWidth": -1, + "minWidth": -1, + "show": true + }, + "yAxisSort": "metrics", + "yLabel": { + "delimiter": "", + "labelTemplate": "", + "usingSplitLabel": false + } + } + ], + "refresh": false, + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": {}, + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "definition": "{\"find\": \"terms\", \"field\": \"Destination\" , \"size\":10000}", + "hide": 0, + "includeAll": true, + "label": "Destination", + "multi": true, + "name": "destination", + "options": [], + "query": "{\"find\": \"terms\", \"field\": \"Destination\" , \"size\":10000}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "definition": "{\"find\": \"terms\", \"field\": \"ExecutionSite\" , \"size\":10000}", + "hide": 0, + "includeAll": true, + "label": "ExecutionSite", + "multi": true, + "name": "executionSite", + "options": [], + "query": "{\"find\": \"terms\", \"field\": \"ExecutionSite\" , \"size\":10000}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "definition": "{\"find\": \"terms\", \"field\": \"User\" , \"size\":10000}", + "hide": 0, + "includeAll": true, + "label": "User", + "multi": true, + "name": "user", + "options": [], + "query": "{\"find\": \"terms\", \"field\": \"User\" , \"size\":10000}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "definition": "{\"find\": \"terms\", \"field\": \"Protocol\" , \"size\":10000 , \"query\":\"OperationType:se*\"}", + "hide": 0, + "includeAll": true, + "label": "Protocol", + "multi": true, + "name": "protocol", + "options": [], + "query": "{\"find\": \"terms\", \"field\": \"Protocol\" , \"size\":10000 , \"query\":\"OperationType:se*\"}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "definition": "{\"find\": \"terms\", \"field\": \"Source\" , \"size\":10000}", + "hide": 0, + "includeAll": true, + "label": "Source", + "multi": true, + "name": "source", + "options": [], + "query": "{\"find\": \"terms\", \"field\": \"Source\" , \"size\":10000}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "definition": "{\"find\": \"terms\", \"field\": \"FinalStatus\" , \"size\":10000}", + "hide": 0, + "includeAll": true, + "label": "Status", + "multi": true, + "name": "finalStatus", + "options": [], + "query": "{\"find\": \"terms\", \"field\": \"FinalStatus\" , \"size\":10000}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "definition": "{\"find\": \"terms\", \"field\": \"OperationType\" , \"size\":10000}", + "hide": 0, + "includeAll": true, + "label": "Operation", + "multi": true, + "name": "operationType", + "options": [], + "query": "{\"find\": \"terms\", \"field\": \"OperationType\" , \"size\":10000}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": "*", + "current": { + "selected": false, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "hide": 2, + "includeAll": true, + "label": "Site", + "multi": true, + "name": "site", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "LCG.CERN.cern", + "value": "LCG.CERN.cern" + }, + { + "selected": false, + "text": "LCG.CNAF.it", + "value": "LCG.CNAF.it" + }, + { + "selected": false, + "text": "LCG.IN2P3.fr", + "value": "LCG.IN2P3.fr" + }, + { + "selected": false, + "text": "LCG.GRIDKA.de", + "value": "LCG.GRIDKA.de" + }, + { + "selected": false, + "text": "LCG.PIC.es", + "value": "LCG.PIC.es" + }, + { + "selected": false, + "text": "LCG.RAL.uk", + "value": "LCG.RAL.uk" + }, + { + "selected": false, + "text": "LCG.RRCKI.ru", + "value": "LCG.RRCKI.ru" + } + ], + "query": "LCG.CERN.cern,LCG.CNAF.it,LCG.IN2P3.fr,LCG.GRIDKA.de,LCG.PIC.es,LCG.RAL.uk,LCG.RRCKI.ru", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Tier1 DMS", + "uid": "XVEc6G4Vk", + "version": 55, + "weekStart": "" +} diff --git a/src/DIRAC/MonitoringSystem/private/Plotters/DataOperationPlotter.py b/src/DIRAC/MonitoringSystem/private/Plotters/DataOperationPlotter.py deleted file mode 100644 index 6d1d515181c..00000000000 --- a/src/DIRAC/MonitoringSystem/private/Plotters/DataOperationPlotter.py +++ /dev/null @@ -1,227 +0,0 @@ -""" -this class is used to define the plot using the plot attributes. -""" - -from DIRAC import S_OK, S_ERROR -from DIRAC.MonitoringSystem.Client.Types.DataOperation import DataOperation -from DIRAC.MonitoringSystem.private.Plotters.BasePlotter import BasePlotter - - -class DataOperationPlotter(BasePlotter): - - """ - .. class:: DataOperationPlotter - - It is used to create the plots. - - param: str _typeName monitoring type - param: list _typeKeyFields list of keys what we monitor (list of attributes) - """ - - _typeName = "DataOperation" - - _typeKeyFields = DataOperation().keyFields - - _reportSuceededTransfersName = "Successful transfers" - - def _reportSuceededTransfers(self, reportRequest): - return self.__reportTransfers(reportRequest, "Succeeded", ("Failed", 0)) - - _reportFailedTransfersName = "Failed transfers" - - def _reportFailedTransfers(self, reportRequest): - return self.__reportTransfers(reportRequest, "Failed", ("Succeeded", 1)) - - def __reportTransfers(self, reportRequest, titleType, togetherFieldsToPlot): - retVal = self._getTimedData( - reportRequest["startTime"], - reportRequest["endTime"], - "TransferTotal", - preCondDict=reportRequest["condDict"], - metadataDict=None, - ) - if not retVal["OK"]: - return retVal - dataDict, granularity = retVal["Value"] - - return S_OK({"data": dataDict, "granularity": granularity, "unit": "files"}) - - def _plotSuceededTransfers(self, reportRequest, plotInfo, filename): - return self.__plotTransfers(reportRequest, plotInfo, filename, "Succeeded", ("Failed", 0)) - - def _plotFailedTransfers(self, reportRequest, plotInfo, filename): - return self.__plotTransfers(reportRequest, plotInfo, filename, "Failed", ("Succeeded", 1)) - - def __plotTransfers(self, reportRequest, plotInfo, filename, titleType, togetherFieldsToPlot): - metadata = { - "title": "{} Transfers by {}".format(titleType, reportRequest["grouping"]), - "ylabel": plotInfo["unit"], - "starttime": reportRequest["startTime"], - "endtime": reportRequest["endTime"], - "span": plotInfo["granularity"], - } - return self._generateTimedStackedBarPlot(filename, plotInfo["data"], metadata) - - _reportQualityName = "Transfer Efficiency" - - def _reportQuality(self, reportRequest): - """It is used to retrieve the data from the database. - - :param dict reportRequest: contains attributes used to create the plot. - :return: S_OK or S_ERROR {'data':value1, 'granularity':value2} value1 is a dictionary, value2 is the bucket length - """ - # Retrieve the number of succeded transfers - retVal = self._getTimedData( - reportRequest["startTime"], - reportRequest["endTime"], - "TransferOK", - preCondDict=reportRequest["condDict"], - metadataDict=None, - ) - # Retrieve the number of total transfers - retTotVal = self._getTimedData( - reportRequest["startTime"], - reportRequest["endTime"], - "TransferTotal", - preCondDict=reportRequest["condDict"], - metadataDict=None, - ) - if not retVal["OK"]: - return retVal - if not retTotVal["OK"]: - return retTotVal - dataDict, granularity = retVal["Value"] - totDataDict, granularity = retTotVal["Value"] - # Check that the dicts are not empty - if dataDict and totDataDict: - # Return the efficiency in dataDict - effDict = self._calculateEfficiencyDict(totDataDict, dataDict) - return S_OK({"data": effDict, "granularity": granularity}) - return S_OK("No data available for this plot selection.") - - def _plotQuality(self, reportRequest, plotInfo, filename): - """Make 2 dimensional pilotSubmission efficiency plot - - :param dict reportRequest: Condition to select data - :param dict plotInfo: Data for plot. - :param str filename: File name - """ - - metadata = { - "title": "Transfer quality by %s" % reportRequest["grouping"], - "starttime": reportRequest["startTime"], - "endtime": reportRequest["endTime"], - "span": plotInfo["granularity"], - } - return self._generateQualityPlot(filename, plotInfo["data"], metadata) - - _reportTransferedDataName = "Cumulative transferred data" - - def _reportTransferedData(self, reportRequest): - """It is used to retrieve the data from the database. - - :param dict reportRequest: contains attributes used to create the plot. - :return: S_OK or S_ERROR {'data':value1, 'granularity':value2} value1 is a dictionary, value2 is the bucket length - """ - retVal = self._getTimedData( - reportRequest["startTime"], - reportRequest["endTime"], - "TransferSize", - preCondDict=reportRequest["condDict"], - metadataDict=None, - ) - if not retVal["OK"]: - return retVal - dataDict, granularity = retVal["Value"] - return S_OK({"data": dataDict, "granularity": granularity}) - - def _plotTransferedData(self, reportRequest, plotInfo, filename): - """It creates the plot. - - :param dict reportRequest: plot attributes - :param dict plotInfo: contains all the data which are used to create the plot - :param str filename: - :return: S_OK or S_ERROR { 'plot' : value1, 'thumbnail' : value2 } value1 and value2 are TRUE/FALSE - """ - metadata = { - "title": "Transfered data by %s" % reportRequest["grouping"], - "starttime": reportRequest["startTime"], - "endtime": reportRequest["endTime"], - "span": plotInfo["granularity"], - "ylabel": "MB", - "sort_labels": "last_value", - } - return self._generateCumulativePlot(filename, plotInfo["data"], metadata) - - def _reportThroughput(self, reportRequest): - """It is used to retrieve the data from the database. - - :param dict reportRequest: contains attributes used to create the plot. - :return: S_OK or S_ERROR {'data':value1, 'granularity':value2} value1 is a dictionary, value2 is the bucket length - """ - retVal = self._getTimedData( - reportRequest["startTime"], - reportRequest["endTime"], - "TransferSize", - preCondDict=reportRequest["condDict"], - metadataDict=None, - ) - if not retVal["OK"]: - return retVal - dataDict, granularity = retVal["Value"] - return S_OK({"data": dataDict, "granularity": granularity, "unit": "Bytes"}) - - def _plotThroughput(self, reportRequest, plotInfo, filename): - """It creates the plot. - - :param dict reportRequest: plot attributes - :param dict plotInfo: contains all the data which are used to create the plot - :param str filename: - :return: S_OK or S_ERROR { 'plot' : value1, 'thumbnail' : value2 } value1 and value2 are TRUE/FALSE - """ - metadata = { - "title": "Throughput by %s" % reportRequest["grouping"], - "ylabel": plotInfo["unit"], - "starttime": reportRequest["startTime"], - "endtime": reportRequest["endTime"], - "span": plotInfo["granularity"], - } - return self._generateTimedStackedBarPlot(filename, plotInfo["data"], metadata) - - _reportDataTransferedName = "Pie chart of transferred data" - - def _reportDataTransfered(self, reportRequest): - """It is used to retrieve the data from the database. - - :param dict reportRequest: contains attributes used to create the plot. - :return: S_OK or S_ERROR {'data':value1, 'granularity':value2} value1 is a dictionary, value2 is the bucket length - """ - retVal = self._getTimedData( - reportRequest["startTime"], - reportRequest["endTime"], - "TransferSize", - preCondDict=reportRequest["condDict"], - metadataDict=None, - ) - if not retVal["OK"]: - return retVal - dataDict, granularity = retVal["Value"] - dataDict = self._sumDictValues(dataDict) - for key in dataDict: - dataDict[key] = int(dataDict[key]) - return S_OK({"data": dataDict}) - - def _plotDataTransfered(self, reportRequest, plotInfo, filename): - """It creates the plot. - - :param dict reportRequest: plot attributes - :param dict plotInfo: contains all the data which are used to create the plot - :param str filename: - :return: S_OK or S_ERROR { 'plot' : value1, 'thumbnail' : value2 } value1 and value2 are TRUE/FALSE - """ - metadata = { - "title": "Total transfered data [bytes] by %s" % reportRequest["grouping"], - "starttime": reportRequest["startTime"], - "endtime": reportRequest["endTime"], - } - return self._generatePiePlot(filename, plotInfo["data"], metadata) From bfc2979888ccf07c088f9b0ca5092f91910f33f6 Mon Sep 17 00:00:00 2001 From: chaen Date: Wed, 12 Oct 2022 17:16:39 +0200 Subject: [PATCH 0026/1135] sweep: #6381 Backport ResolveSE from LHCbDIRAC --- dirac.cfg | 153 ++++++++- .../Resources/FileCatalogs/index.rst | 16 - .../ConfReference/Resources/Sites/index.rst | 265 --------------- .../ConfReference/Resources/index.rst | 13 - .../Configuration/ConfReference/index.rst | 1 - .../ConfigurationStructure/index.rst | 2 +- .../AdministratorGuide/Resources/country.rst | 14 + .../AdministratorGuide/Resources/index.rst | 2 + .../AdministratorGuide/Resources/site.rst | 45 +++ .../AdministratorGuide/Resources/storage.rst | 11 +- .../Client/test/Test_FTS3Objects.py | 36 +-- .../Utilities/DMSHelpers.py | 18 +- .../Utilities/ResolveSE.py | 118 +++++++ .../Utilities/test/Test_resolveSE.py | 306 ++++++++++++++++++ .../Utilities/test/__init__.py | 0 .../Storage/test/Test_StorageFactory.py | 32 +- .../JobWrapper/JobWrapper.py | 10 +- src/DIRAC/tests/Utilities/utils.py | 60 ++++ 18 files changed, 744 insertions(+), 358 deletions(-) delete mode 100644 docs/source/AdministratorGuide/Configuration/ConfReference/Resources/FileCatalogs/index.rst delete mode 100644 docs/source/AdministratorGuide/Configuration/ConfReference/Resources/Sites/index.rst delete mode 100644 docs/source/AdministratorGuide/Configuration/ConfReference/Resources/index.rst create mode 100644 docs/source/AdministratorGuide/Resources/country.rst create mode 100644 docs/source/AdministratorGuide/Resources/site.rst create mode 100644 src/DIRAC/DataManagementSystem/Utilities/ResolveSE.py create mode 100644 src/DIRAC/DataManagementSystem/Utilities/test/Test_resolveSE.py create mode 100644 src/DIRAC/DataManagementSystem/Utilities/test/__init__.py diff --git a/dirac.cfg b/dirac.cfg index 1247730d4f9..a93025a3f1d 100644 --- a/dirac.cfg +++ b/dirac.cfg @@ -765,7 +765,7 @@ Resources } } # FTS endpoint definition http://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/DataManagement/fts.htmlfts-servers-definition - # Passed to the constructor of the pluginFTSEndpoints + FTSEndpoints { FTS3 { @@ -852,6 +852,157 @@ Resources # This option is used in the Job Wrapper and, if set, requires the RequestManagementSystem to be installed Tier1-Failover = CERN-FAILOVER,CNAF-FAILOVER } + # Definition of the sites + # See http://dirac.readthedocs.io/en/latest/AdministratorGuide/Resources/site.html + Sites + { + LCG + { + ##BEGIN SiteConfiguration + LCG.CERN.ch + { + # Local Storages + SE = CERN-RAW, CERN-RDST, CERN-USER # (Optional) SEs Local to the site + + # Overwrite definities of StorageElement (discouraged) + # or StorageElementGroups for that Site + AssociatedSEs + { + # Tier1-Failover is now only CERN-FAILOVER when running a Job at CERN + Tier1-Failover = CERN-FAILOVER + } + Name = CERN-PROD # (Optional) Name of the site from the admin, e.g in GOCDB + Coordinates = 06.0458:46.2325 # (Optional) Geographical coordinates + Mail = grid-cern-prod-admins@cern.ch # (Optional) Site Admin email + MoUTierLevel = 0 # (Optional) Tier level + Description = CERN European Organization for Nuclear Research # (Optional) ... + # Subsection to describe each CE available + CEs + { + # Subsection named as the CE fully qualified name + ce503.cern.ch + { + + # (Optional) CE architecture + architecture = x86_64 + + # (Optional) CE operating system in a DIRAC format (purely for description) + OS = ScientificCERNSLC_Carbon_6.4 + + # (Optional) Boolean attributes that indicates if the site accept pilots (default: True) + Pilot = False + + # Type of CE, can take any CE type DIRAC recognizes (:ref: `CE`) + CEType = HTCondorCE + + # (Optional) Type of 'Inner' CE, normally empty. Default = "InProcess". + # Possibilities: potentially all CE types, but in practice + # the most valid would be: InProcess, Sudo, Singularity, Pool. + # Pool CE in turn uses InProcess (Default) + # or Sudo or Singularity. To specify, use Pool/ce_type. + # This option can also go at the Queue level. + LocalCEType = Pool + + # If the CE is a cream CE the mode of submission + SubmissionMode = Direct + + # (Optional) max number of processors that DIRAC pilots are allowed to exploit. Implicit default = 1 + NumberOfProcessors = 12 + + # (Optional) Number of available worker nodes per allocation. + # Values can be a number (e.g. 2 nodes) or a range of values + # (e.g. from 2 to 4 nodes) which leaves the choice to the batch + # system. + NumberOfNodes = 2 + # NumberOfNodes = 2-4 + + # (Optional) CE allows *whole node* jobs + WholeNode = True + + # (Optional) List of tags specific for the CE + Tag = GPU, 96RAM + + # (Optional) List of required tags that a job to be eligible must have + RequiredTag = GPU,96RAM + + # Queues available for this VO in the CE + Queues + { + # Name of the queue + ce503.cern.ch-condor + { + # Name of the queue in the corresponding CE if not the same + # as the name of the queue section + # (should be avoided) + CEQueueName = pbs-grid + VO = lhcb + + # CE CPU Scaling Reference + SI00 = 3100 + + # Maximum number of jobs in all statuses + MaxTotalJobs = 5000 + + # Maximum number of jobs in waiting status + MaxWaitingJobs = 200 + + # Maximum time allowed to jobs to run in the queue + maxCPUTime = 7776 + + # The URL where to find the outputs + OutputURL = gsiftp://locahost + + # Overwrites NumberOfProcessors at the CE level + NumberOfProcessors = 12 + + # Overwrites NumberOfNodes at the CE level + NumberOfNodes = 12 + + # Overwrites WholeNode at the CE level + WholeNode = True + + # Overwrites LocalCEType at the CE level + LocalCEType = Pool/Singularity + + # List of tags specific for the Queue + Tag = MultiProcessor + + # List of required tags that a job to be eligible must have + RequiredTag = GPU,96RAM + } + } + VO = lhcb + MaxRAM = 0 + UseLocalSchedd = False + DaysToKeepLogs = 1 + } + } + } + ##END + } + } + + ##BEGIN CountriesConfiguration + Countries + { + # Configuration for ``pl`` sites + pl + { + # Redirect to ``de`` configuration + AssignedTo = de + } + de + { + AssociatedSEs + { + # Overwrite the Tier1-Failover StorageElementGroup + # For all German site which do not have a specific + # configuration (see # See https://dirac.readthedocs.io/en/latest/AdministratorGuide/Resources/storage.html#mapping-storages-to-sites-and-countries) + Tier1-Failover = GRIDKA-FAILOVER + } + } + } + ##END } Operations { diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Resources/FileCatalogs/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Resources/FileCatalogs/index.rst deleted file mode 100644 index defefecd85a..00000000000 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Resources/FileCatalogs/index.rst +++ /dev/null @@ -1,16 +0,0 @@ -Resources / FileCatalogs - Subsections -====================================== - -This subsection include the definition of the File Catalogs to be used in the installation. In case there is more than one File Catalog defined in this section, the first one in the section will be used as default by the ReplicaManager client. - -+---------------------------+-------------------------------------------------+----------------------------+ -| **Name** | **Description** | **Example** | -+---------------------------+-------------------------------------------------+----------------------------+ -| *FileCatalog* | Subsection used to configure DIRAC File catalog | FileCatalog | -+---------------------------+-------------------------------------------------+----------------------------+ -| *FileCatalog/AccessType* | Access type allowed to the particular catalog | AccessType = Read-Write | -+---------------------------+-------------------------------------------------+----------------------------+ -| *FileCatalog/Status* | To define the catalog as active or inactive | Status = Active | -+---------------------------+-------------------------------------------------+----------------------------+ -| *FileCatalog/MetaCatalog* | If the Catalog is a MetaDataCatalog | MetaCatalog = True | -+---------------------------+-------------------------------------------------+----------------------------+ diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Resources/Sites/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Resources/Sites/index.rst deleted file mode 100644 index de00b4588fb..00000000000 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Resources/Sites/index.rst +++ /dev/null @@ -1,265 +0,0 @@ -.. _cs-site: - -Resources / Sites - Subsections -=============================== - -In this section each DIRAC site available for the users is described. The convention to name the sites consist of 3 strings: - -- Grid site name, expressed in uppercase, for example: LCG, EELA -- Institution acronym in uppercase, for example: CPPM -- Country: country where the site is located, expressed in lowercase, for example fr - -The three strings are concatenated with "." to produce the name of the sites. - -+------------------------------------+-----------------------------------------------+-----------------------------------+ -| **Name** | **Description** | **Example** | -+------------------------------------+-----------------------------------------------+-----------------------------------+ -| ** | Subsection named with the site name | LCG.CPPM.fr | -+------------------------------------+-----------------------------------------------+-----------------------------------+ -| */Name* | Site name gave by the site administrator | Name = in2p3 | -| | e.g.: the name of the site in GOCDB (optional)| | -+------------------------------------+-----------------------------------------------+-----------------------------------+ -| */CE* | List of CEs using CE FQN | CE = ce01.in2p3.fr | -| | These CEs are updated by the BDII2CSAgent | CE += ce02.in2p3.fr | -| | in the CEs section | | -+------------------------------------+-----------------------------------------------+-----------------------------------+ -| */MoUTierLevel* | Tier Level (optional) | MoUTierLevel = 1 | -+------------------------------------+-----------------------------------------------+-----------------------------------+ -| */CEs/* | Subsection used to describe each CE available | CEs | -+------------------------------------+-----------------------------------------------+-----------------------------------+ -| */Coordinates* | Site geographical coordinates (optional) | Coordinates = -8.637979:41.152461 | -+------------------------------------+-----------------------------------------------+-----------------------------------+ -| */Mail* | Mail address site responsable (optional) | Mail = atsareg@in2p3.fr | -+------------------------------------+-----------------------------------------------+-----------------------------------+ -| */SE* | Closest SE respect to the CE (optional) | SE = se01.in2p3.fr | -+------------------------------------+-----------------------------------------------+-----------------------------------+ - - -CEs sub-subsection -------------------- - -This sub-subsection specifies the attributes of each particular CE of the site. For each DIRAC site there can be more than one CE. - -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| **Name** | **Description** | **Example** | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| ** | Subsection named as the CE fully qualified name | ce01.in2p3.fr | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */architecture* | CE architecture | architecture = x86_64 | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */CEType* | Type of CE, can take values as LCG or CREAM | CEType = ARC | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */LocalCEType* | Type of 'Inner' CE, normally empty. Default = "InProcess". | LocalCEType = Pool | -| | Possibilities: potentially all CE types, but in practice | | -| | the most valid would be: InProcess, Sudo, Singularity, Pool.| | -| | Pool CE in turn uses InProcess (Default) | | -| | or Sudo or Singularity. To specify, use Pool/ce_type. | LocalCEType = Pool/Singularity | -| | This option can also go at the Queue level. | | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */OS* | CE operating system in a DIRAC format | OS = ScientificLinux_Boron_5.3 | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */Pilot* | Boolean attributes than indicates if the site accept pilots | Pilot = True | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */SubmissionMode* | If the CE is a cream CE the mode of submission | SubmissionMode = Direct | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */wnTmpDir* | Worker node temporal directory | wnTmpDir = /tmp | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */NumberOfProcessors* | Number of available processors on worker nodes | NumberOfProcessors = 12 | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */NumberOfNodes* | Number of available worker nodes per allocation. | NumberOfNodes = 2 | -| | Values can be a number (e.g. 2 nodes) or a range of values | | -| | (e.g. from 2 to 4 nodes) which lets the choice to the batch | NumberOfNodes = 2 or 2-4 | -| | system. | | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */WholeNode* | CE allows *whole node* jobs | WholeNode = True | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */Tag* | List of tags specific for the CE | Tag = GPU,96RAM | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */RequiredTag* | List of required tags that a job to be eligible must have | RequiredTag = GPU,96RAM | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */Queues* | Subsection. Queues available for this VO in the CE | Queues | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */Queues/* | Name of the queue exactly how is published | jobmanager-pbs-formation | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */Queues//CEQueueName* | Name of the queue in the corresponding CE if not the same | | -| | as the name of the queue section | CEQueueName = pbs-grid | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */Queues//maxCPUTime* | Maximum time allowed to jobs to run in the queue | maxCPUTime = 1440 | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */Queues//MaxTotalJobs* | If the CE is a CREAM CE the maximum number of jobs in all | MaxTotalJobs =200 | -| | the status | | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */Queues//MaxWaitingJobs* | If the CE is a CREAM CE the maximum number of jobs in | MaxWaitingJobs = 70 | -| | waiting status | | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */Queues//OutputURL* | If the CE is a CREAM CE the URL where to find the outputs | OutputURL = gsiftp://localhost | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */Queues//SI00* | CE CPU Scaling Reference | SI00 = 2130 | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */Queues//NumberOfProcessors* | overrides */NumberOfProcessors* at queue level | NumberOfProcessors = 12 | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */Queues//NumberOfNodes* | overrides */NumberOfNodes* | NumberOfNodes = 2 | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */Queues//WholeNode* | overrides */WholeNode* at queue level | WholeNode = True | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */Queues//Tag* | List of tags specific for the Queue | Tag = GPU,96RAM | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */Queues//RequiredTag* | List of required tags that a job to be eligible must have | RequiredTag = GPU,96RAM | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */Queues//LocalCEType* | Same as */LocalCEType* (see above) but per queue. | LocalCEType = Pool/Singularity | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ - - -An example for this session follows:: - - Sites - { - LCG - { - LCG.CERN.cern - { - SE = CERN-RAW - SE += CERN-RDST - SE += CERN-USER - CE = ce503.cern.ch - CE += ce504.cern.ch - Name = CERN-PROD - Coordinates = 06.0458:46.2325 - Mail = grid-cern-prod-admins@cern.ch - MoUTierLevel = 0 - Description = CERN European Organization for Nuclear Research - CEs - { - ce503.cern.ch - { - wnTmpDir = . - architecture = x86_64 - OS = ScientificCERNSLC_Carbon_6.4 - SI00 = 0 - Pilot = False - CEType = HTCondorCE - SubmissionMode = Direct - Queues - { - ce503.cern.ch-condor - { - VO = lhcb - VO += LHCb - SI00 = 3100 - MaxTotalJobs = 5000 - MaxWaitingJobs = 200 - maxCPUTime = 7776 - LocalCEType = Pool/Singularity - Tag = MultiProcessor - } - } - VO = lhcb - MaxRAM = 0 - UseLocalSchedd = False - DaysToKeepLogs = 1 - } - ce504.cern.ch - { - wnTmpDir = . - architecture = x86_64 - OS = ScientificCERNSLC_Carbon_6.4 - SI00 = 0 - Pilot = False - CEType = HTCondorCE - LocalCEType = Pool - SubmissionMode = Direct - Queues - { - ce504.cern.ch-condor - { - VO = lhcb - VO += LHCb - SI00 = 3100 - MaxTotalJobs = 5000 - MaxWaitingJobs = 200 - maxCPUTime = 7776 - } - } - } - } - } - } - DIRAC - { - DIRAC.HLTFarm.lhcb - { - Name = LHCb-HLTFARM - CE = OnlineCE.lhcb - CEs - { - OnlineCE.lhcb - { - CEType = CREAM - Queues - { - OnlineQueue - { - maxCPUTime = 2880 - } - } - } - } - AssociatedSEs - { - Tier1-RDST = CERN-RDST - Tier1_MC-DST = CERN_MC-DST-EOS - Tier1-Buffer = CERN-BUFFER - Tier1-Failover = CERN-EOS-FAILOVER - Tier1-BUFFER = CERN-BUFFER - Tier1-USER = CERN-USER - SE-USER = CERN-USER - } - } - } - VAC - { - VAC.Manchester.uk - { - Name = UKI-NORTHGRID-MAN-HEP - CE = vac01.blackett.manchester.ac.uk - CE += vac02.blackett.manchester.ac.uk - Coordinates = -2.2302:53.4669 - Mail = ops@NOSPAMtier2.hep.manchester.ac.uk - CEs - { - vac01.blackett.manchester.ac.uk - { - CEType = Vac - architecture = x86_64 - OS = ScientificSL_Carbon_6.4 - wnTmpDir = /scratch - SI00 = 2200 - MaxCPUTime = 1000 - Queues - { - default - { - maxCPUTime = 1000 - } - } - } - vac02.blackett.manchester.ac.uk - { - CEType = Vac - architecture = x86_64 - OS = ScientificSL_Carbon_6.4 - wnTmpDir = /scratch - SI00 = 2200 - MaxCPUTime = 1000 - Queues - { - default - { - maxCPUTime = 1000 - } - } - } - } - } - } - } diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Resources/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Resources/index.rst deleted file mode 100644 index d18a2b3a176..00000000000 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Resources/index.rst +++ /dev/null @@ -1,13 +0,0 @@ -.. _dirac-resources-cs: - -Resources - Section -=================== - -In this section all the physical resources than can be used by DIRAC users are described. - - -.. toctree:: - :maxdepth: 2 - - FileCatalogs/index - Sites/index diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/index.rst index 080d132f3c9..15512da58a8 100644 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/index.rst +++ b/docs/source/AdministratorGuide/Configuration/ConfReference/index.rst @@ -17,7 +17,6 @@ The detailed configuration options for agents, services, and executors are in th :maxdepth: 1 Operations/index - Resources/index Systems/index WebSite/index Tips/index diff --git a/docs/source/AdministratorGuide/Configuration/ConfigurationStructure/index.rst b/docs/source/AdministratorGuide/Configuration/ConfigurationStructure/index.rst index e2415a1f3be..b4bc997436f 100644 --- a/docs/source/AdministratorGuide/Configuration/ConfigurationStructure/index.rst +++ b/docs/source/AdministratorGuide/Configuration/ConfigurationStructure/index.rst @@ -31,7 +31,7 @@ DIRAC Registry The *Registry* contains information about DIRAC users, groups and communities (VOs). -:ref:`Resources ` +Resources The *Resources* section provides description of all the DIRAC computing resources. This includes computing and storage elements as well as descriptions of several DIRAC and third party services. diff --git a/docs/source/AdministratorGuide/Resources/country.rst b/docs/source/AdministratorGuide/Resources/country.rst new file mode 100644 index 00000000000..3de42fd5156 --- /dev/null +++ b/docs/source/AdministratorGuide/Resources/country.rst @@ -0,0 +1,14 @@ +Countries +========= + +The country definition is mostly used for overwriting some ``StorageElementGroups`` for job output upload at the country level. See :ref:`storageMapping` + + +Configuration +------------- + +.. literalinclude:: /dirac.cfg + :start-after: ##BEGIN CountriesConfiguration + :end-before: ## + :dedent: 2 + :caption: Countries configuration diff --git a/docs/source/AdministratorGuide/Resources/index.rst b/docs/source/AdministratorGuide/Resources/index.rst index a0cd6930da5..ecbf596e041 100644 --- a/docs/source/AdministratorGuide/Resources/index.rst +++ b/docs/source/AdministratorGuide/Resources/index.rst @@ -16,6 +16,8 @@ contributing with their computing and storage capacity, available as conventiona .. toctree:: :maxdepth: 2 + site + country catalog computingelements messagequeues diff --git a/docs/source/AdministratorGuide/Resources/site.rst b/docs/source/AdministratorGuide/Resources/site.rst new file mode 100644 index 00000000000..5011a2c712d --- /dev/null +++ b/docs/source/AdministratorGuide/Resources/site.rst @@ -0,0 +1,45 @@ +.. _cs-site: + +Sites +===== + + +Site Names +---------- + +Sites have names resulting from the concatenation of: + +- Domain: Grid site name, expressed in uppercase, for example: LCG, EELA +- Site: Institution, for example: CPPM +- Country: country where the site is located, expressed in lowercase, for example fr + + +The full DIRAC Site Name becomes of the form: [Domain].[Site].[co]. +The full site names are used everywhere when the site resources are assigned to the context of a particular Domain: +in the accounting, monitoring, configuration of the Operations parameters, etc. + +Examples of valid site names are: + +* LCG.CERN.ch +* CLOUD.IN2P3.fr +* VAC.Manchester.uk +* DIRAC.farm.cern + +The [Domain] may imply a (set of) technologies used for exploiting the resources, even though this is not necessarily true. +The use of these Domains is mostly for reporting purposes, +and it is the responsibility of the administrator of the DIRAC installation to chose them +in such a way that they are meaningful for the communities and for the computing resources served by the installation. +In any case, DIRAC will always be a default Domain if nothing else is specified for a given resource. + +The Domain, Site and the country must be unique alphanumeric strings, irrespective of case, with a possible use of the following characters: "_" "-". + + + +Configuration +------------- + +.. literalinclude:: /dirac.cfg + :start-after: ##BEGIN SiteConfiguration + :end-before: ## + :dedent: 4 + :caption: Site configuration diff --git a/docs/source/AdministratorGuide/Resources/storage.rst b/docs/source/AdministratorGuide/Resources/storage.rst index 8136102adb5..a41fc700f11 100644 --- a/docs/source/AdministratorGuide/Resources/storage.rst +++ b/docs/source/AdministratorGuide/Resources/storage.rst @@ -336,8 +336,17 @@ Up to recently, any protocol that was defined as ``AccessProtocols`` was also us This is not true for `CTA `_ . Because ``CTA`` can stage with xroot only, but we may need to use another protocol to transfer to a another site, we need to distinguish between staging and accessing. To the best of my knowledge, only ``CTA`` is like this, and thus, it is the only place where you may need to define ``StageProtocols``. In case of FTS transfer from CTA where the stage and transfer protocols are different, we rely on the multihop mechanism of FTS to do the protocol translations. More technical details are available in :py:mod:`DIRAC.DataManagementSystem.Client.FTS3Job` --------------------- + StorageElementGroups -------------------- StorageElements can be grouped together in a ``StorageElementGroup``. This allows the systems or the users to refer to ``any storage within this group``. + + + +.. _storageMapping: + +Mapping Storages to Sites and Countries +--------------------------------------- + +Both ``Sites`` and ``Countries`` can have ``StorageElement`` (discouraged) or ``StorageElementGroup`` associated. This shows particularly useful if we want to restrict the job output upload to specific locations, due to network constraints for example. This is done using the ``AssociatedSEs`` parameter of the ``Site`` or ``Country``. The resolution order and logic is explained in :py:func:`~DIRAC.DataManagementSystem.Utilities.ResolveSE.getDestinationSEList` and well illustrated with examples in the `associated tests `_ diff --git a/src/DIRAC/DataManagementSystem/Client/test/Test_FTS3Objects.py b/src/DIRAC/DataManagementSystem/Client/test/Test_FTS3Objects.py index e28ee1a6959..00f5e93449c 100644 --- a/src/DIRAC/DataManagementSystem/Client/test/Test_FTS3Objects.py +++ b/src/DIRAC/DataManagementSystem/Client/test/Test_FTS3Objects.py @@ -4,6 +4,8 @@ import errno import DIRAC +from DIRAC.tests.Utilities.utils import generateDIRACConfig + from DIRAC.ConfigurationSystem.private.ConfigurationClient import ConfigurationClient from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData from diraccfg import CFG @@ -38,14 +40,7 @@ def mock_StorageFactory_generateStorageObject(self, storageName, pluginName, par return S_OK(storageObj) -@pytest.fixture(scope="module", autouse=True) -def generateConfig(): - """ - Generate the configuration that will be used for all the test - """ - - testCfgFileName = os.path.join(tempfile.gettempdir(), "test_FTS3Plugin.cfg") - cfgContent = """ +CFG_CONTENT = """ DIRAC { VirtualOrganization = lhcb @@ -160,23 +155,14 @@ def generateConfig(): } } } - """ - with open(testCfgFileName, "w") as f: - f.write(cfgContent) - # Load the configuration - ConfigurationClient(fileToLoadList=[testCfgFileName]) # we replace the configuration by our own one. - yield - - try: - os.remove(testCfgFileName) - except OSError: - pass - # SUPER UGLY: one must recreate the CFG objects of gConfigurationData - # not to conflict with other tests that might be using a local dirac.cfg - gConfigurationData.localCFG = CFG() - gConfigurationData.remoteCFG = CFG() - gConfigurationData.mergedCFG = CFG() - gConfigurationData.generateNewVersion() +""" + + +@pytest.fixture(scope="module", autouse=True) +def loadCS(): + """Load the CFG_CONTENT as a DIRAC Configuration for this module""" + with generateDIRACConfig(CFG_CONTENT, "test_FTS3Objects.cfg"): + yield @pytest.fixture(scope="function", autouse=True) diff --git a/src/DIRAC/DataManagementSystem/Utilities/DMSHelpers.py b/src/DIRAC/DataManagementSystem/Utilities/DMSHelpers.py index 7705964b0af..60505ecb713 100644 --- a/src/DIRAC/DataManagementSystem/Utilities/DMSHelpers.py +++ b/src/DIRAC/DataManagementSystem/Utilities/DMSHelpers.py @@ -11,6 +11,8 @@ PROTOCOL = LOCAL + 1 DOWNLOAD = PROTOCOL + 1 +sLog = gLogger.getSubLogger(__name__) + def resolveSEGroup(seGroupList, allSEs=None): """ @@ -26,7 +28,7 @@ def resolveSEGroup(seGroupList, allSEs=None): if allSEs is None: res = gConfig.getSections("/Resources/StorageElements") if not res["OK"]: - gLogger.fatal("Error getting list of SEs from CS", res["Message"]) + sLog.fatal("Error getting list of SEs from CS", res["Message"]) return [] allSEs = res["Value"] seList = [] @@ -43,7 +45,7 @@ def resolveSEGroup(seGroupList, allSEs=None): if se1 not in allSEs: # Here means se is not a group and is not an SE either, fatal! if se1 == se: - gLogger.fatal("%s is not a valid SE" % se1) + sLog.fatal("%s is not a valid SE" % se1) return [] # If not an SE, it may be a group recursive = resolveSEGroup(se1, allSEs=allSEs) @@ -112,7 +114,7 @@ def getSiteSEMapping(self): # BaseSE storageElements = gConfig.getSections("Resources/StorageElements") if not storageElements["OK"]: - gLogger.warn("Problem retrieving storage elements", storageElements["Message"]) + sLog.warn("Problem retrieving storage elements", storageElements["Message"]) return storageElements storageElements = storageElements["Value"] equivalentSEs = {} @@ -126,12 +128,12 @@ def getSiteSEMapping(self): siteSEMapping = {} gridTypes = gConfig.getSections("Resources/Sites/") if not gridTypes["OK"]: - gLogger.warn("Problem retrieving sections in /Resources/Sites", gridTypes["Message"]) + sLog.warn("Problem retrieving sections in /Resources/Sites", gridTypes["Message"]) return gridTypes gridTypes = gridTypes["Value"] - gLogger.debug("Grid Types are: %s" % (", ".join(gridTypes))) + sLog.debug("Grid Types are: %s" % (", ".join(gridTypes))) # Get a list of sites and their local SEs siteSet = set() storageElementSet = set() @@ -139,7 +141,7 @@ def getSiteSEMapping(self): for grid in gridTypes: result = gConfig.getSections("/Resources/Sites/%s" % grid) if not result["OK"]: - gLogger.warn("Problem retrieving /Resources/Sites/%s section" % grid) + sLog.warn("Problem retrieving /Resources/Sites/%s section" % grid) return result sites = result["Value"] siteSet.update(sites) @@ -188,7 +190,7 @@ def getSiteSEMapping(self): # Add storage elements that may not be associated with a site result = gConfig.getSections("/Resources/StorageElements") if not result["OK"]: - gLogger.warn("Problem retrieving /Resources/StorageElements section", result["Message"]) + sLog.warn("Problem retrieving /Resources/StorageElements section", result["Message"]) return result self.storageElementSet = storageElementSet | set(result["Value"]) self.siteSet = siteSet @@ -400,7 +402,7 @@ def getAllSEsInGroupAtSite(self, seGroup, site): return sesAtSite foundSEs = set(seList) & set(sesAtSite["Value"]) if not foundSEs: - gLogger.warn("No SE found at that site", f"in group {seGroup} at {site}") + sLog.warn("No SE found at that site", f"in group {seGroup} at {site}") return S_OK() return S_OK(sorted(foundSEs)) diff --git a/src/DIRAC/DataManagementSystem/Utilities/ResolveSE.py b/src/DIRAC/DataManagementSystem/Utilities/ResolveSE.py new file mode 100644 index 00000000000..dd5d3149459 --- /dev/null +++ b/src/DIRAC/DataManagementSystem/Utilities/ResolveSE.py @@ -0,0 +1,118 @@ +""" This module allows to resolve output SEs for Job based +on SE and site/country association +""" +from random import shuffle + +from DIRAC import gLogger, gConfig +from DIRAC.Core.Utilities.SiteSEMapping import getSEsForSite +from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup + +sLog = gLogger.getSubLogger(__name__) + + +def _setLocalFirst(seList, localSEs): + """return a shuffled list of SEs from seList, localSEs being first.""" + # Make a copy not to change the original order + seList = list(seList) + shuffle(seList) + # localSEs are put first in the list + return sorted(seList, key=lambda x: x not in localSEs) + + +def getDestinationSEList(outputSE, site, outputmode="Any"): + """Evaluate the output SE list from a workflow and return the concrete list + of SEs to upload output data. The resolution order goes as follow: + + * outputSE as a normal StorageElement + * outputSE as an alias of one SE defined in the ``site`` AssociatedSEs (return local first) + * outputSE as an alias of multiple SE (local SE should come first) + * outputSE as a StorageElementGroup + + Moreover, if output mode is `Local`: + + * return ONLY local SE within the SEGroup if they exist (i.e. in the ``/SE>`` config) + * look at associated countries and countries association + + + :param str outputSE: name of the SE or SEGroup we want to resolve + :param str site: site on which we are running + :param str outputmode: (default "Any") resolution mode + + :returns: list of string + + :raises: + RuntimeError if anything is wrong + + """ + + if not outputSE: + return [] + + if outputmode.lower() not in ("any", "local"): + raise RuntimeError("Unexpected outputmode") + + # Add output SE defined in the job description + sLog.info("Resolving workflow output SE description", str(outputSE)) + + # Check if the SE is defined explicitly for the site + prefix = site.split(".")[0] + country = site.split(".")[-1] + + # Concrete SE name + result = gConfig.getOptions(f"/Resources/StorageElements/{outputSE}") + if result["OK"]: + sLog.info("Found concrete SE", str(outputSE)) + return [outputSE] + + # Get local SEs + localSEs = getSEsForSite(site) + if not localSEs["OK"]: + raise RuntimeError(localSEs["Message"]) + localSEs = localSEs["Value"] + sLog.verbose("Local SE list is:", ", ".join(localSEs)) + + # There is an alias defined for this Site + associatedSEs = gConfig.getValue(f"/Resources/Sites/{prefix}/{site}/AssociatedSEs/{outputSE}", []) + if associatedSEs: + associatedSEs = _setLocalFirst(associatedSEs, localSEs) + sLog.info("Found associated SE for site", f"{associatedSEs} associated to {site}") + return associatedSEs + + groupSEs = resolveSEGroup(outputSE) + if not groupSEs: + raise RuntimeError(f"Failed to resolve SE {outputSE}") + sLog.verbose("Group SE list is:", str(groupSEs)) + + # Find a local SE or an SE considered as local because the country is associated to it + if outputmode.lower() == "local": + # First, check if one SE in the group is local + ses = list(set(localSEs) & set(groupSEs)) + if ses: + sLog.info("Found eligible local SE", str(ses)) + return ses + + # Final check for country associated SE + assignedCountry = country + while True: + # check if country is already one with associated SEs + section = f"/Resources/Countries/{assignedCountry}/AssociatedSEs/{outputSE}" + associatedSEs = gConfig.getValue(section, []) + if associatedSEs: + associatedSEs = _setLocalFirst(associatedSEs, localSEs) + sLog.info("Found associated SEs", f"{associatedSEs} in {section}") + return associatedSEs + + opt = gConfig.getOption(f"/Resources/Countries/{assignedCountry}/AssignedTo") + if opt["OK"] and opt["Value"]: + assignedCountry = opt["Value"] + else: + # No associated SE and no assigned country, give up + raise RuntimeError( + f"Could not establish associated SE nor assigned country for country {assignedCountry}" + ) + + # For collective Any and All modes return the whole group + # Make sure that local SEs are passing first + orderedSEs = _setLocalFirst(groupSEs, localSEs) + sLog.info("Found SEs, local first:", str(orderedSEs)) + return orderedSEs diff --git a/src/DIRAC/DataManagementSystem/Utilities/test/Test_resolveSE.py b/src/DIRAC/DataManagementSystem/Utilities/test/Test_resolveSE.py new file mode 100644 index 00000000000..5bedf465ef6 --- /dev/null +++ b/src/DIRAC/DataManagementSystem/Utilities/test/Test_resolveSE.py @@ -0,0 +1,306 @@ +import os +import pytest +import tempfile + + +from DIRAC.tests.Utilities.utils import generateDIRACConfig + +from DIRAC.DataManagementSystem.Utilities.ResolveSE import getDestinationSEList +from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup + +from DIRAC import gLogger + +gLogger.setLevel("DEBUG") + + +CFG_CONTENT = """ +Resources +{ + StorageElements + { + LogSE + { + } + CERN-BUFFER + { + } + CERN-FAILOVER + { + } + CERN-DST + { + } + CNAF-BUFFER + { + } + CNAF-FAILOVER + { + } + CNAF-DST + { + } + GRIDKA-BUFFER + { + } + GRIDKA-FAILOVER + { + } + GRIDKA-DST + { + } + } + StorageElementGroups + { + # Define a StorageElementGroup which is not + # overwriten in any associatedSEs + Tier1-Failover = CERN-FAILOVER, CNAF-FAILOVER + + # Define a SEGroup which will be + # overwriten at CERN only + Tier1-DST = CERN-DST, CNAF-DST + } + Sites + { + LCG + { + LCG.CERN.cern + { + # Local SEs + SE = CERN-BUFFER, CERN-LogSE, CERN-FAILOVER, CNAF-DST + + AssociatedSEs + { + + # This should have no impact because + # LogSE exists as an SE, so we do not + # check associated SE + + LogSE = CERN-LogSE + + # Here define a name that is used as an alias + + LocalLogSE = CERN-LogSE + + # Tier1-Buffer is defined as associated site + # for all Sites. + # It's not a real use case. + # We would normally define a StorageElementGroup + # and overwrite it if needed in some sites + # (like what we do for Tier1-DST) + Tier1-Buffer = CERN-BUFFER, CNAF-BUFFER + + # Overwite the definition of the Tier1-DST group + # for this site only + Tier1-DST = CERN-DST + } + } + + LCG.CNAF.it + { + SE = CNAF-BUFFER, CNAF-FAILOVER, CNAF-DST + AssociatedSEs + { + # We do NOT define CNAF-LogSE as local, yet it is still returned + # when using ``outputmode = local`` because ``local`` only impacts + # ``StorageElementGroups`` + LocalLogSE = CNAF-LogSE + + # We anyway expect CNAF-BUFFER to be returned first, as it is local + Tier1-Buffer = CERN-BUFFER, CNAF-BUFFER + } + } + + # Do not update anything for this site + LCG.IN2P3.fr + { + } + + # Do not update anything for this site + # but define some aliases in the ``de`` country + LCG.GRIDKA.de + { + } + } + } + + # Countries are taking into account only when using ``Local`` mode + Countries + { + pl + { + # Associate Polish sites to german sites + AssignedTo = de + } + de + { + AssociatedSEs + { + + # This alias will never be used + # because ``LocalLogSE`` is not a + # ``StorageElementGroup``, so the resolution + # does not take place, even when used with ``Local`` + LocalLogSE = GRIDKA-LogSE + + # As for LocalLogSE, we will never reach that + # alias + Tier1-Buffer = GRIDKA-BUFFER + + Tier1-DST = GRIDKA-DST + + } + } + } +} +""" + + +@pytest.fixture(scope="module", autouse=True) +def loadCS(): + """Load the CFG_CONTENT as a DIRAC Configuration for this module""" + with generateDIRACConfig(CFG_CONTENT, "test_resolveSE.cfg"): + yield + + +def test_directSEName(): + """Should return the LogSE as is, full stop""" + # Asking for a specific SE should return that SE, no matter what + for site in ("LCG.CERN.cern", "LCG.CNAF.it", "LCG.IN2P3.fr", "LCG.GRIDKA.de", "LCG.NCBJ.pl", "AnySite"): + assert getDestinationSEList("LogSE", site) == ["LogSE"] + assert getDestinationSEList("LogSE", site, outputmode="Local") == ["LogSE"] + + +def test_emptyInput(): + """Should return empty list""" + + assert getDestinationSEList("", "site") == [] + assert getDestinationSEList("", "site", outputmode="Local") == [] + assert getDestinationSEList([], "site") == [] + assert getDestinationSEList([], "site", outputmode="Local") == [] + + +def test_directSEName_redefined(): + """Redifining an existing SEName has no impact""" + # CERN redefines LogSE in the associatedSEs + # but it should be ignored + assert getDestinationSEList("LogSE", "LCG.CERN.cern") == ["LogSE"] + assert getDestinationSEList("LogSE", "LCG.CERN.cern", outputmode="Local") == ["LogSE"] + + +def test_associatedSE_singleSE(): + """Map a given name to an SE in the sites definition""" + # CERN defines LocalLogSE as "CERN-LogSE" + assert getDestinationSEList("LocalLogSE", "LCG.CERN.cern") == ["CERN-LogSE"] + assert getDestinationSEList("LocalLogSE", "LCG.CERN.cern", outputmode="Local") == ["CERN-LogSE"] + # CNAF defines LocalLogSE as "CNAF-LogSE + assert getDestinationSEList("LocalLogSE", "LCG.CNAF.it") == ["CNAF-LogSE"] + # CNAF-LogSE is NOT defined as local, but it is still returned as + # ``Local`` only impacts StorageElementGroups resolution + assert getDestinationSEList("LocalLogSE", "LCG.CNAF.it", outputmode="Local") == ["CNAF-LogSE"] + + # LocalLogSE does not exist for these sites + + for site in ("LCG.IN2P3.fr", "LCG.GRIDKA.de", "LCG.NCBJ.pl"): + with pytest.raises(RuntimeError): + assert getDestinationSEList("LocalLogSE", site) + + # LocalLogSE is NOT a StorageElementGroup, so ``Local`` + # will not help, even if ``LocalLogSE`` is defined in the + # ``de`` country + # NOTE: That is quite counter intuitive and could be revisited + for site in ("LCG.IN2P3.fr", "LCG.GRIDKA.de", "LCG.NCBJ.pl"): + with pytest.raises(RuntimeError): + assert getDestinationSEList("LocalLogSE", site, outputmode="Local") + + +def test_associatedSE_group(): + """Map a given name to multiple SEs. The order should be different + because the local storages are returned first""" + + # CERN defines Tier1-Buffer as CERN-BUFFER, CNAF-BUFFER + assert getDestinationSEList("Tier1-Buffer", "LCG.CERN.cern") == ["CERN-BUFFER", "CNAF-BUFFER"] + # We could expect ``Local`` to reduce the output to CERN-BUFFER, but it does not + # because Tier1-Buffer is NOT a StorageElement + assert getDestinationSEList("Tier1-Buffer", "LCG.CERN.cern", outputmode="Local") == ["CERN-BUFFER", "CNAF-BUFFER"] + + # CNAF defines Tier1-Buffer as CERN-BUFFER, CNAF-BUFFER but the order is changed + # because local SEs go first + assert getDestinationSEList("Tier1-Buffer", "LCG.CNAF.it") == ["CNAF-BUFFER", "CERN-BUFFER"] + # Same as CERN, with local SEs first + assert getDestinationSEList("Tier1-Buffer", "LCG.CNAF.it") == ["CNAF-BUFFER", "CERN-BUFFER"] + + # Tier1-Buffer does not exist for these sites + for site in ("LCG.IN2P3.fr", "LCG.GRIDKA.de", "LCG.NCBJ.pl"): + with pytest.raises(RuntimeError): + assert getDestinationSEList("Tier1-Buffer", site) + + # Tier1-Buffer does not exist for these sites, and since it is not + # a StorageElementGroup, we do not do the country resolution + for site in ("LCG.IN2P3.fr", "LCG.GRIDKA.de", "LCG.NCBJ.pl"): + with pytest.raises(RuntimeError): + assert getDestinationSEList("Tier1-Buffer", site, outputmode="Local") + + +def test_seGroup(): + """Test resolving a StorageElementGroup not redefined anywhere""" + + # Tier1-Failover is not redifined anywhere + + # Retrieve the full list, sorting local SEs first + assert getDestinationSEList("Tier1-Failover", "LCG.CERN.cern") == ["CERN-FAILOVER", "CNAF-FAILOVER"] + assert getDestinationSEList("Tier1-Failover", "LCG.CNAF.it") == ["CNAF-FAILOVER", "CERN-FAILOVER"] + + # In ``Local`` mode, we ONLY get local SE + assert getDestinationSEList("Tier1-Failover", "LCG.CERN.cern", outputmode="Local") == ["CERN-FAILOVER"] + assert getDestinationSEList("Tier1-Failover", "LCG.CNAF.it", outputmode="Local") == ["CNAF-FAILOVER"] + + # Here we get the full Tier1-Failover list + # We have to compare with ``sorted`` because the order is shuffled + for site in ("LCG.IN2P3.fr", "LCG.GRIDKA.de", "LCG.NCBJ.pl"): + assert sorted(getDestinationSEList("Tier1-Failover", site)) == sorted(["CERN-FAILOVER", "CNAF-FAILOVER"]) + + # When using the ``Local`` mode, we get an error because none of + # the SE is local to any of the site. + for site in ("LCG.IN2P3.fr", "LCG.GRIDKA.de", "LCG.NCBJ.pl"): + with pytest.raises(RuntimeError): + getDestinationSEList("Tier1-Failover", site, outputmode="Local") + + +def test_seGroup_overwrite(): + """Test resolving a StorageElementGroup which is redefined in some places""" + + # CERN redefineds Tier1-DST + assert getDestinationSEList("Tier1-DST", "LCG.CERN.cern") == ["CERN-DST"] + # CERN-DST is local to CERN, so the same when using ``Local`` mode + assert getDestinationSEList("Tier1-DST", "LCG.CERN.cern", outputmode="Local") == ["CERN-DST"] + + # CNAF does NOT redefine Tier1-DST, but it puts local SE first + assert getDestinationSEList("Tier1-DST", "LCG.CNAF.it") == ["CNAF-DST", "CERN-DST"] + # CNAF-DST is local to CNAF, so get only that if using ``Local`` mode + assert getDestinationSEList("Tier1-DST", "LCG.CNAF.it", outputmode="Local") == ["CNAF-DST"] + + # Sites do not redefine Tier1-DST, so we get the full + # Tier1-DST list in a random order + for site in ("LCG.IN2P3.fr", "LCG.GRIDKA.de", "LCG.NCBJ.pl"): + assert sorted(getDestinationSEList("Tier1-DST", site)) == sorted(["CNAF-DST", "CERN-DST"]) + + # There are no SE in Tier1-DST which are ``Local`` to IN2P3, + # and IN2P3 is not associated to any country, + # so we don't find anything + with pytest.raises(RuntimeError): + getDestinationSEList("Tier1-DST", "LCG.IN2P3.fr", outputmode="Local") + + # Here we redefine the Tier1-DST at the country level: + # de has GRIDKA-DST as Tier1-DST + # pl points to de + for site in ("LCG.GRIDKA.de", "LCG.NCBJ.pl"): + assert getDestinationSEList("Tier1-DST", site, outputmode="Local") == ["GRIDKA-DST"] + + +def test_compat_resolveSEGroup(): + """We want to make sure that resolveSEGroup produces the same output + when we do not overwrite the SEGroup definition and use the default outputMode + """ + + for site in ("LCG.CERN.cern", "LCG.CNAF.it", "LCG.IN2P3.fr", "LCG.GRIDKA.de", "LCG.NCBJ.pl", "AnySite"): + assert sorted(getDestinationSEList("Tier1-Failover", site=site)) == sorted(resolveSEGroup("Tier1-Failover")) diff --git a/src/DIRAC/DataManagementSystem/Utilities/test/__init__.py b/src/DIRAC/DataManagementSystem/Utilities/test/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/DIRAC/Resources/Storage/test/Test_StorageFactory.py b/src/DIRAC/Resources/Storage/test/Test_StorageFactory.py index 6ad64999c03..549bc2ede42 100755 --- a/src/DIRAC/Resources/Storage/test/Test_StorageFactory.py +++ b/src/DIRAC/Resources/Storage/test/Test_StorageFactory.py @@ -10,15 +10,9 @@ from DIRAC import S_OK from DIRAC.Resources.Storage.StorageFactory import StorageFactory +from DIRAC.tests.Utilities.utils import generateDIRACConfig -@pytest.fixture(scope="module", autouse=True) -def generateConfig(): - """ - Generate the configuration that will be used for all the test - """ - - testCfgFileName = os.path.join(tempfile.gettempdir(), "Test_StorageFactory.cfg") - cfgContent = """ +CFG_CONTENT = """ DIRAC { VirtualOrganization = lhcb @@ -269,24 +263,14 @@ def generateConfig(): } } } - """ +""" - with open(testCfgFileName, "w") as f: - f.write(cfgContent) - # Load the configuration - ConfigurationClient(fileToLoadList=[testCfgFileName]) # we replace the configuration by our own one. - yield - try: - os.remove(testCfgFileName) - except OSError: - pass - # SUPER UGLY: one must recreate the CFG objects of gConfigurationData - # not to conflict with other tests that might be using a local dirac.cfg - gConfigurationData.localCFG = CFG() - gConfigurationData.remoteCFG = CFG() - gConfigurationData.mergedCFG = CFG() - gConfigurationData.generateNewVersion() +@pytest.fixture(scope="module", autouse=True) +def loadCS(): + """Load the CFG_CONTENT as a DIRAC Configuration for this module""" + with generateDIRACConfig(CFG_CONTENT, "test_StorageFactory.cfg"): + yield def mock_StorageFactory__generateStorageObject(*args, **kwargs): diff --git a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py index 09b9ab87108..c2fc3407957 100755 --- a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py +++ b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py @@ -42,7 +42,7 @@ from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.DataManagementSystem.Client.DataManager import DataManager from DIRAC.DataManagementSystem.Client.FailoverTransfer import FailoverTransfer -from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup +from DIRAC.DataManagementSystem.Utilities.ResolveSE import getDestinationSEList from DIRAC.Resources.Catalog.PoolXMLFile import getGUID from DIRAC.Resources.Catalog.FileCatalog import FileCatalog from DIRAC.RequestManagementSystem.Client.Request import Request @@ -117,10 +117,14 @@ def __init__(self, jobID=None, jobReport=None): self.pilotRef = gConfig.getValue("/LocalSite/PilotReference", "Unknown") self.cpuNormalizationFactor = gConfig.getValue("/LocalSite/CPUNormalizationFactor", 0.0) self.bufferLimit = gConfig.getValue(self.section + "/BufferLimit", 10485760) - self.defaultOutputSE = resolveSEGroup(gConfig.getValue("/Resources/StorageElementGroups/SE-USER", [])) + self.defaultOutputSE = getDestinationSEList( + gConfig.getValue("/Resources/StorageElementGroups/SE-USER", []), self.siteName + ) self.defaultCatalog = gConfig.getValue(self.section + "/DefaultCatalog", []) self.masterCatalogOnlyFlag = gConfig.getValue(self.section + "/MasterCatalogOnlyFlag", True) - self.defaultFailoverSE = resolveSEGroup(gConfig.getValue("/Resources/StorageElementGroups/Tier1-Failover", [])) + self.defaultFailoverSE = getDestinationSEList( + gConfig.getValue("/Resources/StorageElementGroups/Tier1-Failover", []), self.siteName + ) self.defaultOutputPath = "" self.retryUpload = gConfig.getValue(self.section + "/RetryUpload", False) self.dm = DataManager() diff --git a/src/DIRAC/tests/Utilities/utils.py b/src/DIRAC/tests/Utilities/utils.py index bcee62692dc..29157012e95 100644 --- a/src/DIRAC/tests/Utilities/utils.py +++ b/src/DIRAC/tests/Utilities/utils.py @@ -1,7 +1,9 @@ """ few utilities """ +import contextlib import os import shutil +import tempfile # pylint: disable=missing-docstring @@ -38,3 +40,61 @@ class MatchStringWith(str): def __eq__(self, other): return self in str(other) + + +@contextlib.contextmanager +def generateDIRACConfig(cfgContent, testCfgFileName): + """Utility to have a locally loaded DIRAC config for a test. + + To use it: + + .. code-block :: python + + from DIRAC.tests.Utilities.utils import generateDIRACConfig + + CFG_CONTENT = \"\"\" + Resources + { + StorageElements + { + + } + } + \"\"\" + + @pytest.fixture(scope="module", autouse=True) + def loadCS(): + \"\"\" Load the CFG_CONTENT as a DIRAC Configuration for this module \"\"\" + with generateDIRACConfig(CFG_CONTENT, "myConfig.cfg"): + yield + + + :param str cfgContent: the content of the CS you want + + + + + """ + + from diraccfg import CFG + from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData + from DIRAC.ConfigurationSystem.private.ConfigurationClient import ConfigurationClient + + testCfgFilePath = os.path.join(tempfile.gettempdir(), testCfgFileName) + with open(testCfgFilePath, "w") as f: + f.write(cfgContent) + # Load the configuration + ConfigurationClient(fileToLoadList=[testCfgFilePath]) # we replace the configuration by our own one. + + yield + + try: + os.remove(testCfgFilePath) + except OSError: + pass + # SUPER UGLY: one must recreate the CFG objects of gConfigurationData + # not to conflict with other tests that might be using a local dirac.cfg + gConfigurationData.localCFG = CFG() + gConfigurationData.remoteCFG = CFG() + gConfigurationData.mergedCFG = CFG() + gConfigurationData.generateNewVersion() From b7a975ac5dde7a6fa9840a4cc44d3bb6fadda163 Mon Sep 17 00:00:00 2001 From: chaen Date: Thu, 13 Oct 2022 09:23:15 +0200 Subject: [PATCH 0027/1135] sweep: #6412 Cast monitoring error messages --- src/DIRAC/Resources/Storage/StorageElement.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DIRAC/Resources/Storage/StorageElement.py b/src/DIRAC/Resources/Storage/StorageElement.py index 67fcd402c5c..3c5221e6504 100755 --- a/src/DIRAC/Resources/Storage/StorageElement.py +++ b/src/DIRAC/Resources/Storage/StorageElement.py @@ -1441,7 +1441,7 @@ def addAccountingOperation(self, urlDict, startDate, elapsedTime, storageParamet "ExecutionSite": siteName(), "TargetSE": self.name, "Protocol": accountingDict["Protocol"], - "Error": errorMsg, + "Error": str(errorMsg), "Component": "StorageElement", } failedRecords.append(failedRecord) From f9f71794346fbe9ca15cd523f1be19081d272b7d Mon Sep 17 00:00:00 2001 From: Simon Mazenoux Date: Tue, 14 Jun 2022 17:24:34 +0200 Subject: [PATCH 0028/1135] refactor: removing setStatus from jobState by creating a JobStatusUtility --- .../Client/JobState/CachedJobState.py | 13 -- .../Client/JobState/JobState.py | 91 +------- .../Client/JobStatus.py | 4 +- .../Executor/JobScheduling.py | 5 +- .../Service/JobStateUpdateHandler.py | 190 +--------------- .../Utilities/JobStatusUtility.py | 214 ++++++++++++++++++ .../test/Test_JobStatusUtility.py} | 28 +-- 7 files changed, 242 insertions(+), 303 deletions(-) create mode 100644 src/DIRAC/WorkloadManagementSystem/Utilities/JobStatusUtility.py rename src/DIRAC/WorkloadManagementSystem/{Service/tests/Test_JobStateUpdate.py => Utilities/test/Test_JobStatusUtility.py} (93%) diff --git a/src/DIRAC/WorkloadManagementSystem/Client/JobState/CachedJobState.py b/src/DIRAC/WorkloadManagementSystem/Client/JobState/CachedJobState.py index 3d0f4cc2be3..cc9cb183184 100644 --- a/src/DIRAC/WorkloadManagementSystem/Client/JobState/CachedJobState.py +++ b/src/DIRAC/WorkloadManagementSystem/Client/JobState/CachedJobState.py @@ -306,22 +306,9 @@ def setStatus(self, majorStatus, minorStatus=None, appStatus=None, source=None): self.__addLogRecord(majorStatus, minorStatus, appStatus, source) return S_OK() - def setMinorStatus(self, minorStatus, source=None): - self.__cacheAdd("att.MinorStatus", minorStatus) - self.__addLogRecord(minorStatus=minorStatus, source=source) - return S_OK() - def getStatus(self): return self.__cacheResult(("att.Status", "att.MinorStatus"), self.__jobState.getStatus) - def setAppStatus(self, appStatus, source=None): - self.__cacheAdd("att.ApplicationStatus", appStatus) - self.__addLogRecord(appStatus=appStatus, source=source) - return S_OK() - - def getAppStatus(self): - return self.__cacheResult("att.ApplicationStatus", self.__jobState.getAppStatus) - # # Attribs # diff --git a/src/DIRAC/WorkloadManagementSystem/Client/JobState/JobState.py b/src/DIRAC/WorkloadManagementSystem/Client/JobState/JobState.py index afdbaf25585..9c4c1eef157 100644 --- a/src/DIRAC/WorkloadManagementSystem/Client/JobState/JobState.py +++ b/src/DIRAC/WorkloadManagementSystem/Client/JobState/JobState.py @@ -10,6 +10,7 @@ from DIRAC.WorkloadManagementSystem.DB.TaskQueueDB import TaskQueueDB, singleValueDefFields, multiValueDefFields from DIRAC.WorkloadManagementSystem.Service.JobPolicy import RIGHT_GET_INFO, RIGHT_RESCHEDULE from DIRAC.WorkloadManagementSystem.Service.JobPolicy import RIGHT_RESET, RIGHT_CHANGE_STATUS +from DIRAC.WorkloadManagementSystem.Utilities.JobStatusUtility import JobStatusUtility class JobState: @@ -34,18 +35,14 @@ def checkDBAccess(cls): JobState.__db.logDB = JobLoggingDB() JobState.__db.tqDB = TaskQueueDB() - def __init__(self, jid, source="Unknown"): + def __init__(self, jid): self.__jid = jid - self.__source = str(source) self.checkDBAccess() @property def jid(self): return self.__jid - def setSource(self, source): - self.__source = source - def getManifest(self, rawData=False): result = JobState.__db.jobDB.getJobJDL(self.__jid) if not result["OK"] or rawData: @@ -164,50 +161,10 @@ def __checkType(self, value, tList, canBeNone=False): right_setStatus = RIGHT_GET_INFO - def setStatus(self, majorStatus, minorStatus=None, appStatus=None, source=None, updateTime=None): - try: - self.__checkType(majorStatus, str) - self.__checkType(minorStatus, str, canBeNone=True) - self.__checkType(appStatus, str, canBeNone=True) - self.__checkType(source, str, canBeNone=True) - self.__checkType(updateTime, datetime.datetime, canBeNone=True) - except TypeError as excp: - return S_ERROR(str(excp)) - result = JobState.__db.jobDB.setJobStatus( - self.__jid, status=majorStatus, minorStatus=minorStatus, applicationStatus=appStatus + def setStatus(self, majorStatus=None, minorStatus=None, appStatus=None, source=None): + return JobStatusUtility(self.__db.jobDB, self.__db.logDB).setJobStatus( + self.jid, status=majorStatus, minorStatus=minorStatus, appStatus=appStatus, source=source ) - if not result["OK"]: - return result - # HACK: Cause joblogging is crappy - if not minorStatus: - minorStatus = "idem" - if not appStatus: - appStatus = "idem" - if not source: - source = self.__source - return JobState.__db.logDB.addLoggingRecord( - self.__jid, - status=majorStatus, - minorStatus=minorStatus, - applicationStatus=appStatus, - date=updateTime, - source=source, - ) - - right_getMinorStatus = RIGHT_GET_INFO - - def setMinorStatus(self, minorStatus, source=None, updateTime=None): - try: - self.__checkType(minorStatus, str) - self.__checkType(source, str, canBeNone=True) - except TypeError as excp: - return S_ERROR(str(excp)) - result = JobState.__db.jobDB.setJobStatus(self.__jid, minorStatus=minorStatus) - if not result["OK"]: - return result - if not source: - source = self.__source - return JobState.__db.logDB.addLoggingRecord(self.__jid, minorStatus=minorStatus, date=updateTime, source=source) def getStatus(self): result = JobState.__db.jobDB.getJobAttributes(self.__jid, ["Status", "MinorStatus"]) @@ -218,23 +175,6 @@ def getStatus(self): return S_OK((data["Status"], data["MinorStatus"])) return S_ERROR("Job %d not found in the JobDB" % int(self.__jid)) - right_setAppStatus = RIGHT_GET_INFO - - def setAppStatus(self, appStatus, source=None, updateTime=None): - try: - self.__checkType(appStatus, str) - self.__checkType(source, str, canBeNone=True) - except TypeError as excp: - return S_ERROR(str(excp)) - result = JobState.__db.jobDB.setJobStatus(self.__jid, applicationStatus=appStatus) - if not result["OK"]: - return result - if not source: - source = self.__source - return JobState.__db.logDB.addLoggingRecord( - self.__jid, applicationStatus=appStatus, date=updateTime, source=source - ) - right_getAppStatus = RIGHT_GET_INFO def getAppStatus(self): @@ -380,27 +320,6 @@ def resetJob(self, source=""): def getInputData(self): return JobState.__db.jobDB.getInputData(self.__jid) - @classmethod - def checkInputDataStructure(cls, pDict): - if not isinstance(pDict, dict): - return S_ERROR("Input data has to be a dictionary") - for lfn in pDict: - if "Replicas" not in pDict[lfn]: - return S_ERROR("Missing replicas for lfn %s" % lfn) - replicas = pDict[lfn]["Replicas"] - for seName in replicas: - if "SURL" not in replicas or "Disk" not in replicas: - return S_ERROR(f"Missing SURL or Disk for {seName}:{lfn} replica") - return S_OK() - - right_setInputData = RIGHT_GET_INFO - - def set_InputData(self, lfnData): - result = self.checkInputDataStructure(lfnData) - if not result["OK"]: - return result - return JobState.__db.jobDB.setInputData(self.__jid, lfnData) - right_insertIntoTQ = RIGHT_CHANGE_STATUS def insertIntoTQ(self, manifest=None): diff --git a/src/DIRAC/WorkloadManagementSystem/Client/JobStatus.py b/src/DIRAC/WorkloadManagementSystem/Client/JobStatus.py index 44ac8383368..552ddbfbdeb 100644 --- a/src/DIRAC/WorkloadManagementSystem/Client/JobStatus.py +++ b/src/DIRAC/WorkloadManagementSystem/Client/JobStatus.py @@ -4,7 +4,7 @@ from DIRAC import gLogger, S_OK, S_ERROR from DIRAC.Core.Utilities.StateMachine import State, StateMachine - +from DIRAC.WorkloadManagementSystem.Client.JobMonitoringClient import JobMonitoringClient #: SUBMITTING = "Submitting" @@ -97,8 +97,6 @@ def checkJobStateTransition(jobID, candidateState, currentStatus=None, jobMonito """Utility to check if a job state transition is allowed""" if not currentStatus: if not jobMonitoringClient: - from DIRAC.WorkloadManagementSystem.Client.JobMonitoringClient import JobMonitoringClient - jobMonitoringClient = JobMonitoringClient() res = jobMonitoringClient.getJobsStatus(jobID) diff --git a/src/DIRAC/WorkloadManagementSystem/Executor/JobScheduling.py b/src/DIRAC/WorkloadManagementSystem/Executor/JobScheduling.py index 1296aa6989b..68b458874c3 100755 --- a/src/DIRAC/WorkloadManagementSystem/Executor/JobScheduling.py +++ b/src/DIRAC/WorkloadManagementSystem/Executor/JobScheduling.py @@ -23,6 +23,7 @@ from DIRAC.Resources.Storage.StorageElement import StorageElement from DIRAC.ResourceStatusSystem.Client.SiteStatus import SiteStatus from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient, getFilesToStage +from DIRAC.WorkloadManagementSystem.Client.JobState.JobState import JobState from DIRAC.WorkloadManagementSystem.Executor.Base.OptimizerExecutor import OptimizerExecutor from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB from DIRAC.WorkloadManagementSystem.Client import JobStatus @@ -300,13 +301,13 @@ def _applySiteFilter(self, sites, banned=False): filtered -= set(banned) return list(filtered) - def __holdJob(self, jobState, holdMsg, delay=0): + def __holdJob(self, jobState: JobState, holdMsg, delay=0): if delay: self.freezeTask(delay) else: self.freezeTask(self.ex_getOption("HoldTime", 300)) self.jobLog.info("On hold", holdMsg) - return jobState.setAppStatus(holdMsg, source=self.ex_optimizerName()) + return jobState.setStatus(appStatus=holdMsg, source=self.ex_optimizerName()) def __getSitesRequired(self, jobManifest): """Returns any candidate sites specified by the job or sites that have been diff --git a/src/DIRAC/WorkloadManagementSystem/Service/JobStateUpdateHandler.py b/src/DIRAC/WorkloadManagementSystem/Service/JobStateUpdateHandler.py index f293960d8e3..7195f4cbc60 100755 --- a/src/DIRAC/WorkloadManagementSystem/Service/JobStateUpdateHandler.py +++ b/src/DIRAC/WorkloadManagementSystem/Service/JobStateUpdateHandler.py @@ -7,15 +7,14 @@ """ import time -import datetime as dateTime from DIRAC import S_OK, S_ERROR from DIRAC.Core.DISET.RequestHandler import RequestHandler -from DIRAC.Core.Utilities import TimeUtilities from DIRAC.Core.Utilities.DEncode import ignoreEncodeWarning from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.WorkloadManagementSystem.Client import JobStatus +from DIRAC.WorkloadManagementSystem.Utilities.JobStatusUtility import JobStatusUtility class JobStateUpdateHandlerMixin: @@ -83,7 +82,9 @@ def export_updateJobFromStager(cls, jobID, status): if status != JobStatus.STAGING: return S_OK("Job is not in Staging after %d seconds" % trials) - result = cls.__setJobStatus(int(jobID), status=jobStatus, minorStatus=minorStatus, source="StagerSystem") + result = JobStatusUtility(cls.jobDB, cls.jobLoggingDB).setJobStatus( + int(jobID), status=jobStatus, minorStatus=minorStatus, source="StagerSystem" + ) if not result["OK"]: if result["Message"].find("does not exist") != -1: return S_OK() @@ -101,194 +102,17 @@ def export_setJobStatus(cls, jobID, status="", minorStatus="", source="Unknown", Sets optionally the status date and source component which sends the status information. The "force" flag will override the WMS state machine decision. """ - return cls.__setJobStatus( + return JobStatusUtility(cls.jobDB, cls.jobLoggingDB).setJobStatus( int(jobID), status=status, minorStatus=minorStatus, source=source, datetime=datetime, force=force ) - @classmethod - def __setJobStatus( - cls, jobID, status=None, minorStatus=None, appStatus=None, source=None, datetime=None, force=False - ): - """update the job provided statuses (major, minor and application) - If sets also the source and the time stamp (or current time) - This method calls the bulk method internally - """ - sDict = {} - if status: - sDict["Status"] = status - if minorStatus: - sDict["MinorStatus"] = minorStatus - if appStatus: - sDict["ApplicationStatus"] = appStatus - if sDict: - if source: - sDict["Source"] = source - if not datetime: - datetime = str(dateTime.datetime.utcnow()) - return cls._setJobStatusBulk(jobID, {datetime: sDict}, force=force) - return S_OK() - ########################################################################### types_setJobStatusBulk = [[str, int], dict] @classmethod def export_setJobStatusBulk(cls, jobID, statusDict, force=False): """Set various job status fields with a time stamp and a source""" - return cls._setJobStatusBulk(jobID, statusDict, force=force) - - @classmethod - def _setJobStatusBulk(cls, jobID, statusDict, force=False): - """Set various status fields for job specified by its jobId. - Set only the last status in the JobDB, updating all the status - logging information in the JobLoggingDB. The statusDict has datetime - as a key and status information dictionary as values - """ - jobID = int(jobID) - log = cls.log.getLocalSubLogger("JobStatusBulk/Job-%d" % jobID) - - result = cls.jobDB.getJobAttributes(jobID, ["Status", "StartExecTime", "EndExecTime"]) - if not result["OK"]: - return result - if not result["Value"]: - # if there is no matching Job it returns an empty dictionary - return S_ERROR("No Matching Job") - - # If the current status is Stalled and we get an update, it should probably be "Running" - currentStatus = result["Value"]["Status"] - if currentStatus == JobStatus.STALLED: - currentStatus = JobStatus.RUNNING - startTime = result["Value"].get("StartExecTime") - endTime = result["Value"].get("EndExecTime") - # getJobAttributes only returns strings :( - if startTime == "None": - startTime = None - if endTime == "None": - endTime = None - - # Remove useless items in order to make it simpler later, although there should not be any - for sDict in statusDict.values(): - for item in sorted(sDict): - if not sDict[item]: - sDict.pop(item, None) - - # Get the latest time stamps of major status updates - result = cls.jobLoggingDB.getWMSTimeStamps(int(jobID)) - if not result["OK"]: - return result - if not result["Value"]: - return S_ERROR("No registered WMS timeStamps") - # This is more precise than "LastTime". timeStamps is a sorted list of tuples... - timeStamps = sorted((float(t), s) for s, t in result["Value"].items() if s != "LastTime") - lastTime = TimeUtilities.toString(TimeUtilities.fromEpoch(timeStamps[-1][0])) - - # Get chronological order of new updates - updateTimes = sorted(statusDict) - log.debug("*** New call ***", f"Last update time {lastTime} - Sorted new times {updateTimes}") - # Get the status (if any) at the time of the first update - newStat = "" - firstUpdate = TimeUtilities.toEpoch(TimeUtilities.fromString(updateTimes[0])) - for ts, st in timeStamps: - if firstUpdate >= ts: - newStat = st - # Pick up start and end times from all updates - for updTime in updateTimes: - sDict = statusDict[updTime] - newStat = sDict.get("Status", newStat) - - if not startTime and newStat == JobStatus.RUNNING: - # Pick up the start date when the job starts running if not existing - startTime = updTime - log.debug("Set job start time", startTime) - elif not endTime and newStat in JobStatus.JOB_FINAL_STATES: - # Pick up the end time when the job is in a final status - endTime = updTime - log.debug("Set job end time", endTime) - - # We should only update the status to the last one if its time stamp is more recent than the last update - attrNames = [] - attrValues = [] - if updateTimes[-1] >= lastTime: - minor = "" - application = "" - # Get the last status values looping on the most recent upupdateTimes in chronological order - for updTime in [dt for dt in updateTimes if dt >= lastTime]: - sDict = statusDict[updTime] - log.debug("\t", f"Time {updTime} - Statuses {str(sDict)}") - status = sDict.get("Status", currentStatus) - # evaluate the state machine if the status is changing - if not force and status != currentStatus: - res = JobStatus.JobsStateMachine(currentStatus).getNextState(status) - if not res["OK"]: - return res - newStat = res["Value"] - # If the JobsStateMachine does not accept the candidate, don't update - if newStat != status: - # keeping the same status - log.error( - "Job Status Error", - f"{jobID} can't move from {currentStatus} to {status}: using {newStat}", - ) - status = newStat - sDict["Status"] = newStat - # Change the source to indicate this is not what was requested - source = sDict.get("Source", "") - sDict["Source"] = source + "(SM)" - # at this stage status == newStat. Set currentStatus to this new status - currentStatus = newStat - - minor = sDict.get("MinorStatus", minor) - application = sDict.get("ApplicationStatus", application) - - log.debug("Final statuses:", f"status '{status}', minor '{minor}', application '{application}'") - if status: - attrNames.append("Status") - attrValues.append(status) - if minor: - attrNames.append("MinorStatus") - attrValues.append(minor) - if application: - attrNames.append("ApplicationStatus") - attrValues.append(application) - # Here we are forcing the update as it's always updating to the last status - result = cls.jobDB.setJobAttributes(jobID, attrNames, attrValues, update=True, force=True) - if not result["OK"]: - return result - if cls.elasticJobParametersDB: - result = cls.elasticJobParametersDB.setJobParameter(int(jobID), "Status", status) - if not result["OK"]: - return result - # Update start and end time if needed - if endTime: - result = cls.jobDB.setEndExecTime(jobID, endTime) - if not result["OK"]: - return result - if startTime: - result = cls.jobDB.setStartExecTime(jobID, startTime) - if not result["OK"]: - return result - - # Update the JobLoggingDB records - heartBeatTime = None - for updTime in updateTimes: - sDict = statusDict[updTime] - status = sDict.get("Status", "idem") - minor = sDict.get("MinorStatus", "idem") - application = sDict.get("ApplicationStatus", "idem") - source = sDict.get("Source", "Unknown") - result = cls.jobLoggingDB.addLoggingRecord( - jobID, status=status, minorStatus=minor, applicationStatus=application, date=updTime, source=source - ) - if not result["OK"]: - return result - # If the update comes from a job, update the heart beat time stamp with this item's stamp - if source.startswith("Job"): - heartBeatTime = updTime - if heartBeatTime is not None: - result = cls.jobDB.setHeartBeatData(jobID, {"HeartBeatTime": heartBeatTime}) - if not result["OK"]: - return result - - return S_OK((attrNames, attrValues)) + return JobStatusUtility(cls.jobDB, cls.jobLoggingDB).setJobStatusBulk(int(jobID), statusDict, force=force) ########################################################################### types_setJobAttribute = [[str, int], str, str] @@ -330,7 +154,7 @@ def export_setJobApplicationStatus(cls, jobID, appStatus, source="Unknown"): """Set the application status for job specified by its JobId. Internally calling the bulk method """ - return cls.__setJobStatus(jobID, appStatus=appStatus, source=source) + return JobStatusUtility(cls.jobDB, cls.jobLoggingDB).setJobStatus(jobID, appStatus=appStatus, source=source) ########################################################################### types_setJobParameter = [[str, int], str, str] diff --git a/src/DIRAC/WorkloadManagementSystem/Utilities/JobStatusUtility.py b/src/DIRAC/WorkloadManagementSystem/Utilities/JobStatusUtility.py new file mode 100644 index 00000000000..43ea460f1b1 --- /dev/null +++ b/src/DIRAC/WorkloadManagementSystem/Utilities/JobStatusUtility.py @@ -0,0 +1,214 @@ +"""Utility to set the job status in the jobDB""" + +import datetime as dateTime + +from DIRAC import gLogger, S_OK, S_ERROR +from DIRAC.Core.Utilities import TimeUtilities +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader +from DIRAC.WorkloadManagementSystem.Client import JobStatus +from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB +from DIRAC.WorkloadManagementSystem.DB.JobLoggingDB import JobLoggingDB + + +class JobStatusUtility: + def __init__(self, jobDB: JobDB = None, jobLoggingDB: JobLoggingDB = None) -> None: + """ + :raises: RuntimeError, AttributeError + """ + + self.log = gLogger.getSubLogger(self.__class__.__name__) + + if not jobDB: + try: + result = ObjectLoader().loadObject("WorkloadManagementSystem.DB.JobDB", "JobDB") + if not result["OK"]: + raise AttributeError(result["Message"]) + jobDB = result["Value"](parentLogger=self.log) + except RuntimeError: + self.log.error("Can't connect to the jobDB") + raise + self.jobDB = jobDB + + if not jobLoggingDB: + try: + result = ObjectLoader().loadObject("WorkloadManagementSystem.DB.JobLoggingDB", "JobLoggingDB") + if not result["OK"]: + raise AttributeError(result["Message"]) + jobLoggingDB = result["Value"](parentLogger=self.log) + except RuntimeError: + self.log.error("Can't connect to the JobLoggingDB") + raise + self.jobLoggingDB = jobLoggingDB + + def setJobStatus( + self, jobID: int, status=None, minorStatus=None, appStatus=None, source=None, datetime=None, force=False + ): + """ + Update the job provided statuses (major, minor and application) + If sets also the source and the time stamp (or current time) + This method calls the bulk method internally + """ + sDict = {} + if status: + sDict["Status"] = status + if minorStatus: + sDict["MinorStatus"] = minorStatus + if appStatus: + sDict["ApplicationStatus"] = appStatus + if sDict: + if source: + sDict["Source"] = source + if not datetime: + datetime = str(dateTime.datetime.utcnow()) + return self.setJobStatusBulk(jobID, {datetime: sDict}, force=force) + return S_OK() + + def setJobStatusBulk(self, jobID: int, statusDict: dict, force: bool = False): + """Set various status fields for job specified by its jobId. + Set only the last status in the JobDB, updating all the status + logging information in the JobLoggingDB. The statusDict has datetime + as a key and status information dictionary as values + """ + jobID = int(jobID) + log = self.log.getLocalSubLogger("JobStatusBulk/Job-%d" % jobID) + + result = self.jobDB.getJobAttributes(jobID, ["Status", "StartExecTime", "EndExecTime"]) + if not result["OK"]: + return result + if not result["Value"]: + # if there is no matching Job it returns an empty dictionary + return S_ERROR("No Matching Job") + + # If the current status is Stalled and we get an update, it should probably be "Running" + currentStatus = result["Value"]["Status"] + if currentStatus == JobStatus.STALLED: + currentStatus = JobStatus.RUNNING + startTime = result["Value"].get("StartExecTime") + endTime = result["Value"].get("EndExecTime") + # getJobAttributes only returns strings :( + if startTime == "None": + startTime = None + if endTime == "None": + endTime = None + + # Remove useless items in order to make it simpler later, although there should not be any + for sDict in statusDict.values(): + for item in sorted(sDict): + if not sDict[item]: + sDict.pop(item, None) + + # Get the latest time stamps of major status updates + result = self.jobLoggingDB.getWMSTimeStamps(int(jobID)) + if not result["OK"]: + return result + if not result["Value"]: + return S_ERROR("No registered WMS timeStamps") + # This is more precise than "LastTime". timeStamps is a sorted list of tuples... + timeStamps = sorted((float(t), s) for s, t in result["Value"].items() if s != "LastTime") + lastTime = TimeUtilities.toString(TimeUtilities.fromEpoch(timeStamps[-1][0])) + + # Get chronological order of new updates + updateTimes = sorted(statusDict) + log.debug("*** New call ***", "Last update time %s - Sorted new times %s" % (lastTime, updateTimes)) + # Get the status (if any) at the time of the first update + newStat = "" + firstUpdate = TimeUtilities.toEpoch(TimeUtilities.fromString(updateTimes[0])) + for ts, st in timeStamps: + if firstUpdate >= ts: + newStat = st + # Pick up start and end times from all updates + for updTime in updateTimes: + sDict = statusDict[updTime] + newStat = sDict.get("Status", newStat) + + if not startTime and newStat == JobStatus.RUNNING: + # Pick up the start date when the job starts running if not existing + startTime = updTime + log.debug("Set job start time", startTime) + elif not endTime and newStat in JobStatus.JOB_FINAL_STATES: + # Pick up the end time when the job is in a final status + endTime = updTime + log.debug("Set job end time", endTime) + + # We should only update the status to the last one if its time stamp is more recent than the last update + attrNames = [] + attrValues = [] + if updateTimes[-1] >= lastTime: + minor = "" + application = "" + # Get the last status values looping on the most recent upupdateTimes in chronological order + for updTime in [dt for dt in updateTimes if dt >= lastTime]: + sDict = statusDict[updTime] + log.debug("\t", "Time %s - Statuses %s" % (updTime, str(sDict))) + status = sDict.get("Status", currentStatus) + # evaluate the state machine if the status is changing + if not force and status != currentStatus: + res = JobStatus.JobsStateMachine(currentStatus).getNextState(status) + if not res["OK"]: + return res + newStat = res["Value"] + # If the JobsStateMachine does not accept the candidate, don't update + if newStat != status: + # keeping the same status + log.error( + "Job Status Error", + "%s can't move from %s to %s: using %s" % (jobID, currentStatus, status, newStat), + ) + status = newStat + sDict["Status"] = newStat + # Change the source to indicate this is not what was requested + source = sDict.get("Source", "") + sDict["Source"] = source + "(SM)" + # at this stage status == newStat. Set currentStatus to this new status + currentStatus = newStat + + minor = sDict.get("MinorStatus", minor) + application = sDict.get("ApplicationStatus", application) + + log.debug("Final statuses:", "status '%s', minor '%s', application '%s'" % (status, minor, application)) + if status: + attrNames.append("Status") + attrValues.append(status) + if minor: + attrNames.append("MinorStatus") + attrValues.append(minor) + if application: + attrNames.append("ApplicationStatus") + attrValues.append(application) + # Here we are forcing the update as it's always updating to the last status + result = self.jobDB.setJobAttributes(jobID, attrNames, attrValues, update=True, force=True) + if not result["OK"]: + return result + + # Update start and end time if needed + if endTime: + result = self.jobDB.setEndExecTime(jobID, endTime) + if not result["OK"]: + return result + if startTime: + result = self.jobDB.setStartExecTime(jobID, startTime) + if not result["OK"]: + return result + + # Update the JobLoggingDB records + heartBeatTime = None + for updTime in updateTimes: + sDict = statusDict[updTime] + status = sDict.get("Status", "idem") + minor = sDict.get("MinorStatus", "idem") + application = sDict.get("ApplicationStatus", "idem") + source = sDict.get("Source", "Unknown") + result = self.jobLoggingDB.addLoggingRecord( + jobID, status=status, minorStatus=minor, applicationStatus=application, date=updTime, source=source + ) + if not result["OK"]: + return result + # If the update comes from a job, update the heart beat time stamp with this item's stamp + if source.startswith("Job"): + heartBeatTime = updTime + if heartBeatTime is not None: + result = self.jobDB.setHeartBeatData(jobID, {"HeartBeatTime": heartBeatTime}) + if not result["OK"]: + return result + + return S_OK((attrNames, attrValues)) diff --git a/src/DIRAC/WorkloadManagementSystem/Service/tests/Test_JobStateUpdate.py b/src/DIRAC/WorkloadManagementSystem/Utilities/test/Test_JobStatusUtility.py similarity index 93% rename from src/DIRAC/WorkloadManagementSystem/Service/tests/Test_JobStateUpdate.py rename to src/DIRAC/WorkloadManagementSystem/Utilities/test/Test_JobStatusUtility.py index 6c9d4de67bd..913ab1c038a 100644 --- a/src/DIRAC/WorkloadManagementSystem/Service/tests/Test_JobStateUpdate.py +++ b/src/DIRAC/WorkloadManagementSystem/Utilities/test/Test_JobStatusUtility.py @@ -1,22 +1,17 @@ -""" unit test (pytest) of JobStateUpdate service -""" +# pylint: disable=missing-docstring, invalid-name from unittest.mock import MagicMock import pytest from DIRAC import gLogger -gLogger.setLevel("DEBUG") - from DIRAC.WorkloadManagementSystem.Client import JobStatus from DIRAC.WorkloadManagementSystem.Client import JobMinorStatus # sut -from DIRAC.WorkloadManagementSystem.Service.JobStateUpdateHandler import JobStateUpdateHandlerMixin +from DIRAC.WorkloadManagementSystem.Utilities.JobStatusUtility import JobStatusUtility -# mocks -jobDB_mock = MagicMock() -jobLoggingDB_mock = MagicMock() +gLogger.setLevel("DEBUG") @pytest.mark.parametrize( @@ -205,7 +200,6 @@ ], ) def test__setJobStatusBulk( - mocker, statusDict_in, jobDB_getJobAttributes_rv, jobDB_setJobAttributes_rv, @@ -214,18 +208,20 @@ def test__setJobStatusBulk( resExpected, resExpected_value, ): - JobStateUpdateHandlerMixin.jobDB = jobDB_mock - JobStateUpdateHandlerMixin.jobLoggingDB = jobLoggingDB_mock - JobStateUpdateHandlerMixin.log = gLogger - JobStateUpdateHandlerMixin.elasticJobParametersDB = None - + # Arrange + jobDB_mock = MagicMock() jobDB_mock.getJobAttributes.return_value = jobDB_getJobAttributes_rv jobDB_mock.setJobAttributes.return_value = jobDB_setJobAttributes_rv + + jobLoggingDB_mock = MagicMock() jobLoggingDB_mock.getWMSTimeStamps.return_value = jobLoggingDB_getWMSTimeStamps_rv - jsu = JobStateUpdateHandlerMixin() + jsu = JobStatusUtility(jobDB_mock, jobLoggingDB_mock) + + # Act + res = jsu.setJobStatusBulk(1, statusDict_in, force) - res = jsu._setJobStatusBulk(1, statusDict_in, force) + # Assert assert res["OK"] is resExpected if res["OK"]: assert res["Value"] == resExpected_value From 95b26809eded1ff5072359196c32d4795cbe969a Mon Sep 17 00:00:00 2001 From: fstagni Date: Fri, 30 Sep 2022 11:38:51 +0200 Subject: [PATCH 0029/1135] fix: re-added elasticJobParametersDB --- .../Client/JobState/JobState.py | 2 - .../Client/JobStatus.py | 4 +- .../Service/JobStateUpdateHandler.py | 26 ++++----- .../Utilities/JobStatusUtility.py | 53 ++++++++++++++----- .../Utilities/test/Test_JobStatusUtility.py | 6 ++- 5 files changed, 60 insertions(+), 31 deletions(-) diff --git a/src/DIRAC/WorkloadManagementSystem/Client/JobState/JobState.py b/src/DIRAC/WorkloadManagementSystem/Client/JobState/JobState.py index 9c4c1eef157..f0077270307 100644 --- a/src/DIRAC/WorkloadManagementSystem/Client/JobState/JobState.py +++ b/src/DIRAC/WorkloadManagementSystem/Client/JobState/JobState.py @@ -1,7 +1,5 @@ """ This object is a wrapper for setting and getting jobs states """ -import datetime - from DIRAC import gLogger, S_OK, S_ERROR from DIRAC.WorkloadManagementSystem.Client.JobState.JobManifest import JobManifest from DIRAC.WorkloadManagementSystem.Client import JobStatus diff --git a/src/DIRAC/WorkloadManagementSystem/Client/JobStatus.py b/src/DIRAC/WorkloadManagementSystem/Client/JobStatus.py index 552ddbfbdeb..44ac8383368 100644 --- a/src/DIRAC/WorkloadManagementSystem/Client/JobStatus.py +++ b/src/DIRAC/WorkloadManagementSystem/Client/JobStatus.py @@ -4,7 +4,7 @@ from DIRAC import gLogger, S_OK, S_ERROR from DIRAC.Core.Utilities.StateMachine import State, StateMachine -from DIRAC.WorkloadManagementSystem.Client.JobMonitoringClient import JobMonitoringClient + #: SUBMITTING = "Submitting" @@ -97,6 +97,8 @@ def checkJobStateTransition(jobID, candidateState, currentStatus=None, jobMonito """Utility to check if a job state transition is allowed""" if not currentStatus: if not jobMonitoringClient: + from DIRAC.WorkloadManagementSystem.Client.JobMonitoringClient import JobMonitoringClient + jobMonitoringClient = JobMonitoringClient() res = jobMonitoringClient.getJobsStatus(jobID) diff --git a/src/DIRAC/WorkloadManagementSystem/Service/JobStateUpdateHandler.py b/src/DIRAC/WorkloadManagementSystem/Service/JobStateUpdateHandler.py index 7195f4cbc60..95bc66d222e 100755 --- a/src/DIRAC/WorkloadManagementSystem/Service/JobStateUpdateHandler.py +++ b/src/DIRAC/WorkloadManagementSystem/Service/JobStateUpdateHandler.py @@ -35,11 +35,10 @@ def initializeHandler(cls, svcInfoDict): cls.jobLoggingDB = result["Value"](parentLogger=cls.log) except RuntimeError as excp: - return S_ERROR("Can't connect to DB: %s" % excp) + return S_ERROR(f"Can't connect to DB: {excp}") cls.elasticJobParametersDB = None - useESForJobParametersFlag = Operations().getValue("/Services/JobMonitoring/useESForJobParametersFlag", False) - if useESForJobParametersFlag: + if Operations().getValue("/Services/JobMonitoring/useESForJobParametersFlag", False): try: result = ObjectLoader().loadObject( "WorkloadManagementSystem.DB.ElasticJobParametersDB", "ElasticJobParametersDB" @@ -48,7 +47,10 @@ def initializeHandler(cls, svcInfoDict): return result cls.elasticJobParametersDB = result["Value"]() except RuntimeError as excp: - return S_ERROR("Can't connect to DB: %s" % excp) + return S_ERROR(f"Can't connect to DB: {excp}") + + cls.jsu = JobStatusUtility(cls.jobDB, cls.jobLoggingDB, cls.elasticJobParametersDB) + return S_OK() ########################################################################### @@ -76,15 +78,13 @@ def export_updateJobFromStager(cls, jobID, status): status = result["Value"]["Status"] if status == JobStatus.STAGING: if i: - infoStr = "Found job in Staging after %d seconds" % i + infoStr = f"Found job in Staging after {i} seconds" break time.sleep(1) if status != JobStatus.STAGING: - return S_OK("Job is not in Staging after %d seconds" % trials) + return S_OK(f"Job is not in Staging after {trials} seconds") - result = JobStatusUtility(cls.jobDB, cls.jobLoggingDB).setJobStatus( - int(jobID), status=jobStatus, minorStatus=minorStatus, source="StagerSystem" - ) + result = cls.jsu.setJobStatus(int(jobID), status=jobStatus, minorStatus=minorStatus, source="StagerSystem") if not result["OK"]: if result["Message"].find("does not exist") != -1: return S_OK() @@ -102,8 +102,8 @@ def export_setJobStatus(cls, jobID, status="", minorStatus="", source="Unknown", Sets optionally the status date and source component which sends the status information. The "force" flag will override the WMS state machine decision. """ - return JobStatusUtility(cls.jobDB, cls.jobLoggingDB).setJobStatus( - int(jobID), status=status, minorStatus=minorStatus, source=source, datetime=datetime, force=force + return cls.jsu.setJobStatus( + int(jobID), status=status, minorStatus=minorStatus, source=source, dateTime=datetime, force=force ) ########################################################################### @@ -112,7 +112,7 @@ def export_setJobStatus(cls, jobID, status="", minorStatus="", source="Unknown", @classmethod def export_setJobStatusBulk(cls, jobID, statusDict, force=False): """Set various job status fields with a time stamp and a source""" - return JobStatusUtility(cls.jobDB, cls.jobLoggingDB).setJobStatusBulk(int(jobID), statusDict, force=force) + return cls.jsu.setJobStatusBulk(int(jobID), statusDict, force=force) ########################################################################### types_setJobAttribute = [[str, int], str, str] @@ -154,7 +154,7 @@ def export_setJobApplicationStatus(cls, jobID, appStatus, source="Unknown"): """Set the application status for job specified by its JobId. Internally calling the bulk method """ - return JobStatusUtility(cls.jobDB, cls.jobLoggingDB).setJobStatus(jobID, appStatus=appStatus, source=source) + return cls.jsu.setJobStatus(jobID, appStatus=appStatus, source=source) ########################################################################### types_setJobParameter = [[str, int], str, str] diff --git a/src/DIRAC/WorkloadManagementSystem/Utilities/JobStatusUtility.py b/src/DIRAC/WorkloadManagementSystem/Utilities/JobStatusUtility.py index 43ea460f1b1..15ba24f8e68 100644 --- a/src/DIRAC/WorkloadManagementSystem/Utilities/JobStatusUtility.py +++ b/src/DIRAC/WorkloadManagementSystem/Utilities/JobStatusUtility.py @@ -1,47 +1,69 @@ """Utility to set the job status in the jobDB""" -import datetime as dateTime +import datetime from DIRAC import gLogger, S_OK, S_ERROR +from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.Core.Utilities import TimeUtilities from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader from DIRAC.WorkloadManagementSystem.Client import JobStatus from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB from DIRAC.WorkloadManagementSystem.DB.JobLoggingDB import JobLoggingDB +from DIRAC.WorkloadManagementSystem.DB.ElasticJobParametersDB import ElasticJobParametersDB class JobStatusUtility: - def __init__(self, jobDB: JobDB = None, jobLoggingDB: JobLoggingDB = None) -> None: + def __init__( + self, + jobDB: JobDB = None, + jobLoggingDB: JobLoggingDB = None, + elasticJobParametersDB: ElasticJobParametersDB = None, + ) -> None: """ :raises: RuntimeError, AttributeError """ self.log = gLogger.getSubLogger(self.__class__.__name__) - if not jobDB: + self.jobDB = jobDB + self.jobLoggingDB = jobLoggingDB + self.elasticJobParametersDB = elasticJobParametersDB + + if not self.jobDB: try: result = ObjectLoader().loadObject("WorkloadManagementSystem.DB.JobDB", "JobDB") if not result["OK"]: raise AttributeError(result["Message"]) - jobDB = result["Value"](parentLogger=self.log) + self.jobDB = result["Value"](parentLogger=self.log) except RuntimeError: self.log.error("Can't connect to the jobDB") raise - self.jobDB = jobDB - if not jobLoggingDB: + if not self.jobLoggingDB: try: result = ObjectLoader().loadObject("WorkloadManagementSystem.DB.JobLoggingDB", "JobLoggingDB") if not result["OK"]: raise AttributeError(result["Message"]) - jobLoggingDB = result["Value"](parentLogger=self.log) + self.jobLoggingDB = result["Value"](parentLogger=self.log) except RuntimeError: self.log.error("Can't connect to the JobLoggingDB") raise - self.jobLoggingDB = jobLoggingDB + + if not self.elasticJobParametersDB: + if Operations().getValue("/Services/JobMonitoring/useESForJobParametersFlag", False): + try: + result = ObjectLoader().loadObject( + "WorkloadManagementSystem.DB.ElasticJobParametersDB", "ElasticJobParametersDB" + ) + if not result["OK"]: + raise AttributeError(result["Message"]) + self.elasticJobParametersDB = result["Value"](parentLogger=self.log) + except RuntimeError: + self.log.error("Can't connect to the JobLoggingDB") + raise def setJobStatus( - self, jobID: int, status=None, minorStatus=None, appStatus=None, source=None, datetime=None, force=False + self, jobID: int, status=None, minorStatus=None, appStatus=None, source=None, dateTime=None, force=False ): """ Update the job provided statuses (major, minor and application) @@ -58,15 +80,15 @@ def setJobStatus( if sDict: if source: sDict["Source"] = source - if not datetime: - datetime = str(dateTime.datetime.utcnow()) - return self.setJobStatusBulk(jobID, {datetime: sDict}, force=force) + if not dateTime: + dateTime = str(datetime.datetime.utcnow()) + return self.setJobStatusBulk(jobID, {dateTime: sDict}, force=force) return S_OK() def setJobStatusBulk(self, jobID: int, statusDict: dict, force: bool = False): """Set various status fields for job specified by its jobId. Set only the last status in the JobDB, updating all the status - logging information in the JobLoggingDB. The statusDict has datetime + logging information in the JobLoggingDB. The statusDict has dateTime as a key and status information dictionary as values """ jobID = int(jobID) @@ -179,7 +201,10 @@ def setJobStatusBulk(self, jobID: int, statusDict: dict, force: bool = False): result = self.jobDB.setJobAttributes(jobID, attrNames, attrValues, update=True, force=True) if not result["OK"]: return result - + if self.elasticJobParametersDB: + result = self.elasticJobParametersDB.setJobParameter(int(jobID), "Status", status) + if not result["OK"]: + return result # Update start and end time if needed if endTime: result = self.jobDB.setEndExecTime(jobID, endTime) diff --git a/src/DIRAC/WorkloadManagementSystem/Utilities/test/Test_JobStatusUtility.py b/src/DIRAC/WorkloadManagementSystem/Utilities/test/Test_JobStatusUtility.py index 913ab1c038a..1205beed32c 100644 --- a/src/DIRAC/WorkloadManagementSystem/Utilities/test/Test_JobStatusUtility.py +++ b/src/DIRAC/WorkloadManagementSystem/Utilities/test/Test_JobStatusUtility.py @@ -216,7 +216,11 @@ def test__setJobStatusBulk( jobLoggingDB_mock = MagicMock() jobLoggingDB_mock.getWMSTimeStamps.return_value = jobLoggingDB_getWMSTimeStamps_rv - jsu = JobStatusUtility(jobDB_mock, jobLoggingDB_mock) + esJobParameters_mock = MagicMock() + esJobParameters_mock.getJobAttributes.return_value = jobDB_getJobAttributes_rv + esJobParameters_mock.setJobAttributes.return_value = jobDB_setJobAttributes_rv + + jsu = JobStatusUtility(jobDB_mock, jobLoggingDB_mock, esJobParameters_mock) # Act res = jsu.setJobStatusBulk(1, statusDict_in, force) From f8f18b743bb802f3e76d8cec05722f3f6e2d66d2 Mon Sep 17 00:00:00 2001 From: fstagni Date: Thu, 29 Sep 2022 12:14:22 +0200 Subject: [PATCH 0030/1135] fix: minor py3-related fixes --- integration_tests.py | 3 +- src/DIRAC/Core/DISET/private/BaseClient.py | 3 - src/DIRAC/Core/Utilities/Graphs/Graph.py | 33 +++---- src/DIRAC/Core/Utilities/test/Test_File.py | 2 - .../Client/FileCatalogClientCLI.py | 11 +-- .../Service/SystemAdministratorHandler.py | 13 ++- src/DIRAC/Interfaces/API/Dirac.py | 17 +--- src/DIRAC/Interfaces/API/test/Test_DIRAC.py | 4 - .../Client/Operation.py | 3 - .../RequestManagementSystem/Client/Request.py | 3 - .../Computing/ARC6ComputingElement.py | 3 - .../Computing/HTCondorCEComputingElement.py | 8 +- .../Agent/JobAgent.py | 5 -- .../JobWrapper/JobWrapper.py | 30 ------- .../Utilities/Utils.py | 17 ++-- tests/CI/install_client.sh | 6 -- tests/Jenkins/dirac_ci.sh | 2 - tests/Jenkins/utilities.sh | 85 +------------------ 18 files changed, 32 insertions(+), 216 deletions(-) diff --git a/integration_tests.py b/integration_tests.py index 7be977f9813..1c1c04d411d 100755 --- a/integration_tests.py +++ b/integration_tests.py @@ -1,7 +1,6 @@ #!/usr/bin/env python import fnmatch import os -from pathlib import Path import re import shlex import shutil @@ -11,9 +10,9 @@ import time from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager +from pathlib import Path from typing import Optional -import click import git import typer import yaml diff --git a/src/DIRAC/Core/DISET/private/BaseClient.py b/src/DIRAC/Core/DISET/private/BaseClient.py index fbb693906ef..147fbb56c2d 100755 --- a/src/DIRAC/Core/DISET/private/BaseClient.py +++ b/src/DIRAC/Core/DISET/private/BaseClient.py @@ -658,8 +658,5 @@ def _getBaseStub(self): def __bool__(self): return True - # For Python 2 compatibility - __nonzero__ = __bool__ - def __str__(self): return f"" diff --git a/src/DIRAC/Core/Utilities/Graphs/Graph.py b/src/DIRAC/Core/Utilities/Graphs/Graph.py index 8e958fe55fe..1a71372808e 100644 --- a/src/DIRAC/Core/Utilities/Graphs/Graph.py +++ b/src/DIRAC/Core/Utilities/Graphs/Graph.py @@ -6,22 +6,22 @@ """ import datetime -import time +import importlib import os - +import time from matplotlib.backends.backend_agg import FigureCanvasAgg from matplotlib.figure import Figure -from DIRAC.Core.Utilities.Graphs.GraphUtilities import pixelToPoint, evalPrefs, to_timestamp, add_time_to_title + +from DIRAC import gLogger from DIRAC.Core.Utilities.Graphs.GraphData import GraphData +from DIRAC.Core.Utilities.Graphs.GraphUtilities import add_time_to_title, evalPrefs, pixelToPoint, to_timestamp from DIRAC.Core.Utilities.Graphs.Legend import Legend -from DIRAC import gLogger DEBUG = 0 class Graph: - def __init__(self, *args, **kw): - super().__init__(*args, **kw) + """Base class for all other Graphs""" def layoutFigure(self, legend): @@ -275,18 +275,13 @@ def makeGraph(self, data, *args, **kw): for i in range(nPlots): plot_type = plot_prefs[i]["plot_type"] try: - # TODO: Remove when we moved to python3 - exec("import %s" % plot_type) - except ImportError: - print("Trying to use python like import") - try: - exec("from . import %s" % plot_type) - except ImportError as x: - print(f"Failed to import graph type {plot_type}: {str(x)}") - return None + plotModule = importlib.import_module(f"DIRAC.Core.Utilities.Graphs.{plot_type}") + except ModuleNotFoundError as x: + print(f"Failed to import graph type {plot_type}: {str(x)}") + return None ax = plot_axes[i] - plot = eval(f"{plot_type}.{plot_type}(graphData[i],ax,plot_prefs[i])") + plot = getattr(plotModule, plot_type)(graphData[i], ax, plot_prefs[i]) plot.draw() if DEBUG: @@ -342,8 +337,8 @@ def drawWaterMark(self, imagePath=None): ax_wm.axis("off") ax_wm.set_frame_on(False) ax_wm.set_clip_on(False) - except Exception as e: - print(e) + except Exception: + gLogger.exception("Caught expection") def writeGraph(self, fname, fileFormat="PNG"): """Write out the resulting graph to a file with fname in a given format""" @@ -352,4 +347,4 @@ def writeGraph(self, fname, fileFormat="PNG"): if fileFormat.lower() == "png": self.canvas.print_png(fname) else: - gLogger.error("File format '%s' is not supported!" % fileFormat) + gLogger.error("File format '{%s}' is not supported!") diff --git a/src/DIRAC/Core/Utilities/test/Test_File.py b/src/DIRAC/Core/Utilities/test/Test_File.py index 1891f2df5e2..7846398c84c 100644 --- a/src/DIRAC/Core/Utilities/test/Test_File.py +++ b/src/DIRAC/Core/Utilities/test/Test_File.py @@ -101,8 +101,6 @@ def testGetMD5ForFiles(): md5sum = getMD5ForFiles(filesList) reMD5 = re.compile("^[0-9a-fA-F]+$") assert reMD5.match(md5sum) is not None - # OK for python 2.7 - # self.assertRegexpMatches( md5sum, reMD5, "regexp doesn't match" ) @given(nb=floats(allow_nan=False, allow_infinity=False, min_value=1)) diff --git a/src/DIRAC/DataManagementSystem/Client/FileCatalogClientCLI.py b/src/DIRAC/DataManagementSystem/Client/FileCatalogClientCLI.py index 629ec79ad37..166486d8c20 100644 --- a/src/DIRAC/DataManagementSystem/Client/FileCatalogClientCLI.py +++ b/src/DIRAC/DataManagementSystem/Client/FileCatalogClientCLI.py @@ -1,17 +1,12 @@ #!/usr/bin/env python """ File Catalog Client Command Line Interface. """ -# TODO: This should be modernised to use subprocess(32) -try: - import commands -except ImportError: - # Python 3's subprocess module contains a compatibility layer - import subprocess as commands import os.path import time import sys import getopt +import uuid -from DIRAC import S_ERROR, S_OK +from DIRAC import S_ERROR from DIRAC.Core.Utilities.ReturnValues import returnSingleResult from DIRAC.Core.Base.CLI import CLI from DIRAC.Core.Security.ProxyInfo import getProxyInfo @@ -584,7 +579,7 @@ def registerFile(self, args): infoDict["Size"] = int(args[2]) infoDict["SE"] = args[3] if args[4] == "None": - _status, guid = commands.getstatusoutput("uuidgen") + guid = str(uuid.uuid4()) else: guid = args[4] infoDict["GUID"] = guid diff --git a/src/DIRAC/FrameworkSystem/Service/SystemAdministratorHandler.py b/src/DIRAC/FrameworkSystem/Service/SystemAdministratorHandler.py index 75327dac6c1..44337961332 100644 --- a/src/DIRAC/FrameworkSystem/Service/SystemAdministratorHandler.py +++ b/src/DIRAC/FrameworkSystem/Service/SystemAdministratorHandler.py @@ -8,17 +8,14 @@ import importlib import shutil import platform -import psutil import tempfile import subprocess +from datetime import datetime, timedelta import requests +import psutil from packaging.version import Version, InvalidVersion -import subprocess as commands - -from datetime import datetime, timedelta - from diraccfg import CFG from DIRAC import S_OK, S_ERROR, gConfig, rootPath, gLogger, convertToPy3VersionNumber @@ -278,7 +275,7 @@ def export_updateSoftware(self, version): primaryExtension = extension else: otherExtensions.append(extension) - self.log.info("Installing Python 3 based", f"{primaryExtension} {version}") + self.log.info("Installing", f"{primaryExtension} {version}") self.log.info("Will also install", repr(otherExtensions)) # Install DIRACOS @@ -510,7 +507,7 @@ def __readHostInfo(): # Disk occupancy summary = "" - _status, output = commands.getstatusoutput("df") + _status, output = subprocess.getstatusoutput("df") lines = output.split("\n") for i in range(len(lines)): if lines[i].startswith("/dev"): @@ -525,7 +522,7 @@ def __readHostInfo(): # Open files puser = getpass.getuser() - _status, output = commands.getstatusoutput("lsof") + _status, output = subprocess.getstatusoutput("lsof") pipes = 0 files = 0 sockets = 0 diff --git a/src/DIRAC/Interfaces/API/Dirac.py b/src/DIRAC/Interfaces/API/Dirac.py index d61c54af51e..143f7803597 100755 --- a/src/DIRAC/Interfaces/API/Dirac.py +++ b/src/DIRAC/Interfaces/API/Dirac.py @@ -51,13 +51,6 @@ COMPONENT_NAME = "DiracAPI" -try: - # Python 2: "file" is built-in - file_types = file, io.IOBase -except NameError: - # Python 3: "file" fully replaced with IOBase - file_types = (io.IOBase,) - def parseArguments(args): argList = [] @@ -648,14 +641,6 @@ def runLocal(self, job): self.log.info("Attempting to submit job to local site: %s" % DIRAC.siteName()) - # DIRACROOT is used for finding dirac-jobexec in python2 installations - # (it is normally set by the JobWrapper) - # We don't use DIRAC.rootPath as we assume that a DIRAC installation is already done at this point - # DIRAC env variable is only set for python2 installations - if "DIRAC" in os.environ: - os.environ["DIRACROOT"] = os.environ["DIRAC"] - self.log.verbose("DIRACROOT = %s" % (os.environ["DIRACROOT"])) - if "Executable" in parameters: executable = os.path.expandvars(parameters["Executable"]) else: @@ -773,7 +758,7 @@ def __printOutput(fd=None, message=""): print(message, file=sys.stderr) else: print(message) - elif isinstance(fd, file_types): + elif isinstance(fd, io.IOBase): print(message, file=fd) else: print(message) diff --git a/src/DIRAC/Interfaces/API/test/Test_DIRAC.py b/src/DIRAC/Interfaces/API/test/Test_DIRAC.py index ace91f87318..d42e142009c 100644 --- a/src/DIRAC/Interfaces/API/test/Test_DIRAC.py +++ b/src/DIRAC/Interfaces/API/test/Test_DIRAC.py @@ -44,8 +44,6 @@ def osmock(): os.environ["DIRAC"] = "/root/dirac" def expandMock(*args, **kwargs): - if "DIRACROOT" in os.environ and "DIRACROOT" in args[0]: - return args[0].replace("$DIRACROOT", os.environ["DIRACROOT"]) return args[0] def existsMock(*args, **kwargs): @@ -78,8 +76,6 @@ def confMock(): gConf = MagicMock(name="gConfig") def getVal(*args, **kwargs): - if "/LocalSite/Root" in args[0]: - return "/root/dirac" if len(args) == 2: return args[1] return "defaultValue" diff --git a/src/DIRAC/RequestManagementSystem/Client/Operation.py b/src/DIRAC/RequestManagementSystem/Client/Operation.py index c0d5274f3d6..1c20765131f 100644 --- a/src/DIRAC/RequestManagementSystem/Client/Operation.py +++ b/src/DIRAC/RequestManagementSystem/Client/Operation.py @@ -199,9 +199,6 @@ def __bool__(self): """for comparisons""" return True - # For Python 2 compatibility - __nonzero__ = __bool__ - def __len__(self): """nb of subFiles""" return len(self.__files__) diff --git a/src/DIRAC/RequestManagementSystem/Client/Request.py b/src/DIRAC/RequestManagementSystem/Client/Request.py index 8ba7ecaa3e5..aaf2f78dd18 100644 --- a/src/DIRAC/RequestManagementSystem/Client/Request.py +++ b/src/DIRAC/RequestManagementSystem/Client/Request.py @@ -250,9 +250,6 @@ def __bool__(self): """for comparisons""" return True - # For Python 2 compatibility - __nonzero__ = __bool__ - def __len__(self): """nb of subRequests""" return len(self.__operations__) diff --git a/src/DIRAC/Resources/Computing/ARC6ComputingElement.py b/src/DIRAC/Resources/Computing/ARC6ComputingElement.py index ba21427c755..378c24be2cf 100755 --- a/src/DIRAC/Resources/Computing/ARC6ComputingElement.py +++ b/src/DIRAC/Resources/Computing/ARC6ComputingElement.py @@ -6,9 +6,6 @@ Use it only if gridftp services are not supported anymore. Arc6CE should be dropped once the AREXCE will be fully operational. """ - -__RCSID__ = "$Id$" - import os import stat diff --git a/src/DIRAC/Resources/Computing/HTCondorCEComputingElement.py b/src/DIRAC/Resources/Computing/HTCondorCEComputingElement.py index 2d84f2bca7a..7aff2b12059 100644 --- a/src/DIRAC/Resources/Computing/HTCondorCEComputingElement.py +++ b/src/DIRAC/Resources/Computing/HTCondorCEComputingElement.py @@ -47,16 +47,10 @@ import os import tempfile -# TODO: This should be modernised to use subprocess(32) -try: - import commands -except ImportError: - # Python 3's subprocess module contains a compatibility layer - import subprocess as commands +import subprocess as commands import datetime import errno import threading -import json from DIRAC import S_OK, S_ERROR, gConfig from DIRAC.Resources.Computing.ComputingElement import ComputingElement diff --git a/src/DIRAC/WorkloadManagementSystem/Agent/JobAgent.py b/src/DIRAC/WorkloadManagementSystem/Agent/JobAgent.py index 3dee03949db..766531b5caf 100755 --- a/src/DIRAC/WorkloadManagementSystem/Agent/JobAgent.py +++ b/src/DIRAC/WorkloadManagementSystem/Agent/JobAgent.py @@ -4,11 +4,6 @@ The Job Agent constructs a classAd based on the local resource description in the CS and the current resource status that is used for matching. """ - -# ## This agent might potentially still need to be run in python2, so leave this - -# ############################# - import os import sys import re diff --git a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py index c2fc3407957..3ea6ca30fa6 100755 --- a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py +++ b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py @@ -92,11 +92,6 @@ def __init__(self, jobID=None, jobReport=None): # self.root is the path the Wrapper is running at self.root = os.getcwd() - # self.localSiteRoot is the path where the local DIRAC installation used to run the payload - # is taken from - self.localSiteRoot = gConfig.getValue("/LocalSite/Root", DIRAC.rootPath) - # FIXME: Why do we need to load any .cfg file here???? - self.__loadLocalCFGFiles(self.localSiteRoot) result = getCurrentVersion() if result["OK"]: self.diracVersion = result["Value"] @@ -222,7 +217,6 @@ def initialize(self, arguments): if extraOpts and "dirac-jobexec" in self.jobArgs.get("Executable", "").strip(): if os.path.exists(f"{self.root}/{extraOpts}"): shutil.copyfile(f"{self.root}/{extraOpts}", extraOpts) - self.__loadLocalCFGFiles(self.localSiteRoot) else: self.log.info("JobID is not defined, running in current directory") @@ -251,16 +245,6 @@ def __setInitialJobParameters(self): parameters.append(("JobWrapperPID", self.currentPID)) return self.__setJobParamList(parameters) - ############################################################################# - def __loadLocalCFGFiles(self, localRoot): - """Loads any extra CFG files residing in the local DIRAC site root.""" - files = os.listdir(localRoot) - self.log.debug("Checking directory %s for *.cfg files" % localRoot) - for localFile in files: - if re.search(".cfg$", localFile): - gConfig.loadFile(f"{localRoot}/{localFile}") - self.log.verbose("Found local .cfg file '%s'" % localFile) - ############################################################################# def __dictAsInfoString(self, dData, infoString="", currentBase=""): for key in dData: @@ -282,8 +266,6 @@ def execute(self): """The main execution method of the Job Wrapper""" self.log.info("Job Wrapper is starting execution phase for job %s" % (self.jobID)) os.environ["DIRACJOBID"] = str(self.jobID) - os.environ["DIRACROOT"] = self.localSiteRoot - self.log.verbose("DIRACROOT = %s" % (self.localSiteRoot)) os.environ["DIRACSITE"] = DIRAC.siteName() self.log.verbose("DIRACSITE = %s" % (DIRAC.siteName())) @@ -324,22 +306,10 @@ def execute(self): # the argument should include the jobDescription.xml file jobArguments = self.jobArgs.get("Arguments", "") - # This is a workaround for Python 2 style installations - if executable == "$DIRACROOT/scripts/dirac-jobexec": - self.log.warn( - 'Replaced job executable "$DIRACROOT/scripts/dirac-jobexec" with ' - '"dirac-jobexec". Please fix your submission script!' - ) - executable = "dirac-jobexec" - executable = os.path.expandvars(executable) exeThread = None spObject = None - if re.search("DIRACROOT", executable): - executable = executable.replace("$DIRACROOT", self.localSiteRoot) - self.log.verbose("Replaced $DIRACROOT for executable as %s" % (self.localSiteRoot)) - # Try to find the executable on PATH if "/" not in executable: # Returns None if the executable is not found so use "or" to leave it unchanged diff --git a/src/DIRAC/WorkloadManagementSystem/Utilities/Utils.py b/src/DIRAC/WorkloadManagementSystem/Utilities/Utils.py index e135f4a6302..0500cacea14 100644 --- a/src/DIRAC/WorkloadManagementSystem/Utilities/Utils.py +++ b/src/DIRAC/WorkloadManagementSystem/Utilities/Utils.py @@ -1,11 +1,10 @@ """ Utilities for WMS """ -import io import os import sys import json -from DIRAC import gConfig, gLogger, S_OK, S_ERROR +from DIRAC import gLogger, S_OK, S_ERROR from DIRAC.Core.Utilities.File import mkDir from DIRAC.ConfigurationSystem.Client.Helpers import Registry from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager @@ -31,16 +30,12 @@ def createJobWrapper( arguments = {"Job": jobParams, "CE": resourceParams, "Optimizer": optimizerParams} log.verbose("Job arguments are: \n %s" % (arguments)) - siteRoot = gConfig.getValue("/LocalSite/Root", os.getcwd()) - log.debug("SiteRootPythonDir is:\n%s" % siteRoot) - workingDir = gConfig.getValue("/LocalSite/WorkingDirectory", siteRoot) - mkDir("%s/job/Wrapper" % (workingDir)) - + mkDir("%s/job/Wrapper" % (os.getcwd())) diracRoot = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) - jobWrapperFile = f"{workingDir}/job/Wrapper/Wrapper_{jobID}" + jobWrapperFile = f"{os.getcwd()}/job/Wrapper/Wrapper_{jobID}" if os.path.exists(jobWrapperFile): - log.verbose("Removing existing Job Wrapper for %s" % (jobID)) + log.verbose("Removing existing Job Wrapper for {jobID}") os.remove(jobWrapperFile) with open(os.path.join(diracRoot, defaultWrapperLocation)) as fd: wrapperTemplate = fd.read() @@ -58,7 +53,7 @@ def createJobWrapper( # Making real substitutions # wrapperTemplate = wrapperTemplate.replace( "@JOBARGS@", str( arguments ) ) - wrapperTemplate = wrapperTemplate.replace("@SITEPYTHON@", str(siteRoot)) + wrapperTemplate = wrapperTemplate.replace("@SITEPYTHON@", os.getcwd()) jobWrapperJsonFile = jobWrapperFile + ".json" with open(jobWrapperJsonFile, "w", encoding="utf8") as jsonFile: @@ -67,7 +62,7 @@ def createJobWrapper( with open(jobWrapperFile, "w") as wrapper: wrapper.write(wrapperTemplate) - jobExeFile = f"{workingDir}/job/Wrapper/Job{jobID}" + jobExeFile = f"{os.getcwd()}/job/Wrapper/Job{jobID}" jobFileContents = """#!/bin/sh {} {} {} -o LogLevel={} -o /DIRAC/Security/UseServerCertificate=no """.format( diff --git a/tests/CI/install_client.sh b/tests/CI/install_client.sh index 11cff8e49c6..6ed86821c7e 100755 --- a/tests/CI/install_client.sh +++ b/tests/CI/install_client.sh @@ -46,10 +46,4 @@ DIRACSETUP=$(< "${INSTALL_CFG_FILE}" grep "Setup = " | cut -f5 -d " ") echo -e "*** $(date -u) **** Client INSTALLATION START ****\n" -findRelease - -if [[ -z "$DIRAC_RELEASE" ]]; then - export DIRAC_RELEASE=${projectVersion:?} -fi - installDIRAC diff --git a/tests/Jenkins/dirac_ci.sh b/tests/Jenkins/dirac_ci.sh index df9ee846553..4b930a0e14e 100644 --- a/tests/Jenkins/dirac_ci.sh +++ b/tests/Jenkins/dirac_ci.sh @@ -92,8 +92,6 @@ source "${TESTCODE}/DIRAC/tests/Jenkins/utilities.sh" installSite() { echo "==> [installSite]" - findRelease - generateCA generateCertificates diff --git a/tests/Jenkins/utilities.sh b/tests/Jenkins/utilities.sh index 2ca7076537b..702e1f40b10 100644 --- a/tests/Jenkins/utilities.sh +++ b/tests/Jenkins/utilities.sh @@ -37,89 +37,6 @@ default() { # Finders... functions devoted to find DBs, Services, versions, etc.. -#............................................................................. -# -# findRelease: -# -# It reads from releases.cfg and picks the latest version -# which is written to {dirac,externals}.version -# Unless variable ${projectVersion} is set: in this case, we use the specified DIRAC relese. -# -#............................................................................. - -# FIXME: use diraccfg -findRelease() { - echo '==> [findRelease]' - - - if [[ -n "${DIRAC_RELEASE}" ]]; then - echo "==> Specified release ${DIRAC_RELEASE}" - projectVersion=$DIRAC_RELEASE - else - # get the releases.cfg file from integration - (cd "${TESTCODE}/DIRAC/" || (echo 'ERROR: cannot change to ' "${TESTCODE}/DIRAC" && exit 42); - git remote -v - git remote add upstream https://github.com/DIRACGrid/DIRAC.git || true - git fetch --all || true - git branch -a - git show remotes/upstream/integration:releases.cfg > "${TESTCODE}/DIRAC/releases.cfg") - - # PRE='p[[:digit:]]*' - - if [[ -n "${DIRACBRANCH}" ]]; then - echo "==> Looking for DIRAC branch ${DIRACBRANCH}" - else - echo '==> Running on last one' - fi - - # Match project ( DIRAC ) version from releases.cfg - - # If I don't specify a DIRACBRANCH, it will get the latest "production" release - # First, try to find if we are on a production tag - if [[ ! "${projectVersion}" ]]; then - if [[ -n "${DIRACBRANCH}" ]]; then - projectVersion=$(grep '^\s*v[[:digit:]]*r[[:digit:]]*p[[:digit:]]*' "${TESTCODE}/DIRAC/releases.cfg" | grep "${DIRACBRANCH}" | head -1 | sed 's/ //g' || echo "") - else - projectVersion=$(grep '^\s*v[[:digit:]]*r[[:digit:]]*p[[:digit:]]*' "${TESTCODE}/DIRAC/releases.cfg" | head -1 | sed 's/ //g') - fi - # projectVersion=$(cat releases.cfg | grep [^:]v[[:digit:]]r[[:digit:]]*$PRE | head -1 | sed 's/ //g') - fi - - # The special case is when there's no 'p'... (e.g. version v6r15) - if [[ ! "${projectVersion}" ]]; then - if [[ -n "${DIRACBRANCH}" ]]; then - projectVersion=$(grep '^\s*v[[:digit:]]*r[[:digit:]]' "${TESTCODE}/DIRAC/releases.cfg" | grep "${DIRACBRANCH}" | head -1 | sed 's/ //g' || echo "") - else - projectVersion=$(grep '^\s*v[[:digit:]]*r[[:digit:]]' "${TESTCODE}/DIRAC/releases.cfg" | head -1 | sed 's/ //g') - fi - fi - - # In case there are no production tags for the branch, look for pre-releases in that branch - if [[ ! "${projectVersion}" ]]; then - if [[ -n "${DIRACBRANCH}" ]]; then - projectVersion=$(grep '^\s*v[[:digit:]]*r[[:digit:]]*'-pre'' "${TESTCODE}/DIRAC/releases.cfg" | grep "${DIRACBRANCH}" | head -1 | sed 's/ //g') - else - projectVersion=$(grep '^\s*v[[:digit:]]*r[[:digit:]]*'-pre'' "${TESTCODE}/DIRAC/releases.cfg" | head -1 | sed 's/ //g') - fi - fi - - # TODO: This should be made to fail to due set -u and -o pipefail - if [[ ! "${projectVersion}" ]]; then - echo "Failed to set projectVersion" >&2 - exit 1 - fi - - # projectVersionLine=$(grep -n "${projectVersion}" "${TESTCODE}/DIRAC/releases.cfg" | cut -d ':' -f 1 | head -1) - # # start := line number after "{" - # start=$(( projectVersionLine+2 )) - # # end := line number after "}" - # end=$(( start+2 )) - # versions=$(sed -n "$start,$end p" "${TESTCODE}/DIRAC/releases.cfg") - fi - echo "DIRAC:${projectVersion}" -} - - #............................................................................. # # findSystems: @@ -288,7 +205,7 @@ getCFGFile() { echo '==> [getCFGFile]' cp "$INSTALL_CFG_FILE" "${SERVERINSTALLDIR}/" - sed -i "s/VAR_Release/${projectVersion}/g" "${SERVERINSTALLDIR}/install.cfg" + sed -i "s/VAR_Release/${DIRAC_RELEASE}/g" "${SERVERINSTALLDIR}/install.cfg" } From 877afbd455c84288efd40db74f1975cdbbf62c7f Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Tue, 4 Oct 2022 15:42:33 +0200 Subject: [PATCH 0031/1135] style: just sorting the imports --- src/DIRAC/Core/scripts/dirac_configure.py | 20 +++--- src/DIRAC/Workflow/Modules/Script.py | 1 - .../JobWrapper/JobWrapper.py | 66 +++++++++---------- .../JobWrapper/Watchdog.py | 20 +++--- 4 files changed, 52 insertions(+), 55 deletions(-) diff --git a/src/DIRAC/Core/scripts/dirac_configure.py b/src/DIRAC/Core/scripts/dirac_configure.py index a37bd23adbe..49f0e04218c 100755 --- a/src/DIRAC/Core/scripts/dirac_configure.py +++ b/src/DIRAC/Core/scripts/dirac_configure.py @@ -41,17 +41,16 @@ --SkipCAChecks """ -import sys import os +import sys import warnings - import urllib3 import DIRAC -from DIRAC.Core.Utilities.File import mkDir +from DIRAC.ConfigurationSystem.Client.Helpers import Registry, cfgInstallPath, cfgPath from DIRAC.Core.Base.Script import Script from DIRAC.Core.Security.ProxyInfo import getProxyInfo -from DIRAC.ConfigurationSystem.Client.Helpers import cfgInstallPath, cfgPath, Registry +from DIRAC.Core.Utilities.File import mkDir from DIRAC.Core.Utilities.SiteSEMapping import getSEsForSite from DIRAC.FrameworkSystem.Client.BundleDeliveryClient import BundleDeliveryClient @@ -178,7 +177,7 @@ def setIssuer(self, optionValue): def _runConfigurationWizard(setups, defaultSetup): """The implementation of the configuration wizard""" - from prompt_toolkit import prompt, print_formatted_text, HTML + from prompt_toolkit import HTML, print_formatted_text, prompt from prompt_toolkit.completion import FuzzyWordCompleter # It makes no sense to have suggestions if there is no default so adjust the message accordingly @@ -222,8 +221,9 @@ def _runConfigurationWizard(setups, defaultSetup): def runConfigurationWizard(params): """Interactively configure DIRAC using metadata from installed extensions""" import subprocess - from prompt_toolkit import prompt, print_formatted_text, HTML + from DIRAC.Core.Utilities.Extensions import extensionsByPriority, getExtensionMetadata + from prompt_toolkit import HTML, print_formatted_text, prompt for extension in extensionsByPriority(): extensionMetadata = getExtensionMetadata(extension) @@ -289,13 +289,13 @@ def main(): def login(params): - from prompt_toolkit import prompt, print_formatted_text, HTML - from DIRAC.Resources.IdProvider.IdProviderFactory import IdProviderFactory from DIRAC.FrameworkSystem.private.authorization.utils.Tokens import ( - writeTokenDictToTokenFile, getTokenFileLocation, readTokenFromFile, + writeTokenDictToTokenFile, ) + from DIRAC.Resources.IdProvider.IdProviderFactory import IdProviderFactory + from prompt_toolkit import HTML, print_formatted_text, prompt # Init authorization client result = IdProviderFactory().getIdProvider("DIRACCLI", issuer=params.issuer, scope=" ") @@ -359,7 +359,7 @@ def runDiracConfigure(params): Script.registerSwitch("C:", "ConfigurationServer=", "Set as DIRAC configuration server", params.setServer) Script.registerSwitch("I", "IncludeAllServers", "include all Configuration Servers", params.setAllServers) Script.registerSwitch("n:", "SiteName=", "Set as DIRAC Site Name", params.setSiteName) - Script.registerSwitch("N:", "CEName=", "Determiner from ", params.setCEName) + Script.registerSwitch("N:", "CEName=", "Set as Computing Element name", params.setCEName) Script.registerSwitch("V:", "VO=", "Set the VO name", params.setVO) Script.registerSwitch("W:", "gateway=", "Configure as DIRAC Gateway for the site", params.setGateway) diff --git a/src/DIRAC/Workflow/Modules/Script.py b/src/DIRAC/Workflow/Modules/Script.py index 7021dbf4cfe..4471c5bbbf6 100644 --- a/src/DIRAC/Workflow/Modules/Script.py +++ b/src/DIRAC/Workflow/Modules/Script.py @@ -1,7 +1,6 @@ """ The Script class provides a simple way for users to specify an executable or file to run (and is also a simple example of a workflow module). """ -import io import os import re import shlex diff --git a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py index 3ea6ca30fa6..760f0339563 100755 --- a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py +++ b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py @@ -10,53 +10,49 @@ :caption: JobWrapper options """ +import datetime +import glob +import json import os -import stat import re -import sys -import time -import datetime import shutil -import threading +import stat +import sys import tarfile -import glob -import json - +import threading +import time from urllib.parse import unquote import DIRAC -from DIRAC import S_OK, S_ERROR, gConfig, gLogger +from DIRAC import S_ERROR, S_OK, gConfig, gLogger from DIRAC.AccountingSystem.Client.Types.Job import Job as AccountingJob -from DIRAC.Core.Utilities import DErrno -from DIRAC.Core.Utilities import List -from DIRAC.Core.Utilities import DEncode + +from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations +from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup +from DIRAC.ConfigurationSystem.Client.PathFinder import getSystemSection +from DIRAC.Core.Utilities import DEncode, DErrno, List +from DIRAC.Core.Utilities.Adler import fileAdler +from DIRAC.Core.Utilities.File import getGlobbedFiles, getGlobbedTotalSize from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader from DIRAC.Core.Utilities.SiteSEMapping import getSEsForSite -from DIRAC.Core.Utilities.Subprocess import systemCall -from DIRAC.Core.Utilities.Subprocess import Subprocess -from DIRAC.Core.Utilities.File import getGlobbedTotalSize, getGlobbedFiles +from DIRAC.Core.Utilities.Subprocess import Subprocess, systemCall from DIRAC.Core.Utilities.Version import getCurrentVersion -from DIRAC.Core.Utilities.Adler import fileAdler -from DIRAC.ConfigurationSystem.Client.PathFinder import getSystemSection -from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup -from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.DataManagementSystem.Client.DataManager import DataManager from DIRAC.DataManagementSystem.Client.FailoverTransfer import FailoverTransfer -from DIRAC.DataManagementSystem.Utilities.ResolveSE import getDestinationSEList -from DIRAC.Resources.Catalog.PoolXMLFile import getGUID -from DIRAC.Resources.Catalog.FileCatalog import FileCatalog -from DIRAC.RequestManagementSystem.Client.Request import Request +from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup from DIRAC.RequestManagementSystem.Client.Operation import Operation from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient +from DIRAC.RequestManagementSystem.Client.Request import Request from DIRAC.RequestManagementSystem.private.RequestValidator import RequestValidator -from DIRAC.WorkloadManagementSystem.JobWrapper.Watchdog import Watchdog -from DIRAC.WorkloadManagementSystem.Client.JobStateUpdateClient import JobStateUpdateClient -from DIRAC.WorkloadManagementSystem.Client.JobMonitoringClient import JobMonitoringClient +from DIRAC.Resources.Catalog.FileCatalog import FileCatalog +from DIRAC.Resources.Catalog.PoolXMLFile import getGUID +from DIRAC.WorkloadManagementSystem.Client import JobMinorStatus, JobStatus from DIRAC.WorkloadManagementSystem.Client.JobManagerClient import JobManagerClient -from DIRAC.WorkloadManagementSystem.Client.SandboxStoreClient import SandboxStoreClient +from DIRAC.WorkloadManagementSystem.Client.JobMonitoringClient import JobMonitoringClient from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport -from DIRAC.WorkloadManagementSystem.Client import JobStatus -from DIRAC.WorkloadManagementSystem.Client import JobMinorStatus +from DIRAC.WorkloadManagementSystem.Client.JobStateUpdateClient import JobStateUpdateClient +from DIRAC.WorkloadManagementSystem.Client.SandboxStoreClient import SandboxStoreClient +from DIRAC.WorkloadManagementSystem.JobWrapper.Watchdog import Watchdog EXECUTION_RESULT = {} @@ -899,10 +895,10 @@ def __transferOutputDataFiles(self, outputData, outputSE, outputPath): else: pfnGUID = result["Value"] - for outputFile in outputData: - (lfn, localfile) = self.__getLFNfromOutputFile(outputFile, outputPath) + for oData in outputData: + (lfn, localfile) = self.__getLFNfromOutputFile(oData, outputPath) if not os.path.exists(localfile): - self.log.error("Missing specified output data file:", outputFile) + self.log.error("Missing specified output data file:", oData) continue # # file size @@ -956,9 +952,9 @@ def __transferOutputDataFiles(self, outputData, outputSE, outputPath): if not self.defaultFailoverSE: self.log.info( "No failover SEs defined for JobWrapper,", - "cannot try to upload output file %s anywhere else." % outputFile, + "cannot try to upload output file %s anywhere else." % oData, ) - missing.append(outputFile) + missing.append(oData) continue failoverSEs = self.__getSortedSEList(self.defaultFailoverSE) @@ -975,7 +971,7 @@ def __transferOutputDataFiles(self, outputData, outputSE, outputPath): ) if not result["OK"]: self.log.error("Completely failed to upload file to failover SEs", result["Message"]) - missing.append(outputFile) + missing.append(oData) else: self.log.info("File %s successfully uploaded to failover storage element" % lfn) uploaded.append(lfn) diff --git a/src/DIRAC/WorkloadManagementSystem/JobWrapper/Watchdog.py b/src/DIRAC/WorkloadManagementSystem/JobWrapper/Watchdog.py index 208ef7a117d..ffed9ef4898 100755 --- a/src/DIRAC/WorkloadManagementSystem/JobWrapper/Watchdog.py +++ b/src/DIRAC/WorkloadManagementSystem/JobWrapper/Watchdog.py @@ -16,26 +16,28 @@ - Still to implement: - CPU normalization for correct comparison with job limit """ +import datetime +import errno +import getpass import os import re -import time, datetime import resource -import errno import socket -import getpass -import psutil +import time from pathlib import Path -from DIRAC import S_OK, S_ERROR, gLogger +import psutil + +from DIRAC import S_ERROR, S_OK, gLogger +from DIRAC.ConfigurationSystem.Client.Config import gConfig +from DIRAC.ConfigurationSystem.Client.PathFinder import getSystemInstance from DIRAC.Core.Utilities import MJF -from DIRAC.Core.Utilities.Profiler import Profiler from DIRAC.Core.Utilities.Os import getDiskSpace +from DIRAC.Core.Utilities.Profiler import Profiler from DIRAC.Core.Utilities.Subprocess import getChildrenPIDs -from DIRAC.ConfigurationSystem.Client.Config import gConfig -from DIRAC.ConfigurationSystem.Client.PathFinder import getSystemInstance from DIRAC.Resources.Computing.BatchSystems.TimeLeft.TimeLeft import TimeLeft -from DIRAC.WorkloadManagementSystem.Client.JobStateUpdateClient import JobStateUpdateClient from DIRAC.WorkloadManagementSystem.Client import JobMinorStatus +from DIRAC.WorkloadManagementSystem.Client.JobStateUpdateClient import JobStateUpdateClient class Watchdog: From 182a6309ea4f6022ca29f63ec0eb8cb9016803c5 Mon Sep 17 00:00:00 2001 From: Federico Stagni Date: Wed, 12 Oct 2022 15:08:02 +0200 Subject: [PATCH 0032/1135] style: mostly fixed some printouts --- src/DIRAC/Core/Utilities/Graphs/Graph.py | 4 ++-- .../JobWrapper/JobWrapper.py | 2 +- .../WorkloadManagementSystem/Utilities/Utils.py | 14 +++++++------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/DIRAC/Core/Utilities/Graphs/Graph.py b/src/DIRAC/Core/Utilities/Graphs/Graph.py index 1a71372808e..96d4953db73 100644 --- a/src/DIRAC/Core/Utilities/Graphs/Graph.py +++ b/src/DIRAC/Core/Utilities/Graphs/Graph.py @@ -338,7 +338,7 @@ def drawWaterMark(self, imagePath=None): ax_wm.set_frame_on(False) ax_wm.set_clip_on(False) except Exception: - gLogger.exception("Caught expection") + gLogger.exception("Caught exception") def writeGraph(self, fname, fileFormat="PNG"): """Write out the resulting graph to a file with fname in a given format""" @@ -347,4 +347,4 @@ def writeGraph(self, fname, fileFormat="PNG"): if fileFormat.lower() == "png": self.canvas.print_png(fname) else: - gLogger.error("File format '{%s}' is not supported!") + gLogger.error(f"File format '{fileFormat}' is not supported!") diff --git a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py index 760f0339563..e298df5b275 100755 --- a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py +++ b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py @@ -39,7 +39,7 @@ from DIRAC.Core.Utilities.Version import getCurrentVersion from DIRAC.DataManagementSystem.Client.DataManager import DataManager from DIRAC.DataManagementSystem.Client.FailoverTransfer import FailoverTransfer -from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup +from DIRAC.DataManagementSystem.Utilities.ResolveSE import getDestinationSEList from DIRAC.RequestManagementSystem.Client.Operation import Operation from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient from DIRAC.RequestManagementSystem.Client.Request import Request diff --git a/src/DIRAC/WorkloadManagementSystem/Utilities/Utils.py b/src/DIRAC/WorkloadManagementSystem/Utilities/Utils.py index 0500cacea14..e4846a5f10a 100644 --- a/src/DIRAC/WorkloadManagementSystem/Utilities/Utils.py +++ b/src/DIRAC/WorkloadManagementSystem/Utilities/Utils.py @@ -30,21 +30,21 @@ def createJobWrapper( arguments = {"Job": jobParams, "CE": resourceParams, "Optimizer": optimizerParams} log.verbose("Job arguments are: \n %s" % (arguments)) - mkDir("%s/job/Wrapper" % (os.getcwd())) + mkDir(os.path.join(os.getcwd(), "job/Wrapper")) diracRoot = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) jobWrapperFile = f"{os.getcwd()}/job/Wrapper/Wrapper_{jobID}" if os.path.exists(jobWrapperFile): - log.verbose("Removing existing Job Wrapper for {jobID}") + log.verbose("Removing existing Job Wrapper for", jobID) os.remove(jobWrapperFile) with open(os.path.join(diracRoot, defaultWrapperLocation)) as fd: wrapperTemplate = fd.read() if "LogLevel" in jobParams: logLevel = jobParams["LogLevel"] - log.info("Found Job LogLevel JDL parameter with value: %s" % (logLevel)) + log.info("Found Job LogLevel JDL parameter with value", logLevel) else: - log.info("Applying default LogLevel JDL parameter with value: %s" % (logLevel)) + log.info("Applying default LogLevel JDL parameter with value", logLevel) dPython = sys.executable realPythonPath = os.path.realpath(dPython) @@ -102,16 +102,16 @@ def createRelocatedJobWrapper( jobWrapperFile = os.path.join(wrapperPath, "Wrapper_%s" % jobID) if os.path.exists(jobWrapperFile): - log.verbose("Removing existing Job Wrapper for %s" % (jobID)) + log.verbose("Removing existing Job Wrapper for", jobID) os.remove(jobWrapperFile) with open(os.path.join(diracRoot, defaultWrapperLocation)) as fd: wrapperTemplate = fd.read() if "LogLevel" in jobParams: logLevel = jobParams["LogLevel"] - log.info("Found Job LogLevel JDL parameter with value: %s" % (logLevel)) + log.info("Found Job LogLevel JDL parameter with value", logLevel) else: - log.info("Applying default LogLevel JDL parameter with value: %s" % (logLevel)) + log.info("Applying default LogLevel JDL parameter with value", logLevel) # Making real substitutions # wrapperTemplate = wrapperTemplate.replace( "@JOBARGS@", str( arguments ) ) From 65a51473164349a13c2ee21a9aab0da36ead7f34 Mon Sep 17 00:00:00 2001 From: DIRACGrid CI Date: Thu, 13 Oct 2022 08:32:46 +0000 Subject: [PATCH 0033/1135] docs: Add release notes for v8.1.0a2 --- release.notes | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/release.notes b/release.notes index cc658f1fe7f..a90c7676498 100644 --- a/release.notes +++ b/release.notes @@ -1,3 +1,39 @@ +[v8.1.0a2] + +*DataManagementSystem + +FIX: (#6414) Proper casting for data sent to the monitoring +NEW: (#6410) backport getDestinationSEList from LHCbDIRAC +FIX: (#6392) DirectoryLevelTree: fix missing keyword argument name causing failures to upload files for example + +*Tests + +NEW: (#6410) introduce generateDIRACConfig utility to generate DIRAC config during tests + +*Docs + +CHANGE: (#6410) move site documentation from ConfigRef to Resources and dirac.cfg +NEW: (#6410) document Countries + +*WorkloadManagementSystem + +CHANGE: (#6410) JobWrapper uses getDestinationSEList rather that resolveSEGroup +CHANGE: (#6408) ElasticJobParametersDB: create a new index every 1M jobs +FIX: (#6397) JobScheduling: select the correct RescheduleDelay instead of 1 higher + +*MonitoringSystem + +CHANGE: (#6409) Remove DataOperationPlotter + +*Production + +NEW: (#6406) add dirac-prod-complete command + +*Core + +FIX: (#6398) Use safer mode for grid-security directories +FIX: (#6397) TimeUtilities.fromString: when given a datetime.datetime, return the same object, instead of None. Also Fixes JobScheduling issue + [v8.1.0a1] *WorkloadManagementSystem From 927faaa16338409f88c79fec035313ec91f4f329 Mon Sep 17 00:00:00 2001 From: fstagni Date: Fri, 14 Oct 2022 08:51:14 +0200 Subject: [PATCH 0034/1135] feat: remove DIRACSetup from RMS --- src/DIRAC/RequestManagementSystem/Client/Request.py | 3 --- src/DIRAC/RequestManagementSystem/DB/RequestDB.py | 2 -- .../RequestManagementSystem/private/RequestValidator.py | 7 ------- 3 files changed, 12 deletions(-) diff --git a/src/DIRAC/RequestManagementSystem/Client/Request.py b/src/DIRAC/RequestManagementSystem/Client/Request.py index aaf2f78dd18..abb90349fdd 100644 --- a/src/DIRAC/RequestManagementSystem/Client/Request.py +++ b/src/DIRAC/RequestManagementSystem/Client/Request.py @@ -31,7 +31,6 @@ class Request: :param str Name: request' name :param str OwnerDN: request's owner DN :param str OwnerGroup: request owner group - :param str Setup: DIRAC setup :param str SourceComponent: whatever :param int JobID: jobID :param datetime.datetime CreationTime: UTC datetime @@ -70,7 +69,6 @@ def __init__(self, fromDict=None): self._Status = "Done" self.JobID = 0 self.Error = None - self.DIRACSetup = None self.OwnerDN = None self.RequestName = None self.OwnerGroup = None @@ -386,7 +384,6 @@ def _getJSONData(self): "OwnerGroup", "Status", "Error", - "DIRACSetup", "SourceComponent", "JobID", "CreationTime", diff --git a/src/DIRAC/RequestManagementSystem/DB/RequestDB.py b/src/DIRAC/RequestManagementSystem/DB/RequestDB.py index 0b769e7f778..0b2eb47df07 100644 --- a/src/DIRAC/RequestManagementSystem/DB/RequestDB.py +++ b/src/DIRAC/RequestManagementSystem/DB/RequestDB.py @@ -138,7 +138,6 @@ requestTable = Table( "Request", metadata, - Column("DIRACSetup", String(32)), Column("CreationTime", DateTime), Column("JobID", Integer, server_default="0"), Column("OwnerDN", String(255)), @@ -919,7 +918,6 @@ def getRequestInfo(self, requestID): Request.JobID, Request.OwnerDN, Request.OwnerGroup, - Request.DIRACSetup, Request._SourceComponent, Request._CreationTime, Request._SubmitTime, diff --git a/src/DIRAC/RequestManagementSystem/private/RequestValidator.py b/src/DIRAC/RequestManagementSystem/private/RequestValidator.py index d04e8118232..69f33e156fc 100644 --- a/src/DIRAC/RequestManagementSystem/private/RequestValidator.py +++ b/src/DIRAC/RequestManagementSystem/private/RequestValidator.py @@ -159,13 +159,6 @@ def validate(self, request): # # if we're here request is more or less valid return S_OK() - @staticmethod - def _hasDIRACSetup(request): - """required attribute - DIRACSetup""" - if not request.DIRACSetup: - return S_ERROR("DIRACSetup not set") - return S_OK() - @staticmethod def _hasOwner(request): """required attributes OwnerDn and OwnerGroup""" From e6dea882d0bf00646c7b6de6a45e385d2a0df8ed Mon Sep 17 00:00:00 2001 From: fstagni Date: Mon, 17 Oct 2022 14:46:09 +0200 Subject: [PATCH 0035/1135] sweep: #6418 remove PilotSubmissionPlotter for monitoring --- .../Systems/MonitoringSystem/index.rst | 3 +- .../PilotSubmissionMonitoringPlotter.py | 124 ------------------ 2 files changed, 2 insertions(+), 125 deletions(-) delete mode 100644 src/DIRAC/MonitoringSystem/private/Plotters/PilotSubmissionMonitoringPlotter.py diff --git a/docs/source/AdministratorGuide/Systems/MonitoringSystem/index.rst b/docs/source/AdministratorGuide/Systems/MonitoringSystem/index.rst index f6301b09b1f..fc9bcb88da2 100644 --- a/docs/source/AdministratorGuide/Systems/MonitoringSystem/index.rst +++ b/docs/source/AdministratorGuide/Systems/MonitoringSystem/index.rst @@ -167,9 +167,10 @@ It will also fill an index called ``faileddataoperation_index`` containing entri Accessing the Monitoring information ===================================== -After you installed and configured the Monitoring system, you can use the Monitoring web application for the types WMSHistory, PilotSubmission, DataOperation and RMS. +After you installed and configured the Monitoring system, you can use the Monitoring web application for the types WMSHistory and RMS. However, every type can directly be monitored in Kibana dashboards that can be imported into your Elasticsearch (or Opensearch) instance. You can find and import these dashboards from DIRAC/dashboards as per the following example. +Grafana dashboards are also provided for some of the types. *Kibana dashboard for WMSHistory* A dashboard for WMSHistory monitoring ``WMSDashboard`` is available `here `__ for import as a NDJSON (as support for JSON is being removed in the latest versions of Kibana). diff --git a/src/DIRAC/MonitoringSystem/private/Plotters/PilotSubmissionMonitoringPlotter.py b/src/DIRAC/MonitoringSystem/private/Plotters/PilotSubmissionMonitoringPlotter.py deleted file mode 100644 index 7dbf65477d6..00000000000 --- a/src/DIRAC/MonitoringSystem/private/Plotters/PilotSubmissionMonitoringPlotter.py +++ /dev/null @@ -1,124 +0,0 @@ -""" -This class is used to define the plot using the plot attributes. -""" - -from DIRAC import S_OK, S_ERROR - -from DIRAC.MonitoringSystem.Client.Types.PilotSubmissionMonitoring import PilotSubmissionMonitoring -from DIRAC.MonitoringSystem.private.Plotters.BasePlotter import BasePlotter - - -class PilotSubmissionMonitoringPlotter(BasePlotter): - """ - .. class:: PilotSubmissionMonitoringPlotter - - It is used to crate the plots. - - param: str _typeName monitoring type - param: list _typeKeyFields list of keys what we monitor (list of attributes) - """ - - _typeName = "PilotSubmissionMonitoring" - _typeKeyFields = PilotSubmissionMonitoring().keyFields - - _reportNumberOfSubmissions = "Total Number of Submissions" - - def _reportNumberOfSubmissions(self, reportRequest): - """It is used to retrieve the data from the database. - - :param dict reportRequest: contains attributes used to create the plot. - :return: S_OK or S_ERROR {'data':value1, 'granularity':value2} value1 is a dictionary, value2 is the bucket length - """ - retVal = self._getTimedData( - startTime=reportRequest["startTime"], - endTime=reportRequest["endTime"], - selectField="NumTotal", - preCondDict=reportRequest["condDict"], - metadataDict=None, - ) - if not retVal["OK"]: - return retVal - dataDict, granularity = retVal["Value"] - return S_OK({"data": dataDict, "granularity": granularity}) - - def _plotNumberOfSubmissions(self, reportRequest, plotInfo, filename): - """It creates the plot. - - :param dict reportRequest: plot attributes - :param dict plotInfo: contains all the data which are used to create the plot - :param str filename: - :return: S_OK or S_ERROR { 'plot' : value1, 'thumbnail' : value2 } value1 and value2 are TRUE/FALSE - """ - metadata = { - "title": "Pilot Submissions by %s" % reportRequest["grouping"], - "starttime": reportRequest["startTime"], - "endtime": reportRequest["endTime"], - "span": plotInfo["granularity"], - "skipEdgeColor": True, - "ylabel": "Submissions", - } - - plotInfo["data"] = self._fillWithZero( - granularity=plotInfo["granularity"], - startEpoch=reportRequest["startTime"], - endEpoch=reportRequest["endTime"], - dataDict=plotInfo["data"], - ) - - return self._generateStackedLinePlot(filename=filename, dataDict=plotInfo["data"], metadata=metadata) - - _reportNumSucceededName = "Submission Efficiency" - - def _reportNumSucceeded(self, reportRequest): - """It is used to retrieve the data from the database. - - :param dict reportRequest: contains attributes used to create the plot. - :return: S_OK or S_ERROR {'data':value1, 'granularity':value2} value1 is a dictionary, value2 is the bucket length - """ - # Retrieve the number of succeeded submissions - retVal = self._getTimedData( - startTime=reportRequest["startTime"], - endTime=reportRequest["endTime"], - selectField="NumSucceeded", - preCondDict=reportRequest["condDict"], - metadataDict=None, - ) - if not retVal["OK"]: - return retVal - - # Retrieve the number of total submissions - retTotVal = self._getTimedData( - startTime=reportRequest["startTime"], - endTime=reportRequest["endTime"], - selectField="NumTotal", - preCondDict=reportRequest["condDict"], - metadataDict=None, - ) - if not retTotVal["OK"]: - return retTotVal - - dataDict, granularity = retVal["Value"] - totDataDict, granularity = retTotVal["Value"] - # Check that the dicts are not empty - if dataDict and totDataDict: - # Return the efficiency in dataDict - effDict = self._calculateEfficiencyDict(totDataDict, dataDict) - return S_OK({"data": effDict, "granularity": granularity}) - else: - return S_ERROR("No data available for this selection") - - def _plotNumSucceeded(self, reportRequest, plotInfo, filename): - """ - Make 2 dimensional pilotSubmission efficiency plot - - :param dict reportRequest: Condition to select data - :param dict plotInfo: Data for plot. - :param str filename: File name - """ - metadata = { - "title": "Pilot Submission efficiency by %s" % reportRequest["grouping"], - "starttime": reportRequest["startTime"], - "endtime": reportRequest["endTime"], - "span": plotInfo["granularity"], - } - return self._generateQualityPlot(filename, plotInfo["data"], metadata) From 57c3371e3c275d087fb04bf9ae44de6a704d4a3f Mon Sep 17 00:00:00 2001 From: chaen Date: Thu, 13 Oct 2022 09:22:48 +0200 Subject: [PATCH 0036/1135] sweep: #6411 Fix MySQL Optimizer and don't start JobAgent for integration tests --- src/DIRAC/Core/Utilities/MySQL.py | 7 ++++--- tests/Jenkins/utilities.sh | 6 ++++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/DIRAC/Core/Utilities/MySQL.py b/src/DIRAC/Core/Utilities/MySQL.py index f7054e65044..63016da3250 100755 --- a/src/DIRAC/Core/Utilities/MySQL.py +++ b/src/DIRAC/Core/Utilities/MySQL.py @@ -222,7 +222,7 @@ def captureOptimizerTraces(meth): cd ${DIRAC_MYSQL_OPTIMIZER_TRACES_PATH} c=0; for i in $(ls); do newFn=$(echo $i | sed -E "s/_trace_[0-9]+.[0-9]+_(.*)/_trace_${c}_\1/g"); mv $i $newFn; c=$(( c + 1 )); done - This tool is useful then to compare the files https://github.com/cosmicanant/recursive-diff + This tool is useful then to compare the files https://github.com/crusaderky/recursive_diff Note that this method is far from pretty: @@ -251,8 +251,9 @@ def captureOptimizerTraces(meth): def innerMethod(self, *args, **kwargs): # First, get a connection to the DB, and enable the tracing - connection = self.__connectionPool.get(self.__dbName) + connection = self._MySQL__connectionPool.get(self._MySQL__dbName)["Value"] connection.cursor().execute('SET optimizer_trace="enabled=on";') + # We also set some options that worked for my use case. # you may need to tune these parameters if you have huge traces # or more recursive calls. @@ -312,7 +313,7 @@ def innerMethod(self, *args, **kwargs): {"Query": trace_query, "Trace": json.loads(trace_analysis) if trace_analysis else None} ) - json.dump(jsonTraces, f) + json.dump(jsonTraces, f, indent=True) return res diff --git a/tests/Jenkins/utilities.sh b/tests/Jenkins/utilities.sh index 702e1f40b10..e77a64d3391 100644 --- a/tests/Jenkins/utilities.sh +++ b/tests/Jenkins/utilities.sh @@ -156,10 +156,12 @@ findAgents(){ echo 'ERROR: cannot change to ' "${SERVERINSTALLDIR}" >&2 exit 1 fi + + # Always remove the JobAgent, which is not a real agent if [[ -n "${AgentstoExclude}" ]]; then - python -m DIRAC.Core.Utilities.Extensions findAgents | grep -v "${AgentstoExclude}" > agents + python -m DIRAC.Core.Utilities.Extensions findAgents | grep -v "WorkloadManagementSystem JobAgent" | grep -v "${AgentstoExclude}" > agents else - python -m DIRAC.Core.Utilities.Extensions findAgents | grep "${AgentstoSearch}" > agents + python -m DIRAC.Core.Utilities.Extensions findAgents | grep -v "WorkloadManagementSystem JobAgent" | grep "${AgentstoSearch}" > agents fi echo "found $(wc -l agents)" From a35ae4b43f7cdabeceea143f06a2163bb588ef60 Mon Sep 17 00:00:00 2001 From: IgorPelevanyuk Date: Mon, 17 Oct 2022 15:38:08 +0200 Subject: [PATCH 0037/1135] sweep: #6427 Update InstallingDiracServer.rst --- .../ServerInstallations/InstallingDiracServer.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/source/AdministratorGuide/ServerInstallations/InstallingDiracServer.rst b/docs/source/AdministratorGuide/ServerInstallations/InstallingDiracServer.rst index 2622b1e2b54..551e82b3dc0 100644 --- a/docs/source/AdministratorGuide/ServerInstallations/InstallingDiracServer.rst +++ b/docs/source/AdministratorGuide/ServerInstallations/InstallingDiracServer.rst @@ -251,7 +251,8 @@ be taken based on the Python version you wish to install. # i.e., to produce the initial dirac.cfg for the server # # Give a Name to your User Community, it does not need to be the same name as in EGI, - # it can be used to cover more than one VO in the grid sense + # it can be used to cover more than one VO in the grid sense. + # If you are going to setup DIRAC as a multi-VO instance, remove the VirtualOrganization parameter. VirtualOrganization = Name of your VO # Site name SiteName = DIRAC.HostName.ch From 75b87c2fbda605c5ffb4fe2b59f7e307e4af75d6 Mon Sep 17 00:00:00 2001 From: fstagni Date: Mon, 17 Oct 2022 15:44:17 +0200 Subject: [PATCH 0038/1135] sweep: #6421 removed last bits of the Framework/Monitoring --- docs/docs.conf | 3 +- .../Framework/Services/Monitoring/index.rst | 12 --- .../Framework/Services/Plotting/index.rst | 12 --- .../Systems/Framework/Services/index.rst | 2 - .../Systems/Framework/URLs/index.rst | 13 ---- .../Tips/ServicesPorts/index.rst | 4 - .../scalingAndLimitations.rst | 4 - src/DIRAC/Core/scripts/install_full.cfg | 4 +- src/DIRAC/FrameworkSystem/ConfigTemplate.cfg | 32 -------- .../FrameworkSystem/Service/PlotCache.py | 74 ------------------- tests/Jenkins/utilities.sh | 4 +- 11 files changed, 4 insertions(+), 160 deletions(-) delete mode 100644 docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Services/Monitoring/index.rst delete mode 100644 docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Services/Plotting/index.rst delete mode 100644 src/DIRAC/FrameworkSystem/Service/PlotCache.py diff --git a/docs/docs.conf b/docs/docs.conf index 7f1482963cb..69b2269fcbb 100644 --- a/docs/docs.conf +++ b/docs/docs.conf @@ -32,8 +32,7 @@ no_inherited_members = DIRAC.DataManagementSystem.private.HttpStorageAccessHandler # only creating dummy files, because they cannot be safely imported due to sideEffects -create_dummy_files = lfc_dfc_copy, lfc_dfc_db_copy, JobWrapperTemplate, PlotCache, - PlottingHandler +create_dummy_files = lfc_dfc_copy, lfc_dfc_db_copy, JobWrapperTemplate # do not include these files in the documentation tree ignore_folders = diracdoctools, /test, /scripts diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Services/Monitoring/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Services/Monitoring/index.rst deleted file mode 100644 index e8845814243..00000000000 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Services/Monitoring/index.rst +++ /dev/null @@ -1,12 +0,0 @@ -Systems / Framework / / Service / Monitoring - Sub-subsection -======================================================================== - -Monitoring service is in charge of recollect the information necessary to create the plots. - -Extra options required to configure the monitoring system are: - -+----------------+------------------------------------------+--------------------------------+ -| **Name** | **Description** | **Example** | -+----------------+------------------------------------------+--------------------------------+ -| *DataLocation* | Path where data for monitoring is stored | DataLocation = data/Monitoring | -+----------------+------------------------------------------+--------------------------------+ diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Services/Plotting/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Services/Plotting/index.rst deleted file mode 100644 index 77c7e131965..00000000000 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Services/Plotting/index.rst +++ /dev/null @@ -1,12 +0,0 @@ -Systems / Framework / / Service / Plotting - Sub-subsection -======================================================================= - -Plotting Service generates graphs according to the client specifications and data. - -Extra options required to configure plotting system are: - -+-----------------+------------------------------------------+----------------------------+ -| **Name** | **Description** | **Example** | -+-----------------+------------------------------------------+----------------------------+ -| *PlotsLocation* | Path where data for monitoring is stored | PlotsLocation = data/plots | -+-----------------+------------------------------------------+----------------------------+ diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Services/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Services/index.rst index 823e349892d..18ec03fa9d8 100644 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Services/index.rst +++ b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Services/index.rst @@ -35,8 +35,6 @@ Services associated with Framework system are: :maxdepth: 2 BundleDelivery/index - Monitoring/index Notification/index - Plotting/index SystemAdministrator/index UserProfileManager/index diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/URLs/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/URLs/index.rst index 3189ae157b6..50e0b9f1326 100644 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/URLs/index.rst +++ b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/URLs/index.rst @@ -1,15 +1,6 @@ Systems / Framework / / URLs - Sub-subsection ========================================================== -Framework Services URLs. - -+------------------+--------------------------------------+-----------------------------------------------------------------+ -| **Name** | **Description** | **Example** | -+------------------+--------------------------------------+-----------------------------------------------------------------+ -| ** | URL associated with the service, the | Plotting = dips://dirac.eela.if.ufrj.br:9157/Framework/Plotting | -| | value is a URL using dips protocol | | -+------------------+--------------------------------------+-----------------------------------------------------------------+ - Services associated with Framework System: +-----------------------+----------+ @@ -17,12 +8,8 @@ Services associated with Framework System: +-----------------------+----------+ | *BundleDelivery* | 9158 | +-----------------------+----------+ -| *Monitoring* | 9142 | -+-----------------------+----------+ | *Notification* | 9154 | +-----------------------+----------+ -| *Plotting* | 9157 | -+-----------------------+----------+ | *ProxyManagement* | 9152 | +-----------------------+----------+ | *SecurityLogging* | 9153 | diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Tips/ServicesPorts/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Tips/ServicesPorts/index.rst index 5a391150baa..1982c41180e 100644 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Tips/ServicesPorts/index.rst +++ b/docs/source/AdministratorGuide/Configuration/ConfReference/Tips/ServicesPorts/index.rst @@ -29,8 +29,6 @@ Ordered by System / Services +------+----------------------+---------------------+ | 9154 | *Framework* | Notification | +------+----------------------+---------------------+ -| 9157 | *Framework* | Plotting | -+------+----------------------+---------------------+ | 9152 | *Framework* | ProxyManager | +------+----------------------+---------------------+ | 9153 | *Framework* | SecurityLogging | @@ -88,8 +86,6 @@ Ordered by port number +------+----------------------+---------------------+ | 9155 | *Framework* | UserProfileManager | +------+----------------------+---------------------+ -| 9157 | *Framework* | Plotting | -+------+----------------------+---------------------+ | 9158 | *Framework* | BundleDelivery | +------+----------------------+---------------------+ | 9162 | *Framework* | SystemAdministrator | diff --git a/docs/source/AdministratorGuide/ServerInstallations/scalingAndLimitations.rst b/docs/source/AdministratorGuide/ServerInstallations/scalingAndLimitations.rst index 5820e8549b8..5c25fc215da 100644 --- a/docs/source/AdministratorGuide/ServerInstallations/scalingAndLimitations.rst +++ b/docs/source/AdministratorGuide/ServerInstallations/scalingAndLimitations.rst @@ -95,12 +95,8 @@ Services + +---------------------------------------------------------------------------------------------------+-------------+---------------------------------------------------------------------------+-----------+ | | :mod:`ComponentMonitoring ` | YES | | + + +---------------------------------------------------------------------------------------------------+-------------+---------------------------------------------------------------------------+-----------+ -| | :mod:`Monitoring ` | NO | | + -+ +---------------------------------------------------------------------------------------------------+-------------+---------------------------------------------------------------------------+-----------+ | | :mod:`Notification ` | | | + + +---------------------------------------------------------------------------------------------------+-------------+---------------------------------------------------------------------------+-----------+ -| | :mod:`Plotting ` | | | + -+ +---------------------------------------------------------------------------------------------------+-------------+---------------------------------------------------------------------------+-----------+ | | :mod:`ProxyManager ` | YES | | + + +---------------------------------------------------------------------------------------------------+-------------+---------------------------------------------------------------------------+-----------+ | | :mod:`RabbitMQSync ` | | | + diff --git a/src/DIRAC/Core/scripts/install_full.cfg b/src/DIRAC/Core/scripts/install_full.cfg index 8154d9be747..a56c7c34923 100755 --- a/src/DIRAC/Core/scripts/install_full.cfg +++ b/src/DIRAC/Core/scripts/install_full.cfg @@ -90,13 +90,11 @@ LocalInstallation Services += DataManagement/StorageElement Services += DataManagement/FileCatalog Services += DataManagement/StorageElementProxy - Services += DataManagement/FTSManager - Services += Framework/Monitoring + Services += DataManagement/FTS3Manager Services += Framework/Notification Services += Framework/SecurityLogging Services += Framework/UserProfileManager Services += Framework/ProxyManager - Services += Framework/Plotting Services += Framework/BundleDelivery Services += Monitoring/Monitoring Services += WorkloadManagement/SandboxStore diff --git a/src/DIRAC/FrameworkSystem/ConfigTemplate.cfg b/src/DIRAC/FrameworkSystem/ConfigTemplate.cfg index 241a769cd2e..5a2ff523282 100644 --- a/src/DIRAC/FrameworkSystem/ConfigTemplate.cfg +++ b/src/DIRAC/FrameworkSystem/ConfigTemplate.cfg @@ -120,19 +120,6 @@ Services Default = authenticated } } - Plotting - { - Port = 9157 - PlotsLocation = data/plots - Authorization - { - Default = authenticated - FileTransfer - { - Default = authenticated - } - } - } BundleDelivery { Port = 9158 @@ -157,25 +144,6 @@ Services } } } - Monitoring - { - Port = 9142 - Authorization - { - Default = authenticated - FileTransfer - { - Default = authenticated - } - queryField = ServiceAdministrator - tryView = ServiceAdministrator - saveView = ServiceAdministrator - deleteView = ServiceAdministrator - deleteActivity = ServiceAdministrator - deleteActivities = ServiceAdministrator - deleteViews = ServiceAdministrator - } - } Notification { Port = 9154 diff --git a/src/DIRAC/FrameworkSystem/Service/PlotCache.py b/src/DIRAC/FrameworkSystem/Service/PlotCache.py deleted file mode 100644 index 92657651ad2..00000000000 --- a/src/DIRAC/FrameworkSystem/Service/PlotCache.py +++ /dev/null @@ -1,74 +0,0 @@ -""" Cache for the Plotting service plots -""" -import os.path -import time -import threading - -from DIRAC import S_OK, S_ERROR, gLogger -from DIRAC.Core.Utilities.DictCache import DictCache -from DIRAC.Core.Utilities.Graphs import graph - - -class PlotCache: - def __init__(self, plotsLocation=False): - self.plotsLocation = plotsLocation - self.alive = True - self.__graphCache = DictCache(deleteFunction=_deleteGraph) - self.__graphLifeTime = 600 - self.purgeThread = threading.Thread(target=self.purgeExpired) - self.purgeThread.start() - - def setPlotsLocation(self, plotsDir): - self.plotsLocation = plotsDir - for plot in os.listdir(self.plotsLocation): - if plot.find(".png") > 0: - plotLocation = f"{self.plotsLocation}/{plot}" - gLogger.verbose("Purging %s" % plotLocation) - os.unlink(plotLocation) - - def purgeExpired(self): - while self.alive: - time.sleep(self.__graphLifeTime) - self.__graphCache.purgeExpired() - - def getPlot(self, plotHash, plotData, plotMetadata, subplotMetadata): - """ - Get plot from the cache if exists, else generate it - """ - - plotDict = self.__graphCache.get(plotHash) - if plotDict is None: - basePlotFileName = f"{self.plotsLocation}/{plotHash}.png" - if subplotMetadata: - retVal = graph(plotData, basePlotFileName, plotMetadata, metadata=subplotMetadata) - else: - retVal = graph(plotData, basePlotFileName, plotMetadata) - if not retVal["OK"]: - return retVal - plotDict = retVal["Value"] - if plotDict["plot"]: - plotDict["plot"] = os.path.basename(basePlotFileName) - self.__graphCache.add(plotHash, self.__graphLifeTime, plotDict) - return S_OK(plotDict) - - def getPlotData(self, plotFileName): - filename = f"{self.plotsLocation}/{plotFileName}" - try: - with open(filename, "rb") as fd: - data = fd.read() - except Exception as v: - return S_ERROR(f"Can't open file {plotFileName}: {str(v)}") - return S_OK(data) - - -def _deleteGraph(plotDict): - try: - for key in plotDict: - value = plotDict[key] - if value and os.path.isfile(value): - os.unlink(value) - except Exception: - pass - - -gPlotCache = PlotCache() diff --git a/tests/Jenkins/utilities.sh b/tests/Jenkins/utilities.sh index 702e1f40b10..2c643a75703 100644 --- a/tests/Jenkins/utilities.sh +++ b/tests/Jenkins/utilities.sh @@ -660,7 +660,7 @@ diracServices(){ echo '==> [diracServices]' # Ignore tornado services - local services=$(cut -d '.' -f 1 < services | grep -v Tornado | grep -v TokenManager | grep -v PilotsLogging | grep -v StorageElementHandler | grep -v ^ConfigurationSystem | grep -v Plotting | grep -v RAWIntegrity | grep -v RunDBInterface | grep -v ComponentMonitoring | sed 's/System / /g' | sed 's/Handler//g' | sed 's/ /\//g') + local services=$(cut -d '.' -f 1 < services | grep -v Tornado | grep -v TokenManager | grep -v PilotsLogging | grep -v StorageElementHandler | grep -v ^ConfigurationSystem | grep -v RAWIntegrity | grep -v RunDBInterface | grep -v ComponentMonitoring | sed 's/System / /g' | sed 's/Handler//g' | sed 's/ /\//g') # group proxy, will be uploaded explicitly # echo '==> getting/uploading proxy for prod' @@ -711,7 +711,7 @@ diracUninstallServices(){ findServices # Ignore tornado services - local services=$(cut -d '.' -f 1 < services | grep -v TokenManager | grep -v ^ConfigurationSystem | grep -v Plotting | grep -v RAWIntegrity | grep -v RunDBInterface | grep -v ComponentMonitoring | grep -v Tornado | sed 's/System / /g' | sed 's/Handler//g' | sed 's/ /\//g') + local services=$(cut -d '.' -f 1 < services | grep -v TokenManager | grep -v ^ConfigurationSystem | grep -v RAWIntegrity | grep -v RunDBInterface | grep -v ComponentMonitoring | grep -v Tornado | sed 's/System / /g' | sed 's/Handler//g' | sed 's/ /\//g') # group proxy, will be uploaded explicitly # echo '==> getting/uploading proxy for prod' From 3674dea19433b0a89655da07bdafcb23c8a3d2b4 Mon Sep 17 00:00:00 2001 From: fstagni Date: Mon, 17 Oct 2022 16:39:37 +0200 Subject: [PATCH 0039/1135] sweep: #6420 Remove dirac-install saga --- Core/scripts/dirac-install.py | 35 +--------------- README.rst | 25 +++++------- setup.cfg | 3 -- src/DIRAC/Core/scripts/install_site.sh | 55 -------------------------- 4 files changed, 12 insertions(+), 106 deletions(-) delete mode 100755 src/DIRAC/Core/scripts/install_site.sh diff --git a/Core/scripts/dirac-install.py b/Core/scripts/dirac-install.py index ceeab08c728..0553c1065a3 100755 --- a/Core/scripts/dirac-install.py +++ b/Core/scripts/dirac-install.py @@ -1,15 +1,6 @@ #!/usr/bin/env python -import os -import stat import sys -import tempfile -try: - # For Python 3.0 and later - from urllib.request import urlopen -except ImportError: - # Fall back to Python 2's urllib2 - from urllib2 import urlopen sys.stderr.write("#" * 100 + "\n") sys.stderr.write("#" * 100 + "\n") @@ -17,9 +8,9 @@ sys.stderr.write("#" * 100 + "\n") sys.stderr.write("#" * 100 + "\n") sys.stderr.write("\n") -sys.stderr.write("Getting dirac-install from this location is no longer supported!\n") +sys.stderr.write("dirac-install (and python 2 installations) are no longer supported!\n") sys.stderr.write("\n") -sys.stderr.write("Please update your scripts to use:\n") +sys.stderr.write("If you really need it, use:\n") sys.stderr.write(" https://raw.githubusercontent.com/DIRACGrid/management/master/dirac-install.py\n") sys.stderr.write("\n") sys.stderr.write("#" * 100 + "\n") @@ -27,25 +18,3 @@ sys.stderr.write("#" * 100 + "\n") sys.stderr.write("#" * 100 + "\n") sys.stderr.write("#" * 100 + "\n") - -if os.environ.get("DIRAC_DEPRECATED_FAIL", "No").lower() in ("y", "yes", "t", "true", "on", "1"): - raise RuntimeError("Failing as DIRAC_DEPRECATED_FAIL is set") - -# Download dirac-install.py -response = urlopen("https://raw.githubusercontent.com/DIRACGrid/management/master/dirac-install.py") -code = response.getcode() -if code > 200 or code >= 300: - raise RuntimeError("Failed to download dirac-install.py with code %s" % code) - -# Write dirac-install.py to a temporay file -tmpHandle, tmp = tempfile.mkstemp() -fp = os.fdopen(tmpHandle, "wb") -fp.write(response.read()) -fp.close() - -# Make the dirac-install.py temporay file executable -st = os.stat(tmp) -os.chmod(tmp, st.st_mode | stat.S_IEXEC) - -# Replace the current process with the actual dirac-install.py script -os.execv(tmp, sys.argv) diff --git a/README.rst b/README.rst index 6f82b2e967e..3e044dd32d0 100644 --- a/README.rst +++ b/README.rst @@ -18,26 +18,22 @@ DIRAC has been started by the `LHCb collaboration `__. -DIRAC 7.3 adds support for Python 3 based clients and servers (which are production level from DIRAC 8.0). -We recommend transitioning to Python 3 clients during using DIRAC 7.2 and Python 3 servers using DIRAC 7.3. +DIRAC 8.0 drops support for Python 2 based clients and servers. There are three available options for installation: diff --git a/setup.cfg b/setup.cfg index 30ba8409997..f31277b03b9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,9 +24,6 @@ python_requires = >=3.9 package_dir= =src packages = find: -# TODO: This should be treated as a legacy workaround and eventually removed -scripts = - src/DIRAC/Core/scripts/install_site.sh install_requires = boto3 botocore diff --git a/src/DIRAC/Core/scripts/install_site.sh b/src/DIRAC/Core/scripts/install_site.sh deleted file mode 100755 index 8ee06327291..00000000000 --- a/src/DIRAC/Core/scripts/install_site.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash - -usage() { - echo Usage: - echo " install_site.sh [Options] ... CFG_file" - echo - echo "CFG_file - is the name of the installation configuration file which contains" - echo " all the instructions for the DIRAC installation. See DIRAC Administrator " - echo " Guide for the details" - echo "Options:" - echo " -d, --debug debug mode" - echo " -h, --help print this" -} - -while [ "${1}" ] -do - case "${1}" in - - -h | --help ) - usage - exit 0 - ;; - - -d | --debug ) - DEBUG='-o LogLevel=DEBUG' - ;; - - * ) - installCfg=${1} - ;; - - esac - shift -done - -if [[ -z "${installCfg}" ]]; then - usage - exit 1 -fi - -# Get the version of dirac-install requested - if none is requested, the version will come from integration -# -curl -L -o dirac-install https://raw.githubusercontent.com/DIRACGrid/management/master/dirac-install.py || exit -# -# define the target Dir -# -installDir=$(grep TargetPath "${installCfg}" | grep -v '#' | cut -d '=' -f 2 | sed -e 's/ //g') -# -mkdir -p "${installDir}" || exit -# - -python dirac-install -t server "${installCfg}" -source "${installDir}"/bashrc -dirac-configure --cfg "${installCfg}" "$DEBUG" -dirac-setup-site "${DEBUG}" From 21c6605750ef844c146c19362f11c5a83c0e81af Mon Sep 17 00:00:00 2001 From: fstagni Date: Mon, 17 Oct 2022 16:39:51 +0200 Subject: [PATCH 0040/1135] sweep: #6417 fix: removed unused SubmissionMode CE parameter --- dirac.cfg | 3 --- .../Resources/computingelements.rst | 19 +++---------------- .../Client/Helpers/Resources.py | 6 +----- .../ConfigurationSystem/Client/Utilities.py | 6 ------ .../Agent/PushJobAgent.py | 4 +--- .../Agent/SiteDirector.py | 4 +--- 6 files changed, 6 insertions(+), 36 deletions(-) diff --git a/dirac.cfg b/dirac.cfg index a93025a3f1d..0608afeac5f 100644 --- a/dirac.cfg +++ b/dirac.cfg @@ -903,9 +903,6 @@ Resources # This option can also go at the Queue level. LocalCEType = Pool - # If the CE is a cream CE the mode of submission - SubmissionMode = Direct - # (Optional) max number of processors that DIRAC pilots are allowed to exploit. Implicit default = 1 NumberOfProcessors = 12 diff --git a/docs/source/AdministratorGuide/Resources/computingelements.rst b/docs/source/AdministratorGuide/Resources/computingelements.rst index 07e7635c7cd..3a412cbdde4 100644 --- a/docs/source/AdministratorGuide/Resources/computingelements.rst +++ b/docs/source/AdministratorGuide/Resources/computingelements.rst @@ -73,12 +73,7 @@ of the *ComputingElement* is located in the inside the corresponding site sectio ce01.infn.it { # Type of the CE - CEType = CREAM - - # Submission mode should be "direct" in order to work with SiteDirector - # Otherwise the CE will be eligible for the use with third party broker, e.g. - # gLite WMS - SubmissionMode = direct + CEType = HTCondorCE # Section to describe various queue in the CE Queues @@ -117,7 +112,7 @@ automatically create the equivalent single core queues, see the :mod:`~DIRAC.Con configuration. -CREAM Computing Element +HTCondor Computing Element @@@@@@@@@@@@@@@@@@@@@@@@@@@@@ A commented example follows:: @@ -125,9 +120,7 @@ A commented example follows:: # Section placed in the */Resources/Sites///CEs* directory ce01.infn.it { - CEType = CREAM - SubmissionMode = direct - + CEType = HTCondorCE Queues { @@ -158,7 +151,6 @@ configuration follows :: # Type of the local batch system. Available batch system implementations are: # Torque, Condor, GE, LSF, OAR, SLURM BatchSystem = Torque - SubmissionMode = direct SSHHost = pc.domain.ch # SSH connection details to be defined in the local configuration # of the corresponding SiteDirector @@ -218,7 +210,6 @@ Like all SSH Computing Elements, it's defined like the following:: pc.farm.ch { CEType = SSHBatch - SubmissionMode = direct # Parameters of the SSH conection to the site. The /2 indicates how many cores can be used on that host. # It's equivalent to the number of jobs that can run in parallel. @@ -240,10 +231,6 @@ Like all SSH Computing Elements, it's defined like the following:: } - -.. versionadded:: > v6r10 - The SSHOptions option. - The ``SSHOptions`` is needed when for example the user used to run the agent isn't local and requires access to afs. As the way the agents are started isn't a login, they does not have access to afs (as they have no token), so no access to the HOME directory. Even if the HOME environment variable is replaced, ssh still looks up the original home directory. If the ssh key and/or the known_hosts file is hosted on afs, the ssh connection is likely to fail. The solution is to pass explicitely the options to ssh with the SSHOptions option. diff --git a/src/DIRAC/ConfigurationSystem/Client/Helpers/Resources.py b/src/DIRAC/ConfigurationSystem/Client/Helpers/Resources.py index 0aed942041c..a85ac81e6eb 100644 --- a/src/DIRAC/ConfigurationSystem/Client/Helpers/Resources.py +++ b/src/DIRAC/ConfigurationSystem/Client/Helpers/Resources.py @@ -229,7 +229,7 @@ def getQueue(site, ce, queue): return S_OK(resultDict) -def getQueues(siteList=None, ceList=None, ceTypeList=None, community=None, mode=None): +def getQueues(siteList=None, ceList=None, ceTypeList=None, community=None): """Get CE/queue options according to the specified selection""" result = gConfig.getSections("/Resources/Sites") @@ -260,10 +260,6 @@ def getQueues(siteList=None, ceList=None, ceTypeList=None, community=None, mode= continue ces = result["Value"] for ce in ces: - if mode: - ceMode = gConfig.getValue(f"/Resources/Sites/{grid}/{site}/CEs/{ce}/SubmissionMode", "Direct") - if not ceMode or ceMode.lower() != mode.lower(): - continue if ceTypeList: ceType = gConfig.getValue(f"/Resources/Sites/{grid}/{site}/CEs/{ce}/CEType", "") if not ceType or ceType not in ceTypeList: diff --git a/src/DIRAC/ConfigurationSystem/Client/Utilities.py b/src/DIRAC/ConfigurationSystem/Client/Utilities.py index 9aaf35241b4..18fd643d270 100644 --- a/src/DIRAC/ConfigurationSystem/Client/Utilities.py +++ b/src/DIRAC/ConfigurationSystem/Client/Utilities.py @@ -232,7 +232,6 @@ def makeNewQueueName(queueName, ceType): si00 = ceDict.get("SI00", "Unknown") ceType = ceDict.get("CEType", "Unknown") ram = ceDict.get("MaxRAM", "Unknown") - submissionMode = ceDict.get("SubmissionMode", "Unknown") # Current BDII CE info newarch = ceBdiiDict[site]["CEs"][ce].get("GlueHostArchitecturePlatformType", "").strip() @@ -252,9 +251,6 @@ def makeNewQueueName(queueName, ceType): if newCEType == "ARC-CE": newCEType = "ARC" - newSubmissionMode = None - if newCEType in ["ARC", "CREAM"]: - newSubmissionMode = "Direct" newRAM = ceInfo.get("GlueHostMainMemoryRAMSize", "").strip() # Protect from unreasonable values if newRAM and int(newRAM) > 150000: @@ -266,8 +262,6 @@ def makeNewQueueName(queueName, ceType): addToChangeSet((ceSection, "SI00", si00, newsi00), changeSet) addToChangeSet((ceSection, "CEType", ceType, newCEType), changeSet) addToChangeSet((ceSection, "MaxRAM", ram, newRAM), changeSet) - if submissionMode == "Unknown" and newSubmissionMode: - addToChangeSet((ceSection, "SubmissionMode", submissionMode, newSubmissionMode), changeSet) for queue, queueInfo in ceInfo["Queues"].items(): queueStatus = queueInfo["GlueCEStateStatus"] diff --git a/src/DIRAC/WorkloadManagementSystem/Agent/PushJobAgent.py b/src/DIRAC/WorkloadManagementSystem/Agent/PushJobAgent.py index b75986d002e..2ebf5635194 100644 --- a/src/DIRAC/WorkloadManagementSystem/Agent/PushJobAgent.py +++ b/src/DIRAC/WorkloadManagementSystem/Agent/PushJobAgent.py @@ -349,9 +349,7 @@ def _buildQueueDict(self, siteNames, ces, ceTypes): :return: dictionary of queue parameters """ - result = self.resourcesModule.getQueues( - community="", siteList=siteNames, ceList=ces, ceTypeList=ceTypes, mode="Direct" - ) + result = self.resourcesModule.getQueues(community="", siteList=siteNames, ceList=ces, ceTypeList=ceTypes) if not result["OK"]: return result diff --git a/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py b/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py index 370955bf132..c0b447a391a 100644 --- a/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py +++ b/src/DIRAC/WorkloadManagementSystem/Agent/SiteDirector.py @@ -223,9 +223,7 @@ def beginExecution(self): self.log.always("PilotDN:", self.pilotDN) self.log.always("PilotGroup:", self.pilotGroup) - result = self.resourcesModule.getQueues( - community=self.vo, siteList=siteNames, ceList=ces, ceTypeList=ceTypes, mode="Direct" - ) + result = self.resourcesModule.getQueues(community=self.vo, siteList=siteNames, ceList=ces, ceTypeList=ceTypes) if not result["OK"]: return result result = getQueuesResolved( From 02bc3f6b3126fc8637d334c7a6dd45e191cd419c Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Mon, 17 Oct 2022 20:20:25 +0200 Subject: [PATCH 0041/1135] docs: DIRAC release v7r3p31 --- releases.cfg | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/releases.cfg b/releases.cfg index e4e22cbbe74..609360e0e68 100644 --- a/releases.cfg +++ b/releases.cfg @@ -28,6 +28,11 @@ Releases Modules = DIRAC, RESTDIRAC:v0r5, COMDIRAC:v0r20, WebAppDIRAC:v4r4-pre1 DIRACOS = v1r27 } + v7r3p31 + { + Modules = DIRAC, RESTDIRAC:v0r7, COMDIRAC:v0r20, WebAppDIRAC:v4r3p11 + DIRACOS = v1r27 + } v7r3p30 { Modules = DIRAC, RESTDIRAC:v0r7, COMDIRAC:v0r20, WebAppDIRAC:v4r3p11 From 0de2c392224d86d585a53fbd33fdab8e40312a1b Mon Sep 17 00:00:00 2001 From: fstagni Date: Tue, 18 Oct 2022 09:51:02 +0200 Subject: [PATCH 0042/1135] sweep: #6435 fix: WMSUtilities.killPilotsInQueues have a consistent return value --- .../Service/PilotManagerHandler.py | 7 +----- .../Service/WMSUtilities.py | 23 ++++++++----------- 2 files changed, 11 insertions(+), 19 deletions(-) diff --git a/src/DIRAC/WorkloadManagementSystem/Service/PilotManagerHandler.py b/src/DIRAC/WorkloadManagementSystem/Service/PilotManagerHandler.py index dac312dce12..46cb769f71e 100644 --- a/src/DIRAC/WorkloadManagementSystem/Service/PilotManagerHandler.py +++ b/src/DIRAC/WorkloadManagementSystem/Service/PilotManagerHandler.py @@ -358,12 +358,7 @@ def export_killPilot(cls, pilotRefList): pilotRefDict[queue]["PilotList"].append(pilotReference) pilotRefDict[queue]["GridType"] = gridType - failed = killPilotsInQueues(pilotRefDict) - - if failed: - return S_ERROR("Failed to kill at least some pilots") - - return S_OK() + return killPilotsInQueues(pilotRefDict) ############################################################################## types_setJobForPilot = [[str, int], str] diff --git a/src/DIRAC/WorkloadManagementSystem/Service/WMSUtilities.py b/src/DIRAC/WorkloadManagementSystem/Service/WMSUtilities.py index 9993fb5bd99..ac57d4fc6b9 100644 --- a/src/DIRAC/WorkloadManagementSystem/Service/WMSUtilities.py +++ b/src/DIRAC/WorkloadManagementSystem/Service/WMSUtilities.py @@ -81,9 +81,8 @@ def killPilotsInQueues(pilotRefDict): """ ceFactory = ComputingElementFactory() - failed = [] - for key, pilotDict in pilotRefDict.items(): + for key, pilotDict in pilotRefDict.items(): owner, group, site, ce, queue = key.split("@@@") result = getQueue(site, ce, queue) if not result["OK"]: @@ -95,19 +94,17 @@ def killPilotsInQueues(pilotRefDict): return result ce = result["Value"] - # FIXME: quite hacky. Should be either removed, or based on some flag - if gridType in ["CREAM", "ARC", "Globus", "HTCondorCE"]: - group = getGroupOption(group, "VOMSRole", group) - ret = gProxyManager.getPilotProxyFromVOMSGroup(owner, group) - if not ret["OK"]: - gLogger.error("Could not get proxy:", 'User "{}" Group "{}" : {}'.format(owner, group, ret["Message"])) - return S_ERROR("Failed to get the pilot's owner proxy") - proxy = ret["Value"] - ce.setProxy(proxy) + group = getGroupOption(group, "VOMSRole", group) + ret = gProxyManager.getPilotProxyFromVOMSGroup(owner, group) + if not ret["OK"]: + gLogger.error("Could not get proxy:", f"User '{owner}' Group '{group}' : {ret['Message']}") + return S_ERROR("Failed to get the pilot's owner proxy") + proxy = ret["Value"] + ce.setProxy(proxy) pilotList = pilotDict["PilotList"] result = ce.killJob(pilotList) if not result["OK"]: - failed.extend(pilotList) + return result - return failed + return S_OK() From 8772c5d0209449dc28c6ec6bc27716c7d0bcebe7 Mon Sep 17 00:00:00 2001 From: Simon Fayer Date: Tue, 20 Sep 2022 21:02:02 +0100 Subject: [PATCH 0043/1135] feat: Remove VMDIRAC (use CloudCE instead) --- .../Systems/WorkloadManagement/VMDIRAC.rst | 309 ----- .../Systems/WorkloadManagement/index.rst | 1 - setup.cfg | 4 - .../Client/Helpers/Resources.py | 183 --- src/DIRAC/Core/Security/Properties.py | 3 - src/DIRAC/Resources/Cloud/CloudEndpoint.py | 549 -------- src/DIRAC/Resources/Cloud/EC2Endpoint.py | 205 --- src/DIRAC/Resources/Cloud/Endpoint.py | 55 - src/DIRAC/Resources/Cloud/EndpointFactory.py | 46 - src/DIRAC/Resources/Cloud/KeystoneClient.py | 275 ---- src/DIRAC/Resources/Cloud/OcciEndpoint.py | 479 ------- .../Resources/Cloud/OpenNebulaEndpoint.py | 275 ---- .../Resources/Cloud/OpenStackEndpoint.py | 544 -------- src/DIRAC/Resources/Cloud/RocciEndpoint.py | 223 ---- src/DIRAC/Resources/Cloud/Utilities.py | 334 ----- src/DIRAC/Resources/Cloud/__init__.py | 0 src/DIRAC/Resources/Cloud/cloudinit.template | 190 --- .../Resources/Cloud/ec2_instance_type.json | 162 --- .../scripts/dirac_resource_get_parameters.py | 8 +- .../Agent/CloudDirector.py | 513 ------- .../Agent/VirtualMachineMonitorAgent.py | 238 ---- .../Client/ServerUtils.py | 13 - .../Client/VMClient.py | 10 - .../Client/VirtualMachineCLI.py | 269 ---- .../ConfigTemplate.cfg | 29 - .../DB/VirtualMachineDB.py | 1174 ----------------- .../DB/VirtualMachineDB.sql | 2 - .../Service/VirtualMachineManagerHandler.py | 464 ------- .../CloudBootstrap/parse-jobagent-log | 96 -- .../Utilities/CloudBootstrap/power.sh | 9 - .../CloudBootstrap/save-payload-logs | 22 - .../Utilities/CloudBootstrap/vm-bootstrap | 283 ---- .../CloudBootstrap/vm-bootstrap-functions | 184 --- .../Utilities/CloudBootstrap/vm-monitor-agent | 41 - .../Utilities/CloudBootstrap/vm-pilot | 127 -- .../Utilities/Utils.py | 37 - .../scripts/dirac_vm_cli.py | 15 - .../scripts/dirac_vm_endpoint_status.py | 87 -- .../scripts/dirac_vm_get_pilot_output.py | 31 - .../scripts/dirac_vm_instance_stop.py | 29 - .../Test_VirtualMachineDB.py | 219 --- .../Test_VirtualMachineManagerClient.py | 98 -- tests/Jenkins/utilities.sh | 2 +- 43 files changed, 2 insertions(+), 7835 deletions(-) delete mode 100644 docs/source/AdministratorGuide/Systems/WorkloadManagement/VMDIRAC.rst delete mode 100644 src/DIRAC/Resources/Cloud/CloudEndpoint.py delete mode 100644 src/DIRAC/Resources/Cloud/EC2Endpoint.py delete mode 100644 src/DIRAC/Resources/Cloud/Endpoint.py delete mode 100644 src/DIRAC/Resources/Cloud/EndpointFactory.py delete mode 100644 src/DIRAC/Resources/Cloud/KeystoneClient.py delete mode 100644 src/DIRAC/Resources/Cloud/OcciEndpoint.py delete mode 100644 src/DIRAC/Resources/Cloud/OpenNebulaEndpoint.py delete mode 100644 src/DIRAC/Resources/Cloud/OpenStackEndpoint.py delete mode 100644 src/DIRAC/Resources/Cloud/RocciEndpoint.py delete mode 100644 src/DIRAC/Resources/Cloud/Utilities.py delete mode 100644 src/DIRAC/Resources/Cloud/__init__.py delete mode 100644 src/DIRAC/Resources/Cloud/cloudinit.template delete mode 100644 src/DIRAC/Resources/Cloud/ec2_instance_type.json delete mode 100644 src/DIRAC/WorkloadManagementSystem/Agent/CloudDirector.py delete mode 100644 src/DIRAC/WorkloadManagementSystem/Agent/VirtualMachineMonitorAgent.py delete mode 100644 src/DIRAC/WorkloadManagementSystem/Client/VMClient.py delete mode 100644 src/DIRAC/WorkloadManagementSystem/Client/VirtualMachineCLI.py delete mode 100755 src/DIRAC/WorkloadManagementSystem/DB/VirtualMachineDB.py delete mode 100644 src/DIRAC/WorkloadManagementSystem/DB/VirtualMachineDB.sql delete mode 100644 src/DIRAC/WorkloadManagementSystem/Service/VirtualMachineManagerHandler.py delete mode 100644 src/DIRAC/WorkloadManagementSystem/Utilities/CloudBootstrap/parse-jobagent-log delete mode 100644 src/DIRAC/WorkloadManagementSystem/Utilities/CloudBootstrap/power.sh delete mode 100644 src/DIRAC/WorkloadManagementSystem/Utilities/CloudBootstrap/save-payload-logs delete mode 100644 src/DIRAC/WorkloadManagementSystem/Utilities/CloudBootstrap/vm-bootstrap delete mode 100644 src/DIRAC/WorkloadManagementSystem/Utilities/CloudBootstrap/vm-bootstrap-functions delete mode 100644 src/DIRAC/WorkloadManagementSystem/Utilities/CloudBootstrap/vm-monitor-agent delete mode 100644 src/DIRAC/WorkloadManagementSystem/Utilities/CloudBootstrap/vm-pilot delete mode 100755 src/DIRAC/WorkloadManagementSystem/scripts/dirac_vm_cli.py delete mode 100755 src/DIRAC/WorkloadManagementSystem/scripts/dirac_vm_endpoint_status.py delete mode 100755 src/DIRAC/WorkloadManagementSystem/scripts/dirac_vm_get_pilot_output.py delete mode 100755 src/DIRAC/WorkloadManagementSystem/scripts/dirac_vm_instance_stop.py delete mode 100644 tests/Integration/WorkloadManagementSystem/Test_VirtualMachineDB.py delete mode 100644 tests/Integration/WorkloadManagementSystem/Test_VirtualMachineManagerClient.py diff --git a/docs/source/AdministratorGuide/Systems/WorkloadManagement/VMDIRAC.rst b/docs/source/AdministratorGuide/Systems/WorkloadManagement/VMDIRAC.rst deleted file mode 100644 index 84277256fd9..00000000000 --- a/docs/source/AdministratorGuide/Systems/WorkloadManagement/VMDIRAC.rst +++ /dev/null @@ -1,309 +0,0 @@ -.. _installingVMDIRAC: - -================== -Installing VMDIRAC -================== - -.. toctree:: - :maxdepth: 2 - -.. contents:: Table of contents - :depth: 4 - ------------- -Introduction ------------- - -VMDIRAC has now been merged into the main DIRAC release (as of v7r3/7.3). It -consists of the following parts: - -VirtualMachineDB (Database) - This is the database that stores the details of each VM -managed by VMDIRAC. There should be one instance of this on your system. - -CloudDirector (Agent) - This is analogous to the core DIRAC SiteDirector. It inspects -the TaskQueues, works out if there are any compatible jobs waiting (without -existing VMs) for cloud resources and starts VM instances as needed. The VM -details are stored in the database for future reference. You need at least one -CloudDirector, for multi community/VO services, it's advisable to have one -CloudDirector per community. - -VirtualMachineManager (Service) - This service provides the virtual machine -life-cycle management interface; the CLI tools contact this service to -list/kill running VMs. It also has an inbuilt thread that will inspect existing -VMs, tidying up any that are stopped or haven't reported back for a long time. - -VirtualMachineMonitor (Agent) - This agent monitors the health of the current -VM it is running on an reports it back to the VirtualMachineManager. It should -not be installed on the central DIRAC instance, but started in the cloud VM -instances themselves. This lets the VirtualMachineManager know that the -instance is still alive and also handles stopping the instance after any -tasks/jobs have finished. - -VMDIRAC WebApp Extention - This WebApp plugin adds an extra VirtualMachines tab -to the usual DIRAC webinterface for summarising the contents of the database. - ---------------- -Install VMDIRAC ---------------- - -On a DIRAC server (generally collocated with the other WorkloadManagement -system components), configure and install the extra components: - -* Resource configuration - - In the configuration add the User/Password information to connect to the Cloud endpoint. Also you should put a valid path for the host certificate, *e.g.*: - - :: - - Resources - { - Sites - { - Cloud - { - Cloud.LUPM.fr - { - Cloud - { - example.cloud.hostname - { - User = xxxx - Password = xxxx - HostCert = /opt/dirac/etc/grid-security/hostcert.pem - HostKey = /opt/dirac/etc/grid-security/hostkey.pem - CreatePublicIP = True - } - } - } - } - } - } - -* Install the following components: - - * DB: VirtualMachineDB - * Service: WorkloadManagement_VirtualMachineManager - * Agent: WorkloadManagement_CloudDirector - ------------------------------------- -Setup for using cloudinit and Pilot3 ------------------------------------- - -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Using OpenStack with an application credential -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -* The user must have access to the OpenStack cloud and be allowed to start up instances. -* Login to the cloud webinterface with your account. -* Go to the identity -> Application credentials panel -* Create a new credential with any name, no expire and no roles selected -* Copy the ID and secret strings somewhere safe for a moment -* Put the ID and string into a new file on the DIRAC server running the - CloudDirectors in the following format (one line, separated by a space): - -* Make sure the file is owned by the user running dirac and has 0600 permissions. -* Add the location of this file to the Resource Settings. - -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -OpenStack Resource Configuration -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -On the OpenStack Resource you will need the following: - -* user account -* user's ssh key uploaded to the OpenStack server: Associated with the instance for debugging. - This key is user specific, not project specific. -* flavour (ID or name) -* image (ID or name) -* network (ID or name) - - :: - - CLOUD.ExampleName.uk - { - Name = [ExampleName] - # must be unique, use e.g. hostname of the OpenStack webinterface - CE = [hostname.example.ac.uk] - Cloud - { - [hostname.example.ac.uk] - { - # assuming your cloud is using a standard CA - CAPath = /etc/pki/tls/cert.pem - # list your favourite VOs here - VO = gridpp - VO += lz - NetworkID = [network uuid] - Network = [name_of_network] - CEType = OpenStack - MaxInstances = [maximum number of instances] - AuthURL = [https://keystone.example.ac.uk:5000/v3] - Appcred = [path to appcred file created earlier] - # this might be optional - CVMFSProxy = http://[your cvmfs proxy cache]:3128 - Images - { - [image name, e.g. CentOS-7-x86_64-GenericCloud-1905] - { - ImageID = [image uuid] - FlavorName = [flavour name] - # this is currently a dummy value - Platform = [DIRACPlatForm] - } - } - OSKeyName = [ssh key name of the OpenStack user] - Tenant = [Openstack Project Name] - } - } - - CEs - { - [hostname.example.ac.uk] - { - CEType = Cloud - Architecture = x86_64 - Queues - { - [image name] - { - maxCPUTime = 24000000 - } - } - } - } - } - ------------------------------- -Configuration - other examples ------------------------------- - -* In the CS Resources section, configure the cloud endpoint as in this example - - :: - - Resources - { - Sites - { - Cloud - { - Cloud.LUPM.fr - { - CE = 194.214.86.244 - Cloud - { - 194.214.86.244 - { - CEType = Cloud - ex_security_groups = default - ex_force_auth_url = http://194.214.86.244:5000/v3/auth/tokens - ex_force_service_region = LUPM-CLOUD - # This is the max number of VM instances that will be running in parallel - # Each VM can have multiple cores, each one executing a job - MaxInstances = 4 - ex_force_auth_version = 3.x_password - ex_tenant_name = dirac - ex_domain_name = msfg.fr - networks = dirac-net - # This is the public key previously uploaded to the Cloud provider - # It's needed to ssh connect to VMs - keyname = cta_cloud_lupm - # If this option is set, public IP are assigned to VMs - # It's needed to ssh connect to VMs - ipPool = ext-net - } - Images - { - # It can be a public or a private image - Centos6-Officielle - { - ImageID = 35403255-f5f1-4c61-96dc-e59678942c6d - FlavorName = m1.medium - } - } - } - } - } - } - } - - -* CS Operation section - - :: - - Operations - { - CTA - { - Cloud - { - GenericCloudGroup = cta_genpilot - GenericCloudUser = arrabito - user_data_commands = vm-bootstrap - user_data_commands += vm-bootstrap-functions - user_data_commands += vm-pilot - user_data_commands += vm-monitor-agent - user_data_commands += pilotCommands.py - user_data_commands += pilotTools.py - user_data_commands += power.sh - user_data_commands += parse-jobagent-log - user_data_commands += dirac-pilot.py - user_data_commands += save-payload-logs - # url from which command scripts are downloaded. Usually the url of the web server - user_data_commands_base_url = http://cta-dirac.in2p3.fr/DIRAC/defaults - Project = CTA - Version = v1r40 - } - } - } - -* CS Registry section - - The host where VMDIRAC is installed and the certificate of which is used for the VMs, it should have these 2 properties set (as in the example below): - - * Properties = GenericPilot (needed to make pilots running on the VM matching jobs in the TaskQueue) - * Properties = VmRpcOperation (needed by the VirtualMachineMonitorAgent running on the VM to be authorized to send Heartbeats to the VirtualMachineManager service) - - :: - - Registry - { - Hosts - { - dcta-agents01.pic.es - { - DN = /DC=org/DC=terena/DC=tcs/C=ES/ST=Barcelona/L=Bellaterra/O=Institut de Fisica dAltes Energies/CN=dcta-agents01.pic.es - CA = /C=NL/ST=Noord-Holland/L=Amsterdam/O=TERENA/CN=TERENA eScience SSL CA 3 - Properties = FullDelegation - Properties += CSAdministrator - Properties += ProxyManagement - Properties += SiteManager - Properties += Operator - Properties += JobAdministrator - Properties += CSAdministrator - Properties += TrustedHost - Properties += GenericPilot - Properties += VmRpcOperation - } - } - } - ----------------------- -Install VMDIRAC WebApp ----------------------- - -* The VirtualMachines panel is now included in the main WebApp release. -* If using the old (non cloudinit) bootstrap, you must Create sym links for the - old bootstrap scripts: - - :: - - $ ll /opt/dirac/webRoot/www/defaults/bootstrap - total 0 - lrwxrwxrwx 1 dirac dirac 76 Feb 21 08:50 parse-jobagent-log -> /opt/dirac/pro/DIRAC/WorkloadManagementSystem/Utilities/CloudBootstrap/parse-jobagent-log - lrwxrwxrwx 1 dirac dirac 66 Feb 21 08:52 power.sh -> /opt/dirac/pro/DIRAC/WorkloadManagementSystem/CloudBootstrap/power.sh - lrwxrwxrwx 1 dirac dirac 75 Feb 21 08:52 save-payload-logs -> /opt/dirac/pro/DIRAC/WorkloadManagementSystem/CloudBootstrap/save-payload-logs - lrwxrwxrwx 1 dirac dirac 70 Feb 21 11:47 vm-bootstrap -> /opt/dirac/pro/DIRAC/WorkloadManagementSystem/CloudBootstrap/vm-bootstrap - lrwxrwxrwx 1 dirac dirac 80 Feb 21 08:52 vm-bootstrap-functions -> /opt/dirac/pro/DIRAC/WorkloadManagementSystem/CloudBootstrap/vm-bootstrap-functions - lrwxrwxrwx 1 dirac dirac 74 Feb 21 08:53 vm-monitor-agent -> /opt/dirac/pro/DIRAC/WorkloadManagementSystem/CloudBootstrap/vm-monitor-agent - lrwxrwxrwx 1 dirac dirac 66 Feb 21 08:53 vm-pilot -> /opt/dirac/pro/DIRAC/WorkloadManagementSystem/CloudBootstrap/vm-pilot diff --git a/docs/source/AdministratorGuide/Systems/WorkloadManagement/index.rst b/docs/source/AdministratorGuide/Systems/WorkloadManagement/index.rst index 5ceaa110b52..fb06f7cd52e 100644 --- a/docs/source/AdministratorGuide/Systems/WorkloadManagement/index.rst +++ b/docs/source/AdministratorGuide/Systems/WorkloadManagement/index.rst @@ -89,5 +89,4 @@ The following sections add some detail for the WMS systems. JobsMatching tagsAndJobs multiProcessorJobs - VMDIRAC InputDataResolution diff --git a/setup.cfg b/setup.cfg index 3e58a8866d6..abb3e884da9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -286,10 +286,6 @@ console_scripts = dirac-admin-show-task-queues = DIRAC.WorkloadManagementSystem.scripts.dirac_admin_show_task_queues:main [admin] dirac-admin-sync-pilot = DIRAC.WorkloadManagementSystem.scripts.dirac_admin_sync_pilot:main [admin] dirac-jobexec = DIRAC.WorkloadManagementSystem.scripts.dirac_jobexec:main [pilot] - dirac-vm-cli = DIRAC.WorkloadManagementSystem.scripts.dirac_vm_cli:main [admin] - dirac-vm-endpoint-status = DIRAC.WorkloadManagementSystem.scripts.dirac_vm_endpoint_status:main [admin] - dirac-vm-get-pilot-output = DIRAC.WorkloadManagementSystem.scripts.dirac_vm_get_pilot_output:main [admin] - dirac-vm-instance-stop = DIRAC.WorkloadManagementSystem.scripts.dirac_vm_instance_stop:main [admin] dirac-wms-cpu-normalization = DIRAC.WorkloadManagementSystem.scripts.dirac_wms_cpu_normalization:main [pilot] dirac-wms-get-queue-cpu-time = DIRAC.WorkloadManagementSystem.scripts.dirac_wms_get_queue_cpu_time:main [pilot] dirac-wms-get-wn = DIRAC.WorkloadManagementSystem.scripts.dirac_wms_get_wn:main [admin] diff --git a/src/DIRAC/ConfigurationSystem/Client/Helpers/Resources.py b/src/DIRAC/ConfigurationSystem/Client/Helpers/Resources.py index 0aed942041c..f03eb818682 100644 --- a/src/DIRAC/ConfigurationSystem/Client/Helpers/Resources.py +++ b/src/DIRAC/ConfigurationSystem/Client/Helpers/Resources.py @@ -446,189 +446,6 @@ def getInfoAboutProviders(of=None, providerName=None, option="", section=""): return S_OK(gConfig.getValue(f"{gBaseResourcesSection}/{of}Providers/{providerName}/{section}/{option}")) -def findGenericCloudCredentials(vo=False, group=False): - """Get the cloud credentials to use for a specific VO and/or group.""" - if not group and not vo: - return S_ERROR("Need a group or a VO to determine the Generic cloud credentials") - if not vo: - vo = Registry.getVOForGroup(group) - if not vo: - return S_ERROR("Group %s does not have a VO associated" % group) - opsHelper = Operations.Operations(vo=vo) - cloudGroup = opsHelper.getValue("Cloud/GenericCloudGroup", "") - cloudDN = opsHelper.getValue("Cloud/GenericCloudDN", "") - if not cloudDN: - cloudUser = opsHelper.getValue("Cloud/GenericCloudUser", "") - if cloudUser: - result = Registry.getDNForUsername(cloudUser) - if result["OK"]: - cloudDN = result["Value"][0] - else: - return S_ERROR("Failed to find suitable CloudDN") - if cloudDN and cloudGroup: - gLogger.verbose(f"Cloud credentials from CS: {cloudDN}@{cloudGroup}") - result = gProxyManager.userHasProxy(cloudDN, cloudGroup, 86400) - if not result["OK"]: - return result - return S_OK((cloudDN, cloudGroup)) - return S_ERROR("Cloud credentials not found") - - -def getVMTypes(siteList=None, ceList=None, vmTypeList=None, vo=None): - """Get CE/vmType options filtered by the provided parameters.""" - - result = gConfig.getSections("/Resources/Sites") - if not result["OK"]: - return result - - resultDict = {} - - grids = result["Value"] - for grid in grids: - result = gConfig.getSections("/Resources/Sites/%s" % grid) - if not result["OK"]: - continue - sites = result["Value"] - for site in sites: - if siteList is not None and site not in siteList: - continue - if vo: - voList = gConfig.getValue(f"/Resources/Sites/{grid}/{site}/VO", []) - if voList and vo not in voList: - continue - result = gConfig.getSections(f"/Resources/Sites/{grid}/{site}/Cloud") - if not result["OK"]: - continue - ces = result["Value"] - for ce in ces: - if ceList is not None and ce not in ceList: - continue - if vo: - voList = gConfig.getValue(f"/Resources/Sites/{grid}/{site}/Cloud/{ce}/VO", []) - if voList and vo not in voList: - continue - result = gConfig.getOptionsDict(f"/Resources/Sites/{grid}/{site}/Cloud/{ce}") - if not result["OK"]: - continue - ceOptionsDict = result["Value"] - result = gConfig.getSections(f"/Resources/Sites/{grid}/{site}/Cloud/{ce}/VMTypes") - if not result["OK"]: - result = gConfig.getSections(f"/Resources/Sites/{grid}/{site}/Cloud/{ce}/Images") - if not result["OK"]: - return result - vmTypes = result["Value"] - for vmType in vmTypes: - if vmTypeList is not None and vmType not in vmTypeList: - continue - if vo: - voList = gConfig.getValue(f"/Resources/Sites/{grid}/{site}/Cloud/{ce}/VMTypes/{vmType}/VO", []) - if not voList: - voList = gConfig.getValue( - f"/Resources/Sites/{grid}/{site}/Cloud/{ce}/Images/{vmType}/VO", [] - ) - if voList and vo not in voList: - continue - resultDict.setdefault(site, {}) - resultDict[site].setdefault(ce, ceOptionsDict) - resultDict[site][ce].setdefault("VMTypes", {}) - result = gConfig.getOptionsDict(f"/Resources/Sites/{grid}/{site}/Cloud/{ce}/VMTypes/{vmType}") - if not result["OK"]: - result = gConfig.getOptionsDict(f"/Resources/Sites/{grid}/{site}/Cloud/{ce}/Images/{vmType}") - if not result["OK"]: - continue - vmTypeOptionsDict = result["Value"] - resultDict[site][ce]["VMTypes"][vmType] = vmTypeOptionsDict - - return S_OK(resultDict) - - -def getVMTypeConfig(site, ce="", vmtype=""): - """Get the VM image type parameters of the specified queue""" - tags = [] - reqtags = [] - grid = site.split(".")[0] - if not ce: - result = gConfig.getSections(f"/Resources/Sites/{grid}/{site}/Cloud") - if not result["OK"]: - return result - ceList = result["Value"] - if len(ceList) == 1: - ce = ceList[0] - else: - return S_ERROR("No cloud endpoint specified") - - result = gConfig.getOptionsDict(f"/Resources/Sites/{grid}/{site}/Cloud/{ce}") - if not result["OK"]: - return result - resultDict = result["Value"] - ceTags = resultDict.get("Tag") - if ceTags: - tags = fromChar(ceTags) - ceTags = resultDict.get("RequiredTag") - if ceTags: - reqtags = fromChar(ceTags) - resultDict["CEName"] = ce - - if vmtype: - result = gConfig.getOptionsDict(f"/Resources/Sites/{grid}/{site}/Cloud/{ce}/VMTypes/{vmtype}") - if not result["OK"]: - return result - resultDict.update(result["Value"]) - queueTags = resultDict.get("Tag") - if queueTags: - queueTags = fromChar(queueTags) - tags = list(set(tags + queueTags)) - queueTags = resultDict.get("RequiredTag") - if queueTags: - queueTags = fromChar(queueTags) - reqtags = list(set(reqtags + queueTags)) - - if tags: - resultDict["Tag"] = tags - if reqtags: - resultDict["RequiredTag"] = reqtags - resultDict["VMType"] = vmtype - resultDict["Site"] = site - return S_OK(resultDict) - - -def getPilotBootstrapParameters(vo="", runningPod=""): - """Get all of the settings required to bootstrap a cloud instance.""" - op = Operations.Operations(vo=vo) - result = op.getOptionsDict("Cloud") - opParameters = {} - if result["OK"]: - opParameters = result["Value"] - opParameters["VO"] = vo - # FIXME: The majority of these settings can be removed once the old vm-pilot - # scripts have been removed. - opParameters["ReleaseProject"] = op.getValue("Cloud/ReleaseProject", "DIRAC") - opParameters["ReleaseVersion"] = op.getValue("Cloud/ReleaseVersion", op.getValue("Pilot/Version")) - opParameters["Setup"] = gConfig.getValue("/DIRAC/Setup", "unknown") - opParameters["SubmitPool"] = op.getValue("Cloud/SubmitPool") - opParameters["CloudPilotCert"] = op.getValue("Cloud/CloudPilotCert") - opParameters["CloudPilotKey"] = op.getValue("Cloud/CloudPilotKey") - opParameters["pilotFileServer"] = op.getValue("Pilot/pilotFileServer") - result = op.getOptionsDict("Cloud/%s" % runningPod) - if result["OK"]: - opParameters.update(result["Value"]) - - # Get standard pilot version now - if "Version" in opParameters: - gLogger.warn( - "Cloud bootstrap version now uses standard Pilot/Version setting. " - "Please remove all obsolete (Cloud/Version) setting(s)." - ) - pilotVersions = op.getValue("Pilot/Version") - if isinstance(pilotVersions, str): - pilotVersions = [pilotVersions] - if not pilotVersions: - return S_ERROR("Failed to get pilot version.") - opParameters["Version"] = pilotVersions[0].strip() - - return S_OK(opParameters) - - def _platformSortKey(version: str) -> list[str]: # Loosely based on distutils.version.LooseVersion parts = [] diff --git a/src/DIRAC/Core/Security/Properties.py b/src/DIRAC/Core/Security/Properties.py index 35496360ff4..a1d6006564f 100644 --- a/src/DIRAC/Core/Security/Properties.py +++ b/src/DIRAC/Core/Security/Properties.py @@ -56,8 +56,6 @@ class SecurityProperty(str, Enum): FC_MANAGEMENT = "FileCatalogManagement" #: Allow staging files STAGE_ALLOWED = "StageAllowed" - #: Allow VMDIRAC Operations via various handlers - VM_RPC_OPERATION = "VmRpcOperation" def __str__(self) -> str: return str(self.name) @@ -156,4 +154,3 @@ def __call__(self, properties): ALARMS_MANAGEMENT = SecurityProperty.ALARMS_MANAGEMENT.value FC_MANAGEMENT = SecurityProperty.FC_MANAGEMENT.value STAGE_ALLOWED = SecurityProperty.STAGE_ALLOWED.value -VM_RPC_OPERATION = SecurityProperty.VM_RPC_OPERATION.value diff --git a/src/DIRAC/Resources/Cloud/CloudEndpoint.py b/src/DIRAC/Resources/Cloud/CloudEndpoint.py deleted file mode 100644 index 5d6686b1aa6..00000000000 --- a/src/DIRAC/Resources/Cloud/CloudEndpoint.py +++ /dev/null @@ -1,549 +0,0 @@ -""" - CloudEndpoint is a base class for the clients used to connect to different - cloud providers -""" - -import os -import ssl -import time - -from libcloud import security -from libcloud.compute.types import Provider -from libcloud.compute.providers import get_driver -from libcloud.common.exceptions import BaseHTTPError - -# DIRAC -from DIRAC import gLogger, S_OK, S_ERROR -from DIRAC.Core.Utilities.File import makeGuid - -from DIRAC.Resources.Cloud.Endpoint import Endpoint -from DIRAC.Resources.Cloud.Utilities import STATE_MAP - - -class CloudEndpoint(Endpoint): - """CloudEndpoint base class""" - - def __init__(self, parameters=None): - super().__init__(parameters=parameters) - # logger - self.log = gLogger.getSubLogger(self.__class__.__name__) - self.valid = False - result = self.initialize() - if result["OK"]: - self.log.debug("CloudEndpoint created and validated") - self.valid = True - - def initialize(self): - - # Relax security - security.SSL_VERSION = ssl.PROTOCOL_SSLv23 - security.VERIFY_SSL_CERT = False - - # Variables needed to contact the service - connDict = {} - for var in [ - "ex_domain_name", - "ex_force_auth_url", - "ex_force_service_region", - "ex_force_auth_version", - "ex_tenant_name", - "ex_keyname", - "ex_voms_proxy", - ]: - if var in self.parameters: - connDict[var] = self.parameters[var] - - username = self.parameters.get("User") - password = self.parameters.get("Password") - - for key in connDict: - self.log.info(f"{key}: {connDict[key]}") - - # get cloud driver - providerName = self.parameters.get("Provider", "OPENSTACK").upper() - providerCode = getattr(Provider, providerName) - self.driverClass = get_driver(providerCode) - - self.__driver = self.driverClass(username, password, **connDict) - - return self.__checkConnection() - - def __checkConnection(self): - """ - Checks connection status by trying to list the images. - - :return: S_OK | S_ERROR - """ - try: - _result = self.__driver.list_images() - except Exception as errmsg: - return S_ERROR(errmsg) - - return S_OK() - - def __getImageByName(self, imageName): - """ - Given the imageName, returns the current image object from the server. - - :Parameters: - **imageName** - `string` - imageName as stored on the OpenStack image repository ( glance ) - - :return: S_OK( image ) | S_ERROR - """ - try: - images = self.__driver.list_images() - except Exception as errmsg: - return S_ERROR(errmsg) - - image = None - for im in images: - if im.name == imageName: - image = im - break - - if image is None: - return S_ERROR("Image %s not found" % imageName) - - return S_OK(image) - - def __getFlavorByName(self, flavorName): - """ - Given the flavorName, returns the current flavor object from the server. - - :Parameters: - **flavorName** - `string` - flavorName as stored on the OpenStack service - - :return: S_OK( flavor ) | S_ERROR - """ - try: - flavors = self.__driver.list_sizes() - except Exception as errmsg: - return S_ERROR(errmsg) - - flavor = None - for fl in flavors: - if fl.name == flavorName: - flavor = fl - - if flavor is None: - return S_ERROR("Flavor %s not found" % flavorName) - - return S_OK(flavor) - - def __getSecurityGroups(self, securityGroupNames=None): - """ - Given the securityGroupName, returns the current security group object from the server. - - :Parameters: - **securityGroupName** - `string` - securityGroupName as stored on the OpenStack service - - :return: S_OK( securityGroup ) | S_ERROR - """ - - if not securityGroupNames: - securityGroupNames = [] - elif not isinstance(securityGroupNames, list): - securityGroupNames = [securityGroupNames] - - if "default" not in securityGroupNames: - securityGroupNames.append("default") - - try: - secGroups = self.__driver.ex_list_security_groups() - except Exception as errmsg: - return S_ERROR(errmsg) - - return S_OK([secGroup for secGroup in secGroups if secGroup.name in securityGroupNames]) - - def createInstances(self, vmsToSubmit): - outputDict = {} - - for nvm in range(vmsToSubmit): - instanceID = makeGuid()[:8] - createPublicIP = "ipPool" in self.parameters - result = self.createInstance(instanceID, createPublicIP) - if result["OK"]: - node, publicIP = result["Value"] - self.log.debug(f"Created VM instance {node.id}/{instanceID} with publicIP {publicIP}") - nodeDict = {} - nodeDict["PublicIP"] = publicIP - nodeDict["InstanceID"] = instanceID - nodeDict["NumberOfProcessors"] = self.flavor.vcpus - nodeDict["RAM"] = self.flavor.ram - nodeDict["DiskSize"] = self.flavor.disk - nodeDict["Price"] = self.flavor.price - outputDict[node.id] = nodeDict - else: - break - - if not outputDict: - # Submission failed - return result - - return S_OK(outputDict) - - def createInstance(self, instanceID="", createPublicIP=True): - """ - This creates a VM instance for the given boot image - and creates a context script, taken the given parameters. - Successful creation returns instance VM - - Boots a new node on the OpenStack server defined by self.endpointConfig. The - 'personality' of the node is done by self.imageConfig. Both variables are - defined on initialization phase. - - The node name has the following format: -