Merge pull request #8153 from aldbr/v9.0_REMOVE_PilotBundle

[9.0] remove: PilotBundle

Author: chrisburr

src/DIRAC/Resources/Computing/AREXComputingElement.py

@@ -48,6 +48,7 @@
 import os
 import shutil
 import stat
+import tempfile
 import uuid
 
 import requests
@@ -57,7 +58,6 @@
 from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup
 from DIRAC.Core.Security.X509Chain import X509Chain  # pylint: disable=import-error
 from DIRAC.Resources.Computing.ComputingElement import ComputingElement
-from DIRAC.Resources.Computing.PilotBundle import writeScript
 from DIRAC.WorkloadManagementSystem.Client import PilotStatus
 
 MANDATORY_PARAMETERS = ["Queue"]
@@ -495,7 +495,11 @@ def _bundlePreamble(self, executableFile):
         if not os.access(executableFile, os.X_OK):
             os.chmod(executableFile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH + stat.S_IXOTH)
 
-        return writeScript(wrapperContent, os.getcwd())
+        with tempfile.NamedTemporaryFile(
+            "w", delete=False, prefix="preamble-", suffix=f"-{executableFile}"
+        ) as bundleFile:
+            bundleFile.write(wrapperContent)
+            return bundleFile.name
 
     def _getArcJobID(self, executableFile, inputs, outputs, delegation):
         """Get an ARC JobID endpoint to upload executables and inputs.

src/DIRAC/Resources/Computing/LocalComputingElement.py

@@ -43,7 +43,6 @@
 from DIRAC import S_OK, S_ERROR
 
 from DIRAC.Resources.Computing.ComputingElement import ComputingElement
-from DIRAC.Resources.Computing.PilotBundle import bundleProxy, writeScript
 from DIRAC.Core.Utilities.List import uniqueElements
 from DIRAC.Core.Utilities.Subprocess import systemCall
 
@@ -153,26 +152,12 @@ def submitJob(self, executableFile, proxy=None, numberOfJobs=1):
         if not os.access(executableFile, 5):
             os.chmod(executableFile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
 
-        # if no proxy is supplied, the executable can be submitted directly
-        # otherwise a wrapper script is needed to get the proxy to the execution node
-        # The wrapper script makes debugging more complicated and thus it is
-        # recommended to transfer a proxy inside the executable if possible.
-        if self.proxy and not proxy:
-            proxy = self.proxy
-        if proxy:
-            self.log.verbose("Setting up proxy for payload")
-            wrapperContent = bundleProxy(executableFile, proxy)
-            name = writeScript(wrapperContent, os.getcwd())
-            submitFile = name
-        else:  # no proxy
-            submitFile = executableFile
-
         jobStamps = []
         for _i in range(numberOfJobs):
             jobStamps.append(uuid.uuid4().hex)
 
         batchDict = {
-            "Executable": submitFile,
+            "Executable": executableFile,
             "NJobs": numberOfJobs,
             "OutputDir": self.batchOutput,
             "ErrorDir": self.batchError,
@@ -186,8 +171,6 @@ def submitJob(self, executableFile, proxy=None, numberOfJobs=1):
             "NumberOfGPUs": self.numberOfGPUs,
         }
         resultSubmit = self.batchSystem.submitJob(**batchDict)
-        if proxy:
-            os.remove(submitFile)
 
         if resultSubmit["Status"] == 0:
             self.submittedJobs += len(resultSubmit["Jobs"])

src/DIRAC/Resources/Computing/PilotBundle.py

@@ -7,7 +7,6 @@
 from urllib.parse import urlparse
 
 from DIRAC import S_ERROR, S_OK
-from DIRAC.Resources.Computing.PilotBundle import bundleProxy, writeScript
 from DIRAC.Resources.Computing.SSHComputingElement import SSHComputingElement
 from DIRAC.WorkloadManagementSystem.Client import PilotStatus
 
@@ -80,18 +79,6 @@ def submitJob(self, executableFile, proxy, numberOfJobs=1):
         if not os.access(executableFile, 5):
             os.chmod(executableFile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
 
-        # if no proxy is supplied, the executable can be submitted directly
-        # otherwise a wrapper script is needed to get the proxy to the execution node
-        # The wrapper script makes debugging more complicated and thus it is
-        # recommended to transfer a proxy inside the executable if possible.
-        if proxy:
-            self.log.verbose("Setting up proxy for payload")
-            wrapperContent = bundleProxy(executableFile, proxy)
-            name = writeScript(wrapperContent, os.getcwd())
-            submitFile = name
-        else:  # no proxy
-            submitFile = executableFile
-
         # Submit jobs now
         restJobs = numberOfJobs
         submittedJobs = []
@@ -100,7 +87,7 @@ def submitJob(self, executableFile, proxy, numberOfJobs=1):
             if slots not in rankHosts:
                 continue
             for host in rankHosts[slots]:
-                result = self._submitJobToHost(submitFile, min(slots, restJobs), host)
+                result = self._submitJobToHost(executableFile, min(slots, restJobs), host)
                 if not result["OK"]:
                     continue
 
@@ -114,9 +101,6 @@ def submitJob(self, executableFile, proxy, numberOfJobs=1):
             if restJobs <= 0:
                 break
 
-        if proxy:
-            os.remove(submitFile)
-
         result = S_OK(submittedJobs)
         result["PilotStampDict"] = stampDict
         return result

src/DIRAC/Resources/Computing/SSHBatchComputingElement.py

@@ -78,7 +78,6 @@
 from DIRAC.Core.Utilities.List import breakListIntoChunks, uniqueElements
 from DIRAC.Resources.Computing.BatchSystems.executeBatch import executeBatchContent
 from DIRAC.Resources.Computing.ComputingElement import ComputingElement
-from DIRAC.Resources.Computing.PilotBundle import bundleProxy, writeScript
 
 
 class SSH:
@@ -532,23 +531,7 @@ def submitJob(self, executableFile, proxy, numberOfJobs=1):
         if not os.access(executableFile, 5):
             os.chmod(executableFile, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
 
-        # if no proxy is supplied, the executable can be submitted directly
-        # otherwise a wrapper script is needed to get the proxy to the execution node
-        # The wrapper script makes debugging more complicated and thus it is
-        # recommended to transfer a proxy inside the executable if possible.
-        if proxy:
-            self.log.verbose("Setting up proxy for payload")
-            wrapperContent = bundleProxy(executableFile, proxy)
-            name = writeScript(wrapperContent, os.getcwd())
-            submitFile = name
-        else:  # no proxy
-            submitFile = executableFile
-
-        result = self._submitJobToHost(submitFile, numberOfJobs)
-        if proxy:
-            os.remove(submitFile)
-
-        return result
+        return self._submitJobToHost(executableFile, numberOfJobs)
 
     def _submitJobToHost(self, executableFile, numberOfJobs, host=None):
         """Submit prepared executable to the given host"""