Merge pull request #8291 from alex-rg/add_remoteonly_access_option

[9.1] support "remoteonly" value for the Access protocol parameter

Author: fstagni

docs/source/AdministratorGuide/Resources/storage.rst

@@ -20,7 +20,7 @@ DIRAC provides an abstraction of a SE interface that allows to access different
         # The name of the DIRAC Plugin module to be used for implementation
         # of the access protocol
         PluginName = SRM2
-        # Flag specifying the access type (local/remote)
+        # Flag specifying the access type (local/remote/remoteonly)
         Access = remote
         # Protocol name
         Protocol = srm
@@ -50,6 +50,7 @@ Configuration options are:
 * ``UseCatalogURL``: default ``False``. If ``True``, use the url stored in the catalog instead of regenerating it
 * ``ChecksumType``: default ``ADLER32``. NOT ACTIVE !
 * ``Alias``: when set to the name of another storage element, it instanciates the other SE instead.
+* ``Access``: Can be ``local``, ``remote`` or ``remoteonly``. Options specify that the protocol can be used in local (e.g. upload from a WN to local SE), remote+local or remote context.
 * ``ReadAccess``: default ``True``. Allowed for Read if no RSS enabled (:ref:`activateRSS`)
 * ``WriteAccess``: default ``True``. Allowed for Write if no RSS enabled
 * ``CheckAccess``: default ``True``. Allowed for Check if no RSS enabled
@@ -221,7 +222,7 @@ There are also a set of plugins based on the `gfal2 libraries <https://dmc-docs.
 
 Default plugin options:
 
-* ``Access``: ``Remote`` or ``Local``. If ``Local``, then this protocol can be used only if we are running at the site to which the SE is associated. Typically, if a site mounts the storage as NFS, the ``file`` protocol can be used.
+* ``Access``: ``Remote``, ``Local`` or ``RemoteOnly``. If ``Local``, then this protocol can be used only if we are running at the site to which the SE is associated. Typically, if a site mounts the storage as NFS, the ``file`` protocol can be used. If ``RemoteOnly``, this protocol is not used when running at the site. For example, if you need to use a different hostname within the site than from outside.
 * InputProtocols/OutputProtocols: a given plugin normally contain a hard coded list of protocol it is able to generate or accept as input. There are however seldom cases (like SRM) where the site configuration may change these lists. These options are here to accomodate for that case.
 
 GRIDFTP Optimisation
@@ -317,6 +318,32 @@ You need to define a protocol section with SRM, specifying that a ``file`` URL c
     }
 
 
+RemoteOnly protocol definition
+------------------------------
+
+Some sites may want you to have a different endpoint when running on the worker node than when transfering files via FTS. Or they recommend a protocol remotely only. This is possible with the ``RemoteOnly`` only option::
+
+
+    HTTPsConfig
+    {
+      Host = webdav.fromoutside.ac.uk
+      Port = 1094
+      PluginName = Echo
+      Protocol = https
+      Path = lhcb:mydata
+      Access = remoteonly
+    }
+    HTTPsConfigLocal
+    {
+      Host = inner-webdav-gateway..ac.uk
+      Port = 1095
+      PluginName = Echo
+      Protocol = https
+      Path = lhcb:mydata
+      Access = local
+    }
+
+
 
 .. _multiProtocol:
 

src/DIRAC/Resources/Storage/StorageElement.py

@@ -1092,7 +1092,6 @@ def __filterPlugins(self, methodName, protocols=None, inputProtocol=None):
         Returns:
           list: list of storage plugins
         """
-
         log = self.log.getSubLogger("__filterPlugins")
         log.debug(f"Filtering plugins for {methodName} (protocol = {protocols} ; inputProtocol = {inputProtocol})")
 
@@ -1104,6 +1103,8 @@ def __filterPlugins(self, methodName, protocols=None, inputProtocol=None):
         potentialProtocols = []
         allowedProtocols = []
 
+        localSE = self.__isLocalSE()["Value"]
+
         if methodName in self.readMethods + self.checkMethods:
             allowedProtocols = self.localAccessProtocolList
         elif methodName in self.stageMethods:
@@ -1125,12 +1126,12 @@ def __filterPlugins(self, methodName, protocols=None, inputProtocol=None):
                 pluginsToUse = list(self.storages.values())
 
             # The closest list for "OK" methods is the AccessProtocol preference, so we sort based on that
-
+            # note: False < True, so to have local plugin first IF we are in a localSE, we use the remoteProtocol list
             pluginsToUse = sorted(
                 pluginsToUse,
                 key=lambda x: (
                     getIndexInList(x.protocolParameters["Protocol"], self.localAccessProtocolList),
-                    x.protocolSectionName in self.remoteProtocolSections,
+                    x.protocolSectionName in (self.remoteProtocolSections if localSE else self.localAccessProtocolList),
                 ),
             )
 
@@ -1147,12 +1148,11 @@ def __filterPlugins(self, methodName, protocols=None, inputProtocol=None):
 
         log.debug(f"Potential protocols {potentialProtocols}")
 
-        localSE = self.__isLocalSE()["Value"]
-
         for protocolSection, plugin in self.storages.items():
             # Determine whether to use this storage object
             pluginParameters = plugin.getParameters()
             isProxyPlugin = pluginParameters.get("PluginName") == "Proxy"
+            isRemoteOnly = pluginParameters.get("Access", "").lower() == "remoteonly"
 
             if not pluginParameters:
                 log.debug("Failed to get storage parameters.", f"{self.name} {protocolSection}")
@@ -1167,6 +1167,10 @@ def __filterPlugins(self, methodName, protocols=None, inputProtocol=None):
                 log.debug(f"Plugin {protocolSection} not allowed for {methodName}.")
                 continue
 
+            # If protocol is remote only and the context is local, skip it.
+            if isRemoteOnly and localSE:
+                continue
+
             # If we are attempting a putFile and we know the inputProtocol
             if methodName == "putFile" and inputProtocol:
                 if inputProtocol not in pluginParameters["InputProtocols"]:
@@ -1188,7 +1192,6 @@ def __filterPlugins(self, methodName, protocols=None, inputProtocol=None):
         )
 
         log.debug(f"Plugins to be used for {methodName}: {[p.protocolSectionName for p in pluginsToUse]}")
-
         return pluginsToUse
 
     def __executeMethod(self, lfn, *args, **kwargs):

src/DIRAC/Resources/Storage/StorageFactory.py

@@ -333,14 +333,14 @@ def _getConfigStorageProtocols(self, storageName, derivedStorageName=None, seCon
         for protocolSectionName, protocolDict in self.protocols.items():
             # Now update the local and remote protocol lists.
             # A warning will be given if the Access option is not set to local or remote.
-            if protocolDict["Access"].lower() == "remote":
+            if protocolDict["Access"].lower() in ("remote", "remoteonly"):
                 self.remoteProtocolSections.append(protocolSectionName)
             elif protocolDict["Access"].lower() == "local":
                 self.localProtocolSections.append(protocolSectionName)
             else:
                 errStr = (
                     "StorageFactory.__getProtocolDetails: The 'Access' option \
-        for %s:%s is neither 'local' or 'remote'."
+        for %s:%s is not 'local', 'remote' or 'remoteonly'."
                     % (storageName, protocolSectionName)
                 )
                 gLogger.warn(errStr)