feat: Add PreferredURLPattern for URL sorting

Author: chrisburr

docs/source/AdministratorGuide/Configuration/ConfReference/Systems/index.rst

@@ -34,6 +34,39 @@ In each system, per setup, you normally find the following sections:
 * FailoverURLs: Like URLs, but they are only tried if no server in URLs was successfully contacted.
 
 
+Preferred URLs
+--------------
+
+For most services, the standard ``URLs`` and ``FailoverURLs`` mechanism provides a way to specify primary and backup service endpoints.
+
+However, this approach has limitations in certain scenarios:
+
+- Some services (like the Configuration service) have replicas that automatically register themselves in the Configuration System
+- External servers ("voboxes") running at sites may not be accessible from all clients
+- Connection attempts to inaccessible servers cause errors that, while harmless due to fallback mechanisms, slow down DIRAC and generate misleading error messages
+
+To address these issues, you can define a ``PreferredURLPattern`` that identifies a subset of URLs to try first:
+
+.. code-block:: guess
+
+  System
+  {
+    PreferredURLPattern = .*\.main\.invalid/.*
+    URLs
+    {
+      Service = dips://host1.main.invalid:1234/System/Service,dips://host2.main.invalid:1234/System/Service,dips://external.invalid:1234/System/Service
+    }
+  }
+
+In this example:
+
+1. The ``PreferredURLPattern`` specifies a regular expression that matches servers in the ``main.invalid`` domain
+2. When connecting to the service, DIRAC will first try URLs matching this pattern (``host1.main.invalid`` and ``host2.main.invalid``)
+3. Only if these preferred servers fail will DIRAC attempt to connect to other servers (``external.invalid``)
+
+This approach reduces connection errors and improves performance by prioritizing servers that are more likely to be accessible from the client.
+
+
 Main Servers
 ------------
 

src/DIRAC/ConfigurationSystem/Client/PathFinder.py

@@ -1,5 +1,7 @@
 """ Collection of utilities for finding paths in the CS
 """
+import re
+from functools import lru_cache
 from urllib import parse
 
 from DIRAC.Core.Utilities import List
@@ -207,6 +209,22 @@ def getSystemURLs(system, setup=False, failover=False):
     return urlDict
 
 
+@lru_cache(maxsize=1024)
+def checkPreferredURL(url, preferredURLPattern):
+    """Check if the URL matches the preferred URL pattern.
+
+    Default is True if no preferred URL pattern is set.
+
+    :param str url: URL to check
+    :param str preferredURLPattern: preferred URL pattern
+
+    :return: bool -- True if the URL matches the preferred URL pattern
+    """
+    if not preferredURLPattern:
+        return True
+    return re.match(preferredURLPattern, url)
+
+
 def getServiceURLs(system, service=None, setup=False, failover=False):
     """Generate url.
 
@@ -225,8 +243,10 @@ def getServiceURLs(system, service=None, setup=False, failover=False):
     # Add failover URLs at the end of the list
     failover = "Failover" if failover else ""
     for fURLs in ["", "Failover"] if failover else [""]:
-        urlList = []
+        preferredURLs = set()
+        otherURLs = set()
         urls = List.fromChar(gConfigurationData.extractOptionFromCFG(f"{systemSection}/{fURLs}URLs/{service}"))
+        preferredURLPattern = gConfigurationData.extractOptionFromCFG(f"{systemSection}/PreferredURLPattern")
 
         # Be sure that urls not None
         for url in urls or []:
@@ -243,16 +263,19 @@ def getServiceURLs(system, service=None, setup=False, failover=False):
 
                 for srv in mainServers:
                     _url = checkComponentURL(url.replace("$MAINSERVERS$", srv), system, service, pathMandatory=True)
-                    if _url not in urlList:
-                        urlList.append(_url)
+                    preferredURLs.add(_url)
                 continue
 
             _url = checkComponentURL(url, system, service, pathMandatory=True)
-            if _url not in urlList:
-                urlList.append(_url)
+            # Don't apply the preferred URL pattern to failover URLs
+            if failover or checkPreferredURL(_url, preferredURLPattern):
+                preferredURLs.add(_url)
+            else:
+                otherURLs.add(_url)
 
         # Randomize list if needed
-        resList.extend(List.randomize(urlList))
+        resList.extend(List.randomize(preferredURLs))
+        resList.extend(List.randomize(otherURLs))
 
     return resList
 

src/DIRAC/ConfigurationSystem/Client/test/Test_PathFinder.py

@@ -11,20 +11,36 @@
 localCFGData = ConfigurationData(False)
 mergedCFG = CFG()
 mergedCFG.loadFromBuffer(
-    """
+    r"""
 DIRAC
 {
   Setup=TestSetup
   Setups
   {
     TestSetup
     {
+      Configuration=MyCS
       WorkloadManagement=MyWM
     }
   }
 }
 Systems
 {
+  Configuration
+  {
+    MyCS
+    {
+      PreferredURLPattern = dips://.*\.site:.*
+      URLs
+      {
+        Server = dips://server1.site:1234/Configuration/Server
+        Server += dips://server2.site:1234/Configuration/Server
+        Server += dips://server3.site:1234/Configuration/Server
+        Server += dips://server4.site:1234/Configuration/Server
+        Server += dips://server4.external:1234/Configuration/Server
+      }
+    }
+  }
   WorkloadManagement
   {
     MyWM
@@ -204,6 +220,26 @@ def test_getServiceURLs(pathFinder, serviceName, service, failover, result):
     assert set(pathFinder.getServiceURLs(serviceName, service=service, failover=failover)) == result
 
 
+def test_getServiceURLsOrdering(pathFinder):
+    """Ensure the PreferredURLPattern option is respected"""
+    all_results = set()
+    for _ in range(10_000):
+        urls = pathFinder.getServiceURLs("Configuration", service="Server")
+        assert set(urls) == {
+            "dips://server1.site:1234/Configuration/Server",
+            "dips://server2.site:1234/Configuration/Server",
+            "dips://server3.site:1234/Configuration/Server",
+            "dips://server4.site:1234/Configuration/Server",
+            "dips://server4.external:1234/Configuration/Server",
+        }
+        # The last URL should always be the one which isn't preferred
+        assert urls[-1] == "dips://server4.external:1234/Configuration/Server"
+        all_results.add(tuple(urls))
+    # There are 4! = 24 possible orderings of the preferred URLs, we should have seen all
+    # of them at least once in 10_000 iterations
+    assert len(all_results) >= 24
+
+
 @pytest.mark.parametrize(
     "system, setup, failover, result",
     [

src/DIRAC/Core/Utilities/List.py

@@ -3,7 +3,10 @@
 """
 import random
 import sys
-from typing import Any
+from typing import Any, TypeVar
+from collections.abc import Iterable
+
+T = TypeVar("T")
 
 
 def uniqueElements(aList: list) -> list:
@@ -37,7 +40,7 @@ def fromChar(inputString: str, sepChar: str = ","):
     return [fieldString.strip() for fieldString in inputString.split(sepChar) if len(fieldString.strip()) > 0]
 
 
-def randomize(aList: list) -> list:
+def randomize(aList: Iterable[T]) -> list[T]:
     """Return a randomly sorted list.
 
     :param aList: list to permute