change: Use PreferredURLPatterns instead of PreferredURLPattern

Author: chrisburr

docs/source/AdministratorGuide/Configuration/ConfReference/Systems/index.rst

@@ -45,27 +45,32 @@ However, this approach has limitations in certain scenarios:
 - External servers ("voboxes") running at sites may not be accessible from all clients
 - Connection attempts to inaccessible servers cause errors that, while harmless due to fallback mechanisms, slow down DIRAC and generate misleading error messages
 
-To address these issues, you can define a ``PreferredURLPattern`` that identifies a subset of URLs to try first:
+To address these issues, you can define a ``PreferredURLPatterns`` that identifies a subset of URLs to try first:
 
 .. code-block:: guess
 
   System
   {
-    PreferredURLPattern = .*\.main\.invalid/.*
     URLs
     {
       Service = dips://host1.main.invalid:1234/System/Service,dips://host2.main.invalid:1234/System/Service,dips://external.invalid:1234/System/Service
     }
   }
+  DIRAC
+  {
+    PreferredURLPatterns = .*\.main\.invalid/.*
+  }
 
 In this example:
 
-1. The ``PreferredURLPattern`` specifies a regular expression that matches servers in the ``main.invalid`` domain
+1. The ``PreferredURLPatterns`` specifies a regular expression that matches servers in the ``main.invalid`` domain
 2. When connecting to the service, DIRAC will first try URLs matching this pattern (``host1.main.invalid`` and ``host2.main.invalid``)
 3. Only if these preferred servers fail will DIRAC attempt to connect to other servers (``external.invalid``)
 
 This approach reduces connection errors and improves performance by prioritizing servers that are more likely to be accessible from the client.
 
+.. note:: The ``PreferredURLPatterns`` is a list of regular expressions, not a single regular expression. This allows you to specify multiple patterns to match different subsets of servers if desired.
+
 
 Main Servers
 ------------

src/DIRAC/ConfigurationSystem/Client/PathFinder.py

@@ -1,9 +1,12 @@
 """ Collection of utilities for finding paths in the CS
 """
 import re
-from functools import lru_cache
+from copy import deepcopy
+from collections.abc import Iterable
 from urllib import parse
 
+from cachetools import cached, TTLCache
+
 from DIRAC.Core.Utilities import List
 from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
 from DIRAC.ConfigurationSystem.Client.Helpers import Path
@@ -209,20 +212,41 @@ def getSystemURLs(system, setup=False, failover=False):
     return urlDict
 
 
-@lru_cache(maxsize=1024)
-def checkPreferredURL(url, preferredURLPattern):
-    """Check if the URL matches the preferred URL pattern.
+def groupURLsByPriority(urls: Iterable[str]) -> list[set[str]]:
+    """Group URLs by priority.
+
+    :param Iterable[str] preferredURLPatterns: patterns to check in ranked order
+    :param set[str] urls: URLs to check
+
+    :return: list[set[str]] -- list of URL groups, ordered by priority
+    """
+    return deepcopy(_groupURLsByPriority(frozenset(urls)))
+
+
+@cached(cache=TTLCache(maxsize=1024, ttl=300))
+def _groupURLsByPriority(urls: frozenset[str]) -> list[set[str]]:
+    preferredURLPatterns = [
+        re.compile(pattern)
+        for pattern in List.fromChar(gConfigurationData.extractOptionFromCFG("/DIRAC/PreferredURLPatterns"))
+    ]
+    urlGroups = [set() for _ in range(len(preferredURLPatterns) + 1)]
+    for url in urls:
+        urlGroups[findURLPriority(preferredURLPatterns, url)].add(url)
+    return urlGroups
+
 
-    Default is True if no preferred URL pattern is set.
+def findURLPriority(preferredURLPatterns: list[re.Pattern[str]], url: str) -> int:
+    """Find which preferred URL pattern the URL matches.
 
+    :param str preferredURLPatterns: patterns to check in ranked order
     :param str url: URL to check
-    :param str preferredURLPattern: preferred URL pattern
 
-    :return: bool -- True if the URL matches the preferred URL pattern
+    :return: int -- index of the pattern that matched, smallest is the most preferred
     """
-    if not preferredURLPattern:
-        return True
-    return re.match(preferredURLPattern, url)
+    for i, pattern in enumerate(preferredURLPatterns):
+        if re.match(pattern, url):
+            return i
+    return len(preferredURLPatterns)
 
 
 def getServiceURLs(system, service=None, setup=False, failover=False):
@@ -243,10 +267,8 @@ def getServiceURLs(system, service=None, setup=False, failover=False):
     # Add failover URLs at the end of the list
     failover = "Failover" if failover else ""
     for fURLs in ["", "Failover"] if failover else [""]:
-        preferredURLs = set()
-        otherURLs = set()
         urls = List.fromChar(gConfigurationData.extractOptionFromCFG(f"{systemSection}/{fURLs}URLs/{service}"))
-        preferredURLPattern = gConfigurationData.extractOptionFromCFG(f"{systemSection}/PreferredURLPattern")
+        urlList = set()
 
         # Be sure that urls not None
         for url in urls or []:
@@ -263,19 +285,13 @@ def getServiceURLs(system, service=None, setup=False, failover=False):
 
                 for srv in mainServers:
                     _url = checkComponentURL(url.replace("$MAINSERVERS$", srv), system, service, pathMandatory=True)
-                    preferredURLs.add(_url)
+                    urlList.add(_url)
                 continue
 
-            _url = checkComponentURL(url, system, service, pathMandatory=True)
-            # Don't apply the preferred URL pattern to failover URLs
-            if failover or checkPreferredURL(_url, preferredURLPattern):
-                preferredURLs.add(_url)
-            else:
-                otherURLs.add(_url)
+            urlList.add(checkComponentURL(url, system, service, pathMandatory=True))
 
-        # Randomize list if needed
-        resList.extend(List.randomize(preferredURLs))
-        resList.extend(List.randomize(otherURLs))
+        for urlGroup in groupURLsByPriority(urlList):
+            resList.extend(List.randomize(urlGroup))
 
     return resList
 

src/DIRAC/ConfigurationSystem/Client/test/Test_PathFinder.py

@@ -23,21 +23,23 @@
       WorkloadManagement=MyWM
     }
   }
+  PreferredURLPatterns = dips://.*\.site:.*
+  PreferredURLPatterns += dips://.*\.other:.*
 }
 Systems
 {
   Configuration
   {
     MyCS
     {
-      PreferredURLPattern = dips://.*\.site:.*
       URLs
       {
         Server = dips://server1.site:1234/Configuration/Server
         Server += dips://server2.site:1234/Configuration/Server
         Server += dips://server3.site:1234/Configuration/Server
         Server += dips://server4.site:1234/Configuration/Server
-        Server += dips://server4.external:1234/Configuration/Server
+        Server += dips://server.other:1234/Configuration/Server
+        Server += dips://server.external:1234/Configuration/Server
       }
     }
   }
@@ -230,10 +232,13 @@ def test_getServiceURLsOrdering(pathFinder):
             "dips://server2.site:1234/Configuration/Server",
             "dips://server3.site:1234/Configuration/Server",
             "dips://server4.site:1234/Configuration/Server",
-            "dips://server4.external:1234/Configuration/Server",
+            "dips://server.other:1234/Configuration/Server",
+            "dips://server.external:1234/Configuration/Server",
         }
+        # The second to last URL should always be "other"
+        assert urls[-2] == "dips://server.other:1234/Configuration/Server"
         # The last URL should always be the one which isn't preferred
-        assert urls[-1] == "dips://server4.external:1234/Configuration/Server"
+        assert urls[-1] == "dips://server.external:1234/Configuration/Server"
         all_results.add(tuple(urls))
     # There are 4! = 24 possible orderings of the preferred URLs, we should have seen all
     # of them at least once in 10_000 iterations