Merge pull request #8550 from fstagni/v9_readme

[rel-v9r0] branch bootstrap

Author: fstagni

.github/workflows/basic.yml

@@ -171,14 +171,26 @@ jobs:
           path: diracx
       - uses: prefix-dev/setup-pixi@v0.9.3
         with:
-          run-install: false
+          cache: false
           post-cleanup: false
+          manifest-path: diracx/pixi.toml
+          environments: >-
+            diracx-core
+            diracx-db
+            diracx-logic
+            diracx-tasks
+            diracx-routers
+            diracx-client
+            diracx-api
+            diracx-cli
       - name: Apply workarounds
         run: |
           cd diracx
           # Workaround for https://github.com/prefix-dev/pixi/issues/3762
           sed -i.bak 's@editable = true@editable = false@g' pixi.toml
           rm pixi.toml.bak
+          sed -i.bak '/solve-group = "gubbins"/d' pixi.toml
+          rm pixi.toml.bak
           # Add annotations to github actions
           pixi add --pypi --feature diracx-core pytest-github-actions-annotate-failures
           # Add the current DIRAC clone to the pixi.toml
@@ -189,11 +201,13 @@ jobs:
       - uses: prefix-dev/setup-pixi@v0.9.3
         with:
           cache: false
+          locked: false
           manifest-path: diracx/pixi.toml
           environments: >-
             diracx-core
             diracx-db
             diracx-logic
+            diracx-tasks
             diracx-routers
             diracx-client
             diracx-api

README.rst

@@ -16,28 +16,28 @@ DIRAC provides a complete solution to one or more user community requiring acces
 
 DIRAC has been started by the `LHCb collaboration <https://lhcb.web.cern.ch/lhcb/>`_ who still maintains it. It is now used by several communities (AKA VO=Virtual Organizations) for their distributed computing workflows.
 
-DIRAC is written in python 3.9.
+DIRAC is written in python 3.11.
 
-Status rel-v8r0 series (stable, recommended):
+Status rel-v9r0 series (stable, recommended):
 
-.. image:: https://github.com/DIRACGrid/DIRAC/workflows/Basic%20tests/badge.svg?branch=rel-v8r0
-   :target: https://github.com/DIRACGrid/DIRAC/actions?query=workflow%3A%22Basic+tests%22+branch%3Arel-v8r0
+.. image:: https://github.com/DIRACGrid/DIRAC/workflows/Basic%20tests/badge.svg?branch=rel-v9r0
+   :target: https://github.com/DIRACGrid/DIRAC/actions?query=workflow%3A%22Basic+tests%22+branch%3Arel-v9r0
    :alt: Basic Tests Status
 
-.. image:: https://github.com/DIRACGrid/DIRAC/workflows/pilot%20wrapper/badge.svg?branch=rel-v8r0
-   :target: https://github.com/DIRACGrid/DIRAC/actions?query=workflow%3A%22pilot+wrapper%22+branch%3Arel-v8r0
+.. image:: https://github.com/DIRACGrid/DIRAC/workflows/pilot%20wrapper/badge.svg?branch=rel-v9r0
+   :target: https://github.com/DIRACGrid/DIRAC/actions?query=workflow%3A%22pilot+wrapper%22+branch%3Arel-v9r0
    :alt: Pilot Wrapper Status
 
-.. image:: https://github.com/DIRACGrid/DIRAC/workflows/Integration%20tests/badge.svg?branch=rel-v8r0
-   :target: https://github.com/DIRACGrid/DIRAC/actions?query=workflow%3A%22Integration+tests%22+branch%3Arel-v8r0
+.. image:: https://github.com/DIRACGrid/DIRAC/workflows/Integration%20tests/badge.svg?branch=rel-v9r0
+   :target: https://github.com/DIRACGrid/DIRAC/actions?query=workflow%3A%22Integration+tests%22+branch%3Arel-v9r0
    :alt: Integration Tests Status
 
-.. image:: https://readthedocs.org/projects/dirac/badge/?version=rel-v8r0
-   :target: http://dirac.readthedocs.io/en/rel-v8r0/
+.. image:: https://readthedocs.org/projects/dirac/badge/?version=rel-v9r0
+   :target: http://dirac.readthedocs.io/en/rel-v9r0/
    :alt: Documentation Status
 
 
-Status integration branch (devel):
+Status integration branch:
 
 .. image:: https://github.com/DIRACGrid/DIRAC/workflows/Basic%20tests/badge.svg?branch=integration
    :target: https://github.com/DIRACGrid/DIRAC/actions?query=workflow%3A%22Basic+tests%22+branch%3Aintegration
@@ -124,7 +124,7 @@ Code quality
 ~~~~~~~~~~~~
 
 To ensure the code meets DIRAC's coding conventions we recommend installing ``pre-commit`` system wide using your operating system's package manager.
-Alteratively, ``pre-commit`` is included in the Python 3 development environment, see the `development guide <https://dirac.readthedocs.io/en/integration/DeveloperGuide/DevelopmentEnvironment/DeveloperInstallation/editingCode.html>`_ for details on how to create one.
+Alteratively, ``pre-commit`` is included in the development environment, see the `development guide <https://dirac.readthedocs.io/en/integration/DeveloperGuide/DevelopmentEnvironment/DeveloperInstallation/editingCode.html>`_ for details on how to create one.
 
 Once ``pre-commit`` is installed you can enable it by running:
 

docs/docs.conf

@@ -31,6 +31,9 @@ no_inherited_members =
   DIRAC.Core.Utilities.Graphs.GraphUtilities,
   DIRAC.DataManagementSystem.private.HttpStorageAccessHandler,
   DIRAC.FrameworkSystem.private.standardLogging.LogLevels,
+  DIRAC.Resources.IdProvider.OAuth2IdProvider,
+  DIRAC.Resources.IdProvider.CheckInIdProvider,
+  DIRAC.Resources.IdProvider.IAMIdProvider,
 
 # only creating dummy files, because they cannot be safely imported due to sideEffects
 create_dummy_files = lfc_dfc_copy, lfc_dfc_db_copy, JobWrapperTemplate, JobWrapperOfflineTemplate

docs/source/DeveloperGuide/CodeTesting/index.rst

@@ -360,13 +360,11 @@ To deactivate a service from being used with DiracX, you can add it in `integrat
       "WorkloadManagement/JobMonitoring",
     ]
 
-By setting `TEST_DIRACX=Yes` only, it will take the last version of DiracX by default. If you want to provide your own, you have to build your DiracX project, and provide the `dist` folder path when calling `prepare-client`. This path has to be absolute.
+By setting `TEST_DIRACX=Yes` only, it will take the last version of DiracX by default. If you want to test against a local DiracX checkout, provide the source directory path when calling `prepare-environment`. This will build the DiracX container images from source and build wheels for the DIRAC containers.
 
 .. code-block:: bash
 
-    ./integration-tests.py prepare-client TEST_DIRACX=Yes --diracx-dist-dir my-dist-folder/
-
-It will then mount your dist folder into DIRAC and DiracX (in `/diracx_sources`) to install the right dependencies.
+    ./integration-tests.py prepare-environment TEST_DIRACX=Yes --diracx-src-dir /path/to/diracx/
 
 For MacOS, there are two bugs that can be fixed.
 

integration_tests.py

@@ -175,14 +175,14 @@ def create(
     flags: Optional[list[str]] = typer.Argument(None),
     editable: Optional[bool] = None,
     extra_module: Optional[list[str]] = None,
-    diracx_dist_dir: Optional[str] = None,
+    diracx_src_dir: Optional[str] = None,
     release_var: Optional[str] = None,
     run_server_tests: bool = True,
     run_client_tests: bool = True,
     run_pilot_tests: bool = True,
 ):
     """Start a local instance of the integration tests"""
-    prepare_environment(flags, editable, extra_module, diracx_dist_dir, release_var)
+    prepare_environment(flags, editable, extra_module, diracx_src_dir, release_var)
     install_server()
     install_client()
     install_pilot()
@@ -230,7 +230,7 @@ def prepare_environment(
     flags: Optional[list[str]] = typer.Argument(None),
     editable: Optional[bool] = None,
     extra_module: Optional[list[str]] = None,
-    diracx_dist_dir: Optional[str] = None,
+    diracx_src_dir: Optional[str] = None,
     release_var: Optional[str] = None,
 ):
     """Prepare the local environment for installing DIRAC."""
@@ -277,7 +277,7 @@ def prepare_environment(
     extra_services = list(chain(*[config["extra-services"] for config in module_configs.values()]))
 
     typer.secho("Running docker compose to create containers", fg=c.GREEN)
-    with _gen_docker_compose(modules, diracx_dist_dir=diracx_dist_dir) as docker_compose_fn:
+    with _gen_docker_compose(modules, diracx_src_dir=diracx_src_dir) as docker_compose_fn:
         subprocess.run(
             [*DOCKER_COMPOSE_CMD, "-f", docker_compose_fn, "up", "-d", "dirac-server", "dirac-client", "dirac-pilot"]
             + extra_services,
@@ -374,19 +374,32 @@ def prepare_environment(
     typer.secho("Running docker compose to create DiracX containers", fg=c.GREEN)
     typer.secho(f"Will leave a folder behind: {docker_compose_fn_final}", fg=c.YELLOW)
 
-    with _gen_docker_compose(modules, diracx_dist_dir=diracx_dist_dir) as docker_compose_fn:
+    with _gen_docker_compose(modules, diracx_src_dir=diracx_src_dir) as docker_compose_fn:
         # We cannot use the temporary directory created in the context manager because
         # we don't stay in the contect manager (Popen)
         # So we need something that outlives it.
         shutil.copytree(docker_compose_fn.parent, docker_compose_fn_final, dirs_exist_ok=True)
+
+        docker_compose_file = docker_compose_fn_final / "docker-compose.yml"
+
+        # Pre-build any images defined with build: directives so that the
+        # background "up -d" below only needs to start containers (fast).
+        if diracx_src_dir is not None:
+            typer.secho("Building DiracX container images from source", fg=c.GREEN)
+            subprocess.run(
+                [*DOCKER_COMPOSE_CMD, "-f", docker_compose_file, "build"],
+                check=True,
+                env=docker_compose_env,
+            )
+
         # We use Popen because we don't want to wait for this command to finish.
         # It is going to start all the diracx containers, including one which waits
         # for the DIRAC installation to be over.
         subStdout = open(docker_compose_fn_final / "stdout", "w")
         subStderr = open(docker_compose_fn_final / "stderr", "w")
 
         subprocess.Popen(
-            [*DOCKER_COMPOSE_CMD, "-f", docker_compose_fn_final / "docker-compose.yml", "up", "-d", "diracx"],
+            [*DOCKER_COMPOSE_CMD, "-f", docker_compose_file, "up", "-d", "diracx"],
             env=docker_compose_env,
             stdin=None,
             stdout=subStdout,
@@ -591,7 +604,7 @@ class TestExit(typer.Exit):
 
 
 @contextmanager
-def _gen_docker_compose(modules, *, diracx_dist_dir=None):
+def _gen_docker_compose(modules, *, diracx_src_dir=None):
     # Load the docker compose configuration and mount the necessary volumes
     input_fn = Path(__file__).parent / "tests/CI/docker-compose.yml"
     docker_compose = yaml.safe_load(input_fn.read_text())
@@ -607,23 +620,50 @@ def _gen_docker_compose(modules, *, diracx_dist_dir=None):
     docker_compose["services"]["diracx-wait-for-db"]["volumes"].extend(volumes[:])
 
     module_configs = _load_module_configs(modules)
-    if diracx_dist_dir is not None:
-        for container_name in [
-            "dirac-client",
-            "dirac-pilot",
-            "dirac-server",
-            "diracx-init-cs",
-            "diracx-wait-for-db",
-            "diracx-init-db",
-            "diracx",
-        ]:
+    if diracx_src_dir is not None:
+        diracx_src_dir = str(Path(diracx_src_dir).absolute())
+
+        # Build wheels from the diracx source for the DIRAC containers
+        diracx_wheel_dir = Path(tempfile.mkdtemp(prefix="diracx-wheels-"))
+        typer.secho(f"Building diracx wheels in {diracx_wheel_dir}", fg=c.GREEN)
+        subprocess.run(
+            [
+                sys.executable,
+                "-m",
+                "pip",
+                "wheel",
+                "--no-deps",
+                f"--wheel-dir={diracx_wheel_dir}",
+                *[f"{diracx_src_dir}/diracx-{pkg}" for pkg in ["core", "client", "cli"]],
+            ],
+            check=True,
+        )
+
+        # Mount wheels into DIRAC containers so installDIRACX can find them
+        for container_name in ["dirac-client", "dirac-pilot", "dirac-server"]:
             docker_compose["services"][container_name].setdefault("volumes", []).append(
-                f"{diracx_dist_dir}:/diracx_sources"
+                f"{diracx_wheel_dir}:/diracx_sources"
             )
             docker_compose["services"][container_name].setdefault("environment", []).append(
                 "DIRACX_CUSTOM_SOURCE_PREFIXES=/diracx_sources"
             )
 
+        # Build diracx container images from source instead of pulling pre-built ones
+        diracx_build_services = {
+            "container-services": ["diracx", "diracx-init-keystore", "diracx-init-db"],
+            "container-client": ["diracx-init-cs"],
+        }
+        for pixi_env, container_names in diracx_build_services.items():
+            for container_name in container_names:
+                service = docker_compose["services"][container_name]
+                del service["image"]
+                service.pop("pull_policy", None)
+                service["build"] = {
+                    "context": diracx_src_dir,
+                    "dockerfile": "containers/Dockerfile",
+                    "args": {"PIXI_ENV": pixi_env},
+                }
+
     # Add any extension services
     for module_name, module_configs in module_configs.items():
         for service_name, service_config in module_configs["extra-services"].items():

setup.cfg

@@ -30,7 +30,7 @@ install_requires =
     certifi
     cwltool
     diraccfg
-    DIRACCommon==v9.0.20
+    DIRACCommon
     diracx-client >=v0.0.1
     diracx-core >=v0.0.1
     diracx-cli >=v0.0.1

src/DIRAC/Core/Utilities/ElasticSearchDB.py

@@ -265,7 +265,7 @@ def getDoc(self, index: str, docID: str) -> dict:
         """
         sLog.debug(f"Retrieving document {docID} in index {index}")
         try:
-            return S_OK(self.client.get(index, docID)["_source"])
+            return S_OK(self.client.get(index=index, id=docID)["_source"])
         except NotFoundError:
             sLog.warn("Could not find the document in index", index)
             return S_OK({})
@@ -282,7 +282,7 @@ def getDocs(self, indexFunc, docIDs: list[str], vo: str) -> list[dict]:
         sLog.debug(f"Retrieving documents {docIDs}")
         docs = [{"_index": indexFunc(docID, vo), "_id": docID} for docID in docIDs]
         try:
-            response = self.client.mget({"docs": docs})
+            response = self.client.mget(body={"docs": docs})
         except RequestError as re:
             return S_ERROR(re)
         else:
@@ -300,12 +300,12 @@ def updateDoc(self, index: str, docID: str, body) -> dict:
         """
         sLog.debug(f"Updating document {docID} in index {index}")
         try:
-            self.client.update(index, docID, body)
+            self.client.update(index=index, id=docID, body=body)
         except ConflictError:
             # updates are rather "heavy" operations from ES point of view, needing seqNo to be updated.
             # Not ideal, but we just wait and retry.
             time.sleep(1)
-            self.client.update(index, docID, body, params={"retry_on_conflict": 3})
+            self.client.update(index=index, id=docID, body=body, params={"retry_on_conflict": 3})
         except RequestError as re:
             return S_ERROR(re)
         return S_OK()
@@ -319,7 +319,7 @@ def deleteDoc(self, index: str, docID: str):
         """
         sLog.debug(f"Deleting document {docID} in index {index}")
         try:
-            return S_OK(self.client.delete(index, docID))
+            return S_OK(self.client.delete(index=index, id=docID))
         except RequestError as re:
             return S_ERROR(re)
         except NotFoundError:
@@ -334,7 +334,7 @@ def existsDoc(self, index: str, docID: str) -> bool:
         :param docID: document ID
         """
         sLog.debug(f"Checking if document {docID} in index {index} exists")
-        return self.client.exists(index, docID)
+        return self.client.exists(index=index, id=docID)
 
     @ifConnected
     def _Search(self, indexname):
@@ -365,7 +365,7 @@ def getIndexes(self, indexName=None):
             indexName = ""
         sLog.debug(f"Getting indices alias of {indexName}")
         # we only return indexes which belong to a specific prefix for example 'lhcb-production' or 'dirac-production etc.
-        return list(self.client.indices.get_alias(f"{indexName}*"))
+        return list(self.client.indices.get_alias(index=f"{indexName}*"))
 
     @ifConnected
     def getDocTypes(self, indexName):
@@ -378,7 +378,7 @@ def getDocTypes(self, indexName):
         result = []
         try:
             sLog.debug("Getting mappings for ", indexName)
-            result = self.client.indices.get_mapping(indexName)
+            result = self.client.indices.get_mapping(index=indexName)
         except Exception as e:  # pylint: disable=broad-except
             sLog.exception()
             return S_ERROR(e)
@@ -409,7 +409,7 @@ def existingIndex(self, indexName):
         """
         sLog.debug(f"Checking existance of index {indexName}")
         try:
-            return S_OK(self.client.indices.exists(indexName))
+            return S_OK(self.client.indices.exists(index=indexName))
         except TransportError as e:
             sLog.exception()
             return S_ERROR(e)
@@ -446,7 +446,7 @@ def deleteIndex(self, indexName):
         """
         sLog.info("Deleting index", indexName)
         try:
-            retVal = self.client.indices.delete(indexName)
+            retVal = self.client.indices.delete(index=indexName)
         except NotFoundError:
             sLog.warn("Index does not exist", indexName)
             return S_OK("Nothing to delete")

src/DIRAC/Core/Utilities/MySQL.py

@@ -401,7 +401,7 @@ def __getWithRetry(self, dbName, totalRetries, retriesLeft):
 
     def __ping(self, conn):
         try:
-            conn.ping(True)
+            conn.ping()
             return True
         except Exception:
             return False

src/DIRAC/FrameworkSystem/Client/BundleDeliveryClient.py

@@ -1,5 +1,5 @@
-""" Client for interacting with Framework/BundleDelivery service
-"""
+"""Client for interacting with Framework/BundleDelivery service"""
+
 import getpass
 import os
 import tarfile
@@ -143,9 +143,10 @@ def syncCAs(self):
         if "X509_CERT_DIR" in os.environ:
             X509_CERT_DIR = os.environ["X509_CERT_DIR"]
             del os.environ["X509_CERT_DIR"]
+        result = self.syncDir("CAs", Locations.getCAsLocation())
         if X509_CERT_DIR:
             os.environ["X509_CERT_DIR"] = X509_CERT_DIR
-        return self.syncDir("CAs", Locations.getCAsLocation())
+        return result
 
     def syncCRLs(self):
         """Synchronize CRLs
@@ -156,9 +157,10 @@ def syncCRLs(self):
         if "X509_CERT_DIR" in os.environ:
             X509_CERT_DIR = os.environ["X509_CERT_DIR"]
             del os.environ["X509_CERT_DIR"]
+        result = self.syncDir("CRLs", Locations.getCAsLocation())
         if X509_CERT_DIR:
             os.environ["X509_CERT_DIR"] = X509_CERT_DIR
-        return self.syncDir("CRLs", Locations.getCAsLocation())
+        return result
 
     def getCAs(self):
         """This method can be used to create the CAs. If the file can not be created,