diff --git a/src/DIRAC/FrameworkSystem/Client/BundleDeliveryClient.py b/src/DIRAC/FrameworkSystem/Client/BundleDeliveryClient.py index d51ab956769..66947900838 100644 --- a/src/DIRAC/FrameworkSystem/Client/BundleDeliveryClient.py +++ b/src/DIRAC/FrameworkSystem/Client/BundleDeliveryClient.py @@ -1,5 +1,5 @@ -""" Client for interacting with Framework/BundleDelivery service -""" +"""Client for interacting with Framework/BundleDelivery service""" + import getpass import os import tarfile @@ -143,9 +143,10 @@ def syncCAs(self): if "X509_CERT_DIR" in os.environ: X509_CERT_DIR = os.environ["X509_CERT_DIR"] del os.environ["X509_CERT_DIR"] + result = self.syncDir("CAs", Locations.getCAsLocation()) if X509_CERT_DIR: os.environ["X509_CERT_DIR"] = X509_CERT_DIR - return self.syncDir("CAs", Locations.getCAsLocation()) + return result def syncCRLs(self): """Synchronize CRLs @@ -156,9 +157,10 @@ def syncCRLs(self): if "X509_CERT_DIR" in os.environ: X509_CERT_DIR = os.environ["X509_CERT_DIR"] del os.environ["X509_CERT_DIR"] + result = self.syncDir("CRLs", Locations.getCAsLocation()) if X509_CERT_DIR: os.environ["X509_CERT_DIR"] = X509_CERT_DIR - return self.syncDir("CRLs", Locations.getCAsLocation()) + return result def getCAs(self): """This method can be used to create the CAs. If the file can not be created, diff --git a/src/DIRAC/FrameworkSystem/Service/BundleDeliveryHandler.py b/src/DIRAC/FrameworkSystem/Service/BundleDeliveryHandler.py index 46ace20e700..362904eee3d 100644 --- a/src/DIRAC/FrameworkSystem/Service/BundleDeliveryHandler.py +++ b/src/DIRAC/FrameworkSystem/Service/BundleDeliveryHandler.py @@ -1,9 +1,9 @@ -""" Handler for CAs + CRLs bundles -""" +"""Handler for CAs + CRLs bundles""" import io import os import tarfile +from pathlib import Path from DIRAC import S_ERROR, S_OK, gConfig, gLogger from DIRAC.Core.DISET.RequestHandler import RequestHandler @@ -66,12 +66,15 @@ def updateBundles(self): buffer_ = io.BytesIO() filesToBundle = sorted(File.getGlobbedFiles(bundlePaths)) if filesToBundle: - commonPath = os.path.commonprefix(filesToBundle) - commonEnd = len(commonPath) - gLogger.info(f"Bundle will have {len(filesToBundle)} files with common path {commonPath}") + paths = [Path(f) for f in filesToBundle] + # Path.parents is path-component-aware, unlike os.path.commonprefix + commonParent = ( + Path(os.path.commonpath(paths)).parent if len(paths) == 1 else Path(os.path.commonpath(paths)) + ) + gLogger.info(f"Bundle will have {len(filesToBundle)} files with common path {commonParent}") with tarfile.open("dummy", "w:gz", buffer_) as tarBuffer: - for filePath in filesToBundle: - tarBuffer.add(filePath, filePath[commonEnd:]) + for p in paths: + tarBuffer.add(str(p), str(p.relative_to(commonParent))) zippedData = buffer_.getvalue() buffer_.close() hash_ = File.getMD5ForFiles(filesToBundle) diff --git a/tests/CI/docker-compose.yml b/tests/CI/docker-compose.yml index 4ed608f5a94..845cceb9829 100644 --- a/tests/CI/docker-compose.yml +++ b/tests/CI/docker-compose.yml @@ -1,6 +1,8 @@ volumes: # Volume used to store the certificates of dirac certs_data: + # Volume used to store the crls of dirac + crls_data: # Volume used to store the config of diracx diracx-cs-store: # Volume used to store the pair of keys to sign the tokens @@ -18,7 +20,13 @@ services: ports: - 3306:3306 healthcheck: - test: ["CMD", "sh", "-c", "${MYSQL_ADMIN_COMMAND} ping -h localhost > /tmp/health.log 2>&1;"] + test: + [ + "CMD", + "sh", + "-c", + "${MYSQL_ADMIN_COMMAND} ping -h localhost > /tmp/health.log 2>&1;", + ] timeout: 20s retries: 10 start_period: 60s @@ -33,7 +41,8 @@ services: - 9200:9200 env_file: "${ES_VER}.env" healthcheck: - test: ["CMD", "curl", "-f", "-u", "elastic:changeme", "http://localhost:9200"] + test: + ["CMD", "curl", "-f", "-u", "elastic:changeme", "http://localhost:9200"] interval: 5s timeout: 2s retries: 15 @@ -53,7 +62,13 @@ services: depends_on: - iam-init-keystore healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8080/.well-known/openid-configuration"] + test: + [ + "CMD", + "curl", + "-f", + "http://localhost:8080/.well-known/openid-configuration", + ] interval: 5s timeout: 2s retries: 15 @@ -116,6 +131,7 @@ services: container_name: dirac-init-certificates volumes: - certs_data:/ca/certs/ + - crls_data:/ca/crl/ entrypoint: | /entrypoint.sh pull_policy: always @@ -146,6 +162,7 @@ services: nofile: 8192 volumes: - certs_data:/ca/certs + - crls_data:/ca/crl/ - diracx-cs-store:/cs_store - diracx-key-store:/keystore environment: @@ -154,7 +171,6 @@ services: command: ["sleep", "infinity"] # This is necessary because of the issue described in https://github.com/moby/moby/issues/42275. What is added here is a hack/workaround. pull_policy: always - dirac-client: platform: linux/amd64 image: ${CI_REGISTRY_IMAGE}/${HOST_OS}-dirac @@ -165,6 +181,7 @@ services: - dirac-server volumes: - certs_data:/ca/certs + - crls_data:/ca/crl/ ulimits: nofile: 8192 command: ["sleep", "infinity"] # This is necessary because of the issue described in https://github.com/moby/moby/issues/42275. What is added here is a hack/workaround. @@ -180,6 +197,7 @@ services: - dirac-server volumes: - certs_data:/ca/certs + - crls_data:/ca/crl/ - type: bind source: ${CVMFS_DIR} target: /cvmfs @@ -195,7 +213,6 @@ services: start_period: 60s command: ["sleep", "infinity"] # This is necessary because of the issue described in https://github.com/moby/moby/issues/42275. What is added here is a hack/workaround. - diracx-chmod: platform: linux/amd64 image: ghcr.io/diracgrid/diracx/secret-generation:latest @@ -210,7 +227,6 @@ services: bash -xc 'chmod -R o=u /keystore && chmod -R o=u /cs_store' pull_policy: always - diracx-init-keystore: platform: linux/amd64 image: ghcr.io/diracgrid/diracx/services:dev @@ -308,7 +324,14 @@ services: /entrypoint.sh bash -xc 'uvicorn --factory diracx.routers:create_app --host=0.0.0.0' healthcheck: - test: ["CMD", "/entrypoint.sh", "python", "-c", "import requests; requests.get('http://localhost:8000/.well-known/openid-configuration').raise_for_status()"] + test: + [ + "CMD", + "/entrypoint.sh", + "python", + "-c", + "import requests; requests.get('http://localhost:8000/.well-known/openid-configuration').raise_for_status()", + ] interval: 5s timeout: 2s retries: 15 diff --git a/tests/CI/run_pilot.sh b/tests/CI/run_pilot.sh index f7827b8503f..a94ecce1b41 100755 --- a/tests/CI/run_pilot.sh +++ b/tests/CI/run_pilot.sh @@ -23,22 +23,27 @@ mkdir -p /home/dirac/etc/grid-security/vomsdir mkdir -p /home/dirac/etc/grid-security/vomses cp /ca/certs/ca.cert.pem /home/dirac/etc/grid-security/certificates +cp /ca/certs/ca.crl.pem /home/dirac/etc/grid-security/certificates touch /home/dirac/etc/grid-security/vomsdir/vomsdir touch /home/dirac/etc/grid-security/vomses/vomses +# Generate the hash link file required by openSSL to index CA certificates +caHash=$(openssl x509 -in /home/dirac/etc/grid-security/certificates/ca.cert.pem -noout -hash) +ln -s ca.cert.pem "/home/dirac/etc/grid-security/certificates/$caHash.0" +tar --create --file "/home/dirac/etc/grid-security/certificates/$caHash.r0" --gzip /home/dirac/etc/grid-security/certificates/ca.crl.pem # Copy over the pilot proxy cp /ca/certs/pilot_proxy /tmp/x509up_u$UID eval "${PILOT_DOWNLOAD_COMMAND}" -echo "${PILOT_JSON}" > pilot.json -jq < pilot.json +echo "${PILOT_JSON}" >pilot.json +jq /dev/null; then +if command -v python &>/dev/null; then py='python' -elif command -v python3 &> /dev/null; then +elif command -v python3 &>/dev/null; then py='python3' -elif command -v python2 &> /dev/null; then +elif command -v python2 &>/dev/null; then py='python2' fi diff --git a/tests/Jenkins/dirac_ci.sh b/tests/Jenkins/dirac_ci.sh index cfdef01a8d7..b6e2c1e9671 100644 --- a/tests/Jenkins/dirac_ci.sh +++ b/tests/Jenkins/dirac_ci.sh @@ -115,8 +115,9 @@ installSite() { echo "==> CAs and certificates" - # Copy the CA to the list of trusted CA + # Copy the CA and CRL to the list of trusted CA cp "/ca/certs/ca.cert.pem" "${SERVERINSTALLDIR}/diracos/etc/grid-security/certificates/" + cp "/ca/certs/ca.crl.pem" "${SERVERINSTALLDIR}/diracos/etc/grid-security/certificates/" # Copy the cert and host key to the certificates directory cp /ca/certs/hostcert.pem "${SERVERINSTALLDIR}/diracos/etc/grid-security/" @@ -128,6 +129,7 @@ installSite() { # because otherwise the BundleDeliveryClient will send the full path, which # will be wrong on the client ln -s "ca.cert.pem" "${SERVERINSTALLDIR}/diracos/etc/grid-security/certificates/$caHash.0" + tar --create --file "${SERVERINSTALLDIR}/diracos/etc/grid-security/certificates/$caHash.r0" --gzip "${SERVERINSTALLDIR}/diracos/etc/grid-security/certificates/ca.crl.pem" rm -rf "${SERVERINSTALLDIR}/etc" ln -s "${SERVERINSTALLDIR}/diracos/etc" "${SERVERINSTALLDIR}/etc"