[8.0] fix: the architecture script can be set in the options

.github/workflows/basic.yml

@@ -7,10 +7,10 @@ jobs:
     runs-on: ubuntu-latest
     if: github.event_name != 'push' || github.repository == 'DIRACGrid/DIRAC'
     steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v2
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
       with:
-        python-version: '3.9'
+        python-version: '3.11'
     - name: Install pre-commit
       run: pip install pre-commit
     - name: Run pre-commit
@@ -22,7 +22,7 @@ jobs:
     timeout-minutes: 30
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Run shellcheck
       # TODO This should cover more than just tests/CI
       # Excluded codes related to sourcing files
@@ -37,10 +37,10 @@ jobs:
     timeout-minutes: 30
 
     steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v2
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
       with:
-        python-version: '3.9'
+        python-version: '3.11'
     - name: Installing dependencies
       run: |
         python -m pip install pycodestyle
@@ -70,17 +70,17 @@ jobs:
           #   * `test_BaseType_Unicode` and `test_nestedStructure` fail due to
           #     DISET's string and unicode types being poorly defined
           - pytest --runslow -k 'not test_BaseType_Unicode and not test_nestedStructure'
-          - pylint -E src/
+          - pylint -j 0 -E src/
+          - mypy
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Fail-fast for outdated pipelines
       run: .github/workflows/fail-fast.sh
-    - uses: conda-incubator/setup-miniconda@master
+    - uses: conda-incubator/setup-miniconda@v3
       with:
         environment-file: environment.yml
-        miniforge-variant: Mambaforge
-        use-mamba: true
+
     - name: Run tests
       run: |
         # FIXME: The unit tests currently only work with editable installs
@@ -94,7 +94,7 @@ jobs:
     runs-on: ubuntu-latest
     if: github.event_name != 'push' || github.repository == 'DIRACGrid/DIRAC'
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Fail-fast for outdated pipelines
       run: .github/workflows/fail-fast.sh
     - name: prepare environment
@@ -104,7 +104,7 @@ jobs:
     - name: run pilot wrapper test
       run: |
         eval "$(conda shell.bash hook)" && conda activate test-env
-        pylint -E \
+        pylint -j 0 -E \
           tests/Integration/WorkloadManagementSystem/Test_GenerateAndExecutePilotWrapper.py \
           src/DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py \
           src/DIRAC/Resources/Computing/BatchSystems/*.py

.github/workflows/cvmfs.yml

@@ -0,0 +1,36 @@
+name: Deployment_test
+
+on:
+  workflow_dispatch:
+
+jobs:
+  deploy_CVMFS:
+    runs-on: "ubuntu-latest"
+    steps:
+    - name: prepare environment
+      run: |
+        conda create -c conda-forge -n cvmfs-env ca-policy-lcg openssl==3.0.7 gct
+    - name: Deploy on CVMFS
+      env:
+          CVMFS_PROXY_BASE64: ${{ secrets.CVMFS_PROXY_BASE64 }}
+      run: |
+        eval "$(conda shell.bash hook)" && conda activate cvmfs-env
+          conda info
+          conda info --envs
+          conda list
+          conda install ca-policy-lcg openssl==3.0.7 gct
+
+          echo "$CVMFS_PROXY_BASE64" | base64 --decode > cvmfs.proxy
+          export X509_USER_PROXY=cvmfs.proxy
+          chmod 600 cvmfs.proxy
+          ls -l
+          cat cvmfs.proxy
+          export PATH=/usr/share/miniconda3/bin:/opt/conda/bin/:/opt/conda/condabin:$PATH
+          type -a openssl
+          openssl version
+          type -a gsissh
+
+          mkdir -p ~/.ssh/ && touch ~/.ssh/known_hosts
+          ssh-keyscan cvmfs-upload01.gridpp.rl.ac.uk >> ~/.ssh/known_hosts
+
+          gsissh -p 1975 -t cvmfs-upload01.gridpp.rl.ac.uk /home/diracsgm/admin/sync_packages.sh -v

.github/workflows/deployment.yml

@@ -17,20 +17,24 @@ jobs:
     name: PyPI deployment
     runs-on: "ubuntu-latest"
     if: github.event_name != 'push' || github.repository == 'DIRACGrid/DIRAC'
+    permissions:
+      id-token: write
+      attestations: write
+      contents: write
     defaults:
       run:
         shell: bash -l {0}
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           token: ${{ secrets.PAT || github.token }}
       - run: |
           git fetch --prune --unshallow
           git config --global user.email "ci@diracgrid.org"
           git config --global user.name "DIRACGrid CI"
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v5
         with:
-          python-version: '3.9'
+          python-version: '3.11'
       - name: Installing dependencies
         run: |
           python -m pip install \
@@ -62,18 +66,29 @@ jobs:
         run: |
           set -xeuo pipefail
           IFS=$'\n\t'
+          # Only do a real release for workflow_dispatch events from DIRACGrid/DIRAC for integration for Python 3 compatible branches
           if [[ "${{ github.repository }}" == "DIRACGrid/DIRAC" ]]; then
             if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
-              if [[ "${{ github.event.ref }}" =~ ^refs/heads/(integration|rel-v([8-9]|[1-9][0-9]+)\.[0-9]+)$ ]]; then
+              if [[ "${{ github.event.ref }}" =~ ^refs/heads/(integration|rel-v([8-9]|[1-9][0-9]+)r[0-9]+)$ ]]; then
                 echo "Will create a real release"
                 export NEW_VERSION="v${{ github.event.inputs.version }}"
                 if [[ "${NEW_VERSION}" == "v" ]]; then
+                  # If version wasn't given as an input to the workflow, use setuptools_scm to guess while removing "dev" portion of the version number
                   NEW_VERSION=v$(python -m setuptools_scm | sed 's@Guessed Version @@g' | sed -E 's@(\.dev|\+g).+@@g')
                   export NEW_VERSION
                 fi
                 echo "Release will be named $NEW_VERSION"
                 # Validate the version
+                # Ensure the version doesn't look like a PEP-440 "dev release" (which might happen if the automatic version bump has issues)
                 python -c $'from packaging.version import Version; v = Version('"'$NEW_VERSION'"$')\nif v.is_devrelease:\n    raise ValueError(v)'
+                if [[ "${{ github.event.ref }}" =~ ^refs/heads/integration$ ]]; then
+                  # If we're releasing from integration it must be a pre-release
+                  python -c $'from packaging.version import Version; v = Version('"'$NEW_VERSION'"$')\nif not v.is_prerelease:\n    raise ValueError("integration should only be used for pre-releases")'
+                elif [[ "${{ github.event.ref }}" != "$(python -c $'from packaging.version import Version; v = Version('"'$NEW_VERSION'"$')\nprint(f"refs/heads/rel-v{v.major}r{v.minor}")')" ]]; then
+                  # If we're not releasing from integration the version should match the rel-vXrY branch name
+                  echo "$NEW_VERSION is an invalid version for ${{ github.event.ref }}"
+                  exit 2
+                fi
                 # Commit the release notes
                 mv release.notes release.notes.old
                 {
@@ -86,8 +101,8 @@ jobs:
                 git show
                 # Create the tag
                 git tag "$NEW_VERSION"
-                echo ::set-output name=create-release::true
-                echo ::set-output name=new-version::"$NEW_VERSION"
+                echo "create-release=true" >> $GITHUB_OUTPUT
+                echo "new-version=$NEW_VERSION" >> $GITHUB_OUTPUT
               fi
             fi
           fi
@@ -114,3 +129,33 @@ jobs:
         with:
           user: __token__
           password: ${{ secrets.PYPI_API_TOKEN }}
+
+  deploy_CVMFS:
+    runs-on: "ubuntu-latest"
+    if: github.event_name == 'workflow_dispatch'
+    needs: deploy-pypi
+    steps:
+      - name: prepare environment
+        run: |
+          conda create -c conda-forge -n cvmfs-env ca-policy-lcg openssl==3.0.7 gct
+      - name: Deploy on CVMFS
+        env:
+            CVMFS_PROXY_BASE64: ${{ secrets.CVMFS_PROXY_BASE64 }}
+        run: |
+          eval "$(conda shell.bash hook)" && conda activate cvmfs-env
+          conda install ca-policy-lcg openssl==3.0.7 gct
+          echo "$CVMFS_PROXY_BASE64" | base64 --decode > cvmfs.proxy
+          export X509_USER_PROXY=cvmfs.proxy
+          chmod 600 cvmfs.proxy
+          export PATH=/usr/share/miniconda3/bin:/opt/conda/bin/:/opt/conda/condabin:$PATH
+          type -a openssl
+          openssl version
+          type -a gsissh
+          mkdir -p ~/.ssh/ && touch ~/.ssh/known_hosts
+          ssh-keyscan cvmfs-upload01.gridpp.rl.ac.uk >> ~/.ssh/known_hosts
+          gsissh -p 1975 -t cvmfs-upload01.gridpp.rl.ac.uk /home/diracsgm/admin/sync_packages.sh -v
+      - name: setup tmate session
+        if: ${{ failure() }}
+        uses: mxschmitt/action-tmate@v3
+        with:
+          limit-access-to-actor: true

.github/workflows/integration.yml

@@ -31,46 +31,51 @@ jobs:
       matrix:
         # TEST_NAME is a dummy variable used to make it easier to read the web interface
         include:
-          - TEST_NAME: "MySQL 5.7"
-            ARGS: MYSQL_VER=mysql:5.7
-          - TEST_NAME: "MariaDB 10.6, opensearch:1.0.0"
-            ARGS: MYSQL_VER=mariadb:10.6.3 ES_VER=opensearchproject/opensearch:1.0.0
-          - TEST_NAME: "HTTPS"
-            ARGS: TEST_HTTPS=Yes
-          - TEST_NAME: "Force JEncode"
-            ARGS: DIRAC_USE_JSON_ENCODE=Yes DIRAC_USE_JSON_DECODE=Yes
+          - TEST_NAME: "MariaDB 11.4"
+            ARGS: MYSQL_VER=mariadb:11.4.3
+          - TEST_NAME: "Force DEncode and MySQL8"
+            ARGS: DIRAC_USE_JSON_ENCODE=NO MYSQL_VER=mysql:8.0.40
+          - TEST_NAME: "Backward Compatibility"
+            ARGS: CLIENT_INSTALLATION_BRANCH=rel-v8r0 PILOT_INSTALLATION_BRANCH=rel-v8r0
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Fail-fast for outdated pipelines
       run: .github/workflows/fail-fast.sh
     - run: |
         git fetch --prune --unshallow
-    - uses: actions/setup-python@v2
+    - uses: actions/setup-python@v5
       with:
-        python-version: '3.9'
+        python-version: '3.11'
+    - uses: cvmfs-contrib/github-action-cvmfs@v4
     - name: Installing dependencies
       run: |
         python -m pip install \
             gitpython \
             packaging \
             pyyaml \
+            requests \
             typer
     - name: Prepare environment
       run: ./integration_tests.py prepare-environment ${{ matrix.ARGS }}
     - name: Install server
       run: ./integration_tests.py install-server
     - name: Install client
       run: ./integration_tests.py install-client
+    - name: Install pilot
+      run: ./integration_tests.py install-pilot
     - name: Server tests
       run: ./integration_tests.py test-server || touch server-tests-failed
     - name: Client tests
       run: ./integration_tests.py test-client || touch client-tests-failed
-    - name: Elasticsearch logs
-      run: docker logs elasticsearch
+    - name: Pilot tests
+      run: ./integration_tests.py test-pilot || touch pilot-tests-failed
+    - name: Opensearch logs
+      run: docker logs opensearch
     - name: Check test status
       run: |
         has_error=0
         if [ -f server-tests-failed ]; then has_error=1; echo "Server tests failed"; fi
         if [ -f client-tests-failed ]; then has_error=1; echo "Client tests failed"; fi
+        if [ -f pilot-tests-failed ]; then has_error=1; echo "pilot tests failed"; fi
         if [ ${has_error} = 1 ]; then exit 1; fi

.github/workflows/labeler.yml

@@ -6,6 +6,6 @@ jobs:
   triage:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/labeler@v3
+    - uses: actions/labeler@v5
       with:
         repo-token: "${{ secrets.PAT }}"

.github/workflows/pilotWrapper.yml

@@ -14,18 +14,53 @@ jobs:
           - 2.7.5
           - 2.7.13
           - 3.6.8
-          - 3.9.4
+          - 3.11.4
+        pilot_branch:
+          - master
+          - devel
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
+    - uses: cvmfs-contrib/github-action-cvmfs@v3
+
+    - name: Test CernVM-FS
+      run: ls /cvmfs/dirac.egi.eu
+
     - name: Fail-fast for outdated pipelines
       run: .github/workflows/fail-fast.sh
+
     - name: prepare environment
       run: |
         conda config --set add_pip_as_python_dependency false
         conda create -c conda-forge -c free -n python_${{ matrix.python }} python=${{ matrix.python }}
     - name: run pilot wrapper test
       run: |
+        export INVALID_UTF8_VAR=$'\xff'
         cp tests/Integration/WorkloadManagementSystem/Test_GenerateAndExecutePilotWrapper.py .
         eval "$(conda shell.bash hook)" && conda activate python_${{ matrix.python }}
-        python Test_GenerateAndExecutePilotWrapper.py file://${{ github.workspace }}/src/DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py
+        # use github APIs to get the artifacts URLS from https://github.com/DIRACGrid/Pilot/, for those named Pilot_${{ matrix.pilot_branch }}
+        url=$(curl -s https://api.github.com/repos/DIRACGrid/Pilot/actions/artifacts | jq -r '.artifacts[] | select(.name == "Pilot_${{ matrix.pilot_branch }}") | .archive_download_url')
+        echo $url
+
+        # download and unzip the url above
+        curl -L \
+              -H "Accept: application/vnd.github+json" \
+              -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
+              -H "X-GitHub-Api-Version: 2022-11-28" \
+              $url --output Pilot_${{ matrix.pilot_branch }}.zip
+
+        file_type=$(file --mime-type -b Pilot_${{ matrix.pilot_branch }}.zip)
+
+        if [ "$file_type" != "application/zip" ]; then
+          echo "The downloaded file is not a ZIP file. File type: $file_type"
+          exit 1
+        fi
+
+
+        mkdir -p ${{ matrix.pilot_branch }}/pilot
+        cp Pilot_${{ matrix.pilot_branch }}.zip ${{ matrix.pilot_branch }}/pilot
+        cd ${{ matrix.pilot_branch }}/pilot
+        unzip Pilot_${{ matrix.pilot_branch }}.zip
+        cd ../..
+
+        python Test_GenerateAndExecutePilotWrapper.py file://${{ github.workspace }}/src/DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py file://${{ github.workspace }}/${{ matrix.pilot_branch }}