diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index b21927e4321..c78f5d98d52 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -7,10 +7,10 @@ jobs: runs-on: ubuntu-latest if: github.event_name != 'push' || github.repository == 'DIRACGrid/DIRAC' steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: - python-version: '3.9' + python-version: '3.11' - name: Install pre-commit run: pip install pre-commit - name: Run pre-commit @@ -22,7 +22,7 @@ jobs: timeout-minutes: 30 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Run shellcheck # TODO This should cover more than just tests/CI # Excluded codes related to sourcing files @@ -37,10 +37,10 @@ jobs: timeout-minutes: 30 steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: - python-version: '3.9' + python-version: '3.11' - name: Installing dependencies run: | python -m pip install pycodestyle @@ -70,17 +70,17 @@ jobs: # * `test_BaseType_Unicode` and `test_nestedStructure` fail due to # DISET's string and unicode types being poorly defined - pytest --runslow -k 'not test_BaseType_Unicode and not test_nestedStructure' - - pylint -E src/ + - pylint -j 0 -E src/ + - mypy steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Fail-fast for outdated pipelines run: .github/workflows/fail-fast.sh - - uses: conda-incubator/setup-miniconda@master + - uses: conda-incubator/setup-miniconda@v3 with: environment-file: environment.yml - miniforge-variant: Mambaforge - use-mamba: true + - name: Run tests run: | # FIXME: The unit tests currently only work with editable installs @@ -94,7 +94,7 @@ jobs: runs-on: ubuntu-latest if: github.event_name != 'push' || github.repository == 'DIRACGrid/DIRAC' steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Fail-fast for outdated pipelines run: .github/workflows/fail-fast.sh - name: prepare environment @@ -104,7 +104,7 @@ jobs: - name: run pilot wrapper test run: | eval "$(conda shell.bash hook)" && conda activate test-env - pylint -E \ + pylint -j 0 -E \ tests/Integration/WorkloadManagementSystem/Test_GenerateAndExecutePilotWrapper.py \ src/DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py \ src/DIRAC/Resources/Computing/BatchSystems/*.py diff --git a/.github/workflows/cvmfs.yml b/.github/workflows/cvmfs.yml new file mode 100644 index 00000000000..17a38ace821 --- /dev/null +++ b/.github/workflows/cvmfs.yml @@ -0,0 +1,36 @@ +name: Deployment_test + +on: + workflow_dispatch: + +jobs: + deploy_CVMFS: + runs-on: "ubuntu-latest" + steps: + - name: prepare environment + run: | + conda create -c conda-forge -n cvmfs-env ca-policy-lcg openssl==3.0.7 gct + - name: Deploy on CVMFS + env: + CVMFS_PROXY_BASE64: ${{ secrets.CVMFS_PROXY_BASE64 }} + run: | + eval "$(conda shell.bash hook)" && conda activate cvmfs-env + conda info + conda info --envs + conda list + conda install ca-policy-lcg openssl==3.0.7 gct + + echo "$CVMFS_PROXY_BASE64" | base64 --decode > cvmfs.proxy + export X509_USER_PROXY=cvmfs.proxy + chmod 600 cvmfs.proxy + ls -l + cat cvmfs.proxy + export PATH=/usr/share/miniconda3/bin:/opt/conda/bin/:/opt/conda/condabin:$PATH + type -a openssl + openssl version + type -a gsissh + + mkdir -p ~/.ssh/ && touch ~/.ssh/known_hosts + ssh-keyscan cvmfs-upload01.gridpp.rl.ac.uk >> ~/.ssh/known_hosts + + gsissh -p 1975 -t cvmfs-upload01.gridpp.rl.ac.uk /home/diracsgm/admin/sync_packages.sh -v diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index e3a15079fb1..605c51a9b60 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -17,20 +17,24 @@ jobs: name: PyPI deployment runs-on: "ubuntu-latest" if: github.event_name != 'push' || github.repository == 'DIRACGrid/DIRAC' + permissions: + id-token: write + attestations: write + contents: write defaults: run: shell: bash -l {0} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: token: ${{ secrets.PAT || github.token }} - run: | git fetch --prune --unshallow git config --global user.email "ci@diracgrid.org" git config --global user.name "DIRACGrid CI" - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v5 with: - python-version: '3.9' + python-version: '3.11' - name: Installing dependencies run: | python -m pip install \ @@ -62,18 +66,29 @@ jobs: run: | set -xeuo pipefail IFS=$'\n\t' + # Only do a real release for workflow_dispatch events from DIRACGrid/DIRAC for integration for Python 3 compatible branches if [[ "${{ github.repository }}" == "DIRACGrid/DIRAC" ]]; then if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then - if [[ "${{ github.event.ref }}" =~ ^refs/heads/(integration|rel-v([8-9]|[1-9][0-9]+)\.[0-9]+)$ ]]; then + if [[ "${{ github.event.ref }}" =~ ^refs/heads/(integration|rel-v([8-9]|[1-9][0-9]+)r[0-9]+)$ ]]; then echo "Will create a real release" export NEW_VERSION="v${{ github.event.inputs.version }}" if [[ "${NEW_VERSION}" == "v" ]]; then + # If version wasn't given as an input to the workflow, use setuptools_scm to guess while removing "dev" portion of the version number NEW_VERSION=v$(python -m setuptools_scm | sed 's@Guessed Version @@g' | sed -E 's@(\.dev|\+g).+@@g') export NEW_VERSION fi echo "Release will be named $NEW_VERSION" # Validate the version + # Ensure the version doesn't look like a PEP-440 "dev release" (which might happen if the automatic version bump has issues) python -c $'from packaging.version import Version; v = Version('"'$NEW_VERSION'"$')\nif v.is_devrelease:\n raise ValueError(v)' + if [[ "${{ github.event.ref }}" =~ ^refs/heads/integration$ ]]; then + # If we're releasing from integration it must be a pre-release + python -c $'from packaging.version import Version; v = Version('"'$NEW_VERSION'"$')\nif not v.is_prerelease:\n raise ValueError("integration should only be used for pre-releases")' + elif [[ "${{ github.event.ref }}" != "$(python -c $'from packaging.version import Version; v = Version('"'$NEW_VERSION'"$')\nprint(f"refs/heads/rel-v{v.major}r{v.minor}")')" ]]; then + # If we're not releasing from integration the version should match the rel-vXrY branch name + echo "$NEW_VERSION is an invalid version for ${{ github.event.ref }}" + exit 2 + fi # Commit the release notes mv release.notes release.notes.old { @@ -86,8 +101,8 @@ jobs: git show # Create the tag git tag "$NEW_VERSION" - echo ::set-output name=create-release::true - echo ::set-output name=new-version::"$NEW_VERSION" + echo "create-release=true" >> $GITHUB_OUTPUT + echo "new-version=$NEW_VERSION" >> $GITHUB_OUTPUT fi fi fi @@ -114,3 +129,33 @@ jobs: with: user: __token__ password: ${{ secrets.PYPI_API_TOKEN }} + + deploy_CVMFS: + runs-on: "ubuntu-latest" + if: github.event_name == 'workflow_dispatch' + needs: deploy-pypi + steps: + - name: prepare environment + run: | + conda create -c conda-forge -n cvmfs-env ca-policy-lcg openssl==3.0.7 gct + - name: Deploy on CVMFS + env: + CVMFS_PROXY_BASE64: ${{ secrets.CVMFS_PROXY_BASE64 }} + run: | + eval "$(conda shell.bash hook)" && conda activate cvmfs-env + conda install ca-policy-lcg openssl==3.0.7 gct + echo "$CVMFS_PROXY_BASE64" | base64 --decode > cvmfs.proxy + export X509_USER_PROXY=cvmfs.proxy + chmod 600 cvmfs.proxy + export PATH=/usr/share/miniconda3/bin:/opt/conda/bin/:/opt/conda/condabin:$PATH + type -a openssl + openssl version + type -a gsissh + mkdir -p ~/.ssh/ && touch ~/.ssh/known_hosts + ssh-keyscan cvmfs-upload01.gridpp.rl.ac.uk >> ~/.ssh/known_hosts + gsissh -p 1975 -t cvmfs-upload01.gridpp.rl.ac.uk /home/diracsgm/admin/sync_packages.sh -v + - name: setup tmate session + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 + with: + limit-access-to-actor: true diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 41a048ec057..f3b9620f46a 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -31,30 +31,30 @@ jobs: matrix: # TEST_NAME is a dummy variable used to make it easier to read the web interface include: - - TEST_NAME: "MySQL 5.7" - ARGS: MYSQL_VER=mysql:5.7 - - TEST_NAME: "MariaDB 10.6, opensearch:1.0.0" - ARGS: MYSQL_VER=mariadb:10.6.3 ES_VER=opensearchproject/opensearch:1.0.0 - - TEST_NAME: "HTTPS" - ARGS: TEST_HTTPS=Yes - - TEST_NAME: "Force JEncode" - ARGS: DIRAC_USE_JSON_ENCODE=Yes DIRAC_USE_JSON_DECODE=Yes + - TEST_NAME: "MariaDB 11.4" + ARGS: MYSQL_VER=mariadb:11.4.3 + - TEST_NAME: "Force DEncode and MySQL8" + ARGS: DIRAC_USE_JSON_ENCODE=NO MYSQL_VER=mysql:8.0.40 + - TEST_NAME: "Backward Compatibility" + ARGS: CLIENT_INSTALLATION_BRANCH=rel-v8r0 PILOT_INSTALLATION_BRANCH=rel-v8r0 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Fail-fast for outdated pipelines run: .github/workflows/fail-fast.sh - run: | git fetch --prune --unshallow - - uses: actions/setup-python@v2 + - uses: actions/setup-python@v5 with: - python-version: '3.9' + python-version: '3.11' + - uses: cvmfs-contrib/github-action-cvmfs@v4 - name: Installing dependencies run: | python -m pip install \ gitpython \ packaging \ pyyaml \ + requests \ typer - name: Prepare environment run: ./integration_tests.py prepare-environment ${{ matrix.ARGS }} @@ -62,15 +62,20 @@ jobs: run: ./integration_tests.py install-server - name: Install client run: ./integration_tests.py install-client + - name: Install pilot + run: ./integration_tests.py install-pilot - name: Server tests run: ./integration_tests.py test-server || touch server-tests-failed - name: Client tests run: ./integration_tests.py test-client || touch client-tests-failed - - name: Elasticsearch logs - run: docker logs elasticsearch + - name: Pilot tests + run: ./integration_tests.py test-pilot || touch pilot-tests-failed + - name: Opensearch logs + run: docker logs opensearch - name: Check test status run: | has_error=0 if [ -f server-tests-failed ]; then has_error=1; echo "Server tests failed"; fi if [ -f client-tests-failed ]; then has_error=1; echo "Client tests failed"; fi + if [ -f pilot-tests-failed ]; then has_error=1; echo "pilot tests failed"; fi if [ ${has_error} = 1 ]; then exit 1; fi diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index e568a2e6fe7..f19795e53f3 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -6,6 +6,6 @@ jobs: triage: runs-on: ubuntu-latest steps: - - uses: actions/labeler@v3 + - uses: actions/labeler@v5 with: repo-token: "${{ secrets.PAT }}" diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml deleted file mode 100644 index 1a277181188..00000000000 --- a/.github/workflows/nightly.yml +++ /dev/null @@ -1,42 +0,0 @@ -# This workflow runs every 24 hours for the following purpose: -# -# - Create + upload *integration* + *master* tarball -# -# All created images are then uploaded to GitHub packages - -name: Nightlies - -# on: push - -on: - schedule: - # every day at 6 am (so that the master of DIRACOS is already created) - - cron: '0 6 * * *' - -jobs: - # running dirac-distribution in the proper image - dirac-distribute: - runs-on: ubuntu-latest - if: github.repository == 'DIRACGrid/DIRAC' - - strategy: - fail-fast: False - matrix: - branch: - - master - - integration - - steps: - - uses: actions/checkout@v1 - - name: create - run: | - docker pull ghcr.io/diracgrid/management/dirac-distribution:latest - docker run ghcr.io/diracgrid/management/dirac-distribution:latest bash -c \ - "python3 dirac-distribution.py -r ${{ matrix.branch }} | tail -n 1 > /tmp/deploy.sh && "\ - "sed -i 's/lhcbprod/${{ secrets.KRB_USERNAME }}/g' /tmp/deploy.sh && "\ - "cat /tmp/deploy.sh && "\ - "echo ${{ secrets.KRB_PASSWORD }} | kinit ${{ secrets.KRB_USERNAME }}@CERN.CH && "\ - "echo readyToUpload && "\ - "export USER=${{ secrets.KRB_USERNAME }} && "\ - "echo reallyReadyToUpload && "\ - "source /tmp/deploy.sh" diff --git a/.github/workflows/pilotWrapper.yml b/.github/workflows/pilotWrapper.yml index ce293d55001..3010db7c0b6 100644 --- a/.github/workflows/pilotWrapper.yml +++ b/.github/workflows/pilotWrapper.yml @@ -14,18 +14,53 @@ jobs: - 2.7.5 - 2.7.13 - 3.6.8 - - 3.9.4 + - 3.11.4 + pilot_branch: + - master + - devel steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 + - uses: cvmfs-contrib/github-action-cvmfs@v3 + + - name: Test CernVM-FS + run: ls /cvmfs/dirac.egi.eu + - name: Fail-fast for outdated pipelines run: .github/workflows/fail-fast.sh + - name: prepare environment run: | conda config --set add_pip_as_python_dependency false conda create -c conda-forge -c free -n python_${{ matrix.python }} python=${{ matrix.python }} - name: run pilot wrapper test run: | + export INVALID_UTF8_VAR=$'\xff' cp tests/Integration/WorkloadManagementSystem/Test_GenerateAndExecutePilotWrapper.py . eval "$(conda shell.bash hook)" && conda activate python_${{ matrix.python }} - python Test_GenerateAndExecutePilotWrapper.py file://${{ github.workspace }}/src/DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py + # use github APIs to get the artifacts URLS from https://github.com/DIRACGrid/Pilot/, for those named Pilot_${{ matrix.pilot_branch }} + url=$(curl -s https://api.github.com/repos/DIRACGrid/Pilot/actions/artifacts | jq -r '.artifacts[] | select(.name == "Pilot_${{ matrix.pilot_branch }}") | .archive_download_url') + echo $url + + # download and unzip the url above + curl -L \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + $url --output Pilot_${{ matrix.pilot_branch }}.zip + + file_type=$(file --mime-type -b Pilot_${{ matrix.pilot_branch }}.zip) + + if [ "$file_type" != "application/zip" ]; then + echo "The downloaded file is not a ZIP file. File type: $file_type" + exit 1 + fi + + + mkdir -p ${{ matrix.pilot_branch }}/pilot + cp Pilot_${{ matrix.pilot_branch }}.zip ${{ matrix.pilot_branch }}/pilot + cd ${{ matrix.pilot_branch }}/pilot + unzip Pilot_${{ matrix.pilot_branch }}.zip + cd ../.. + + python Test_GenerateAndExecutePilotWrapper.py file://${{ github.workspace }}/src/DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py file://${{ github.workspace }}/${{ matrix.pilot_branch }} diff --git a/.github/workflows/pr-sweep.yml b/.github/workflows/pr-sweep.yml index 286cfd524a7..26f89a2c78a 100644 --- a/.github/workflows/pr-sweep.yml +++ b/.github/workflows/pr-sweep.yml @@ -7,9 +7,10 @@ jobs: pr-sweep: runs-on: ubuntu-latest concurrency: pr-sweep + timeout-minutes: 30 if: github.repository == 'DIRACGrid/DIRAC' steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 token: ${{ secrets.PAT }} diff --git a/.github/workflows/semantic.yml b/.github/workflows/semantic.yml index 07af3517572..3550803141c 100644 --- a/.github/workflows/semantic.yml +++ b/.github/workflows/semantic.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check Commit Format - uses: gsactions/commit-message-checker@v1 + uses: gsactions/commit-message-checker@v2 with: pattern: '^((docs|feat|fix|refactor|style|test|sweep)( ?\(.*\))?: .+|Revert ".+")$' excludeDescription: 'true' # optional: this excludes the description body of a pull request @@ -23,7 +23,7 @@ jobs: flags: 'gim' error: 'Your commit has to follow the format "(): "".' - name: Check Commit Length - uses: gsactions/commit-message-checker@v1 + uses: gsactions/commit-message-checker@v2 with: pattern: '^.{20,150}$' error: 'Commit messages should be between 20 and 150 chars' diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index af1455c5172..163eabb7018 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # See https://pre-commit.com/hooks.html for more hooks default_language_version: - python: python3.9 + python: python3.11 exclude: | (?x)^( @@ -11,15 +11,19 @@ exclude: | repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.1.0 + rev: v5.0.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer + exclude: | + (?x)^( + src/DIRAC/Core/Security/test/certs/ca/ca.key.pem + )$ - id: check-yaml - id: check-added-large-files - repo: https://github.com/psf/black - rev: 22.3.0 + rev: 23.1.0 hooks: - id: black exclude: | @@ -42,3 +46,26 @@ repos: src/DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py| tests/Integration/WorkloadManagementSystem/Test_GenerateAndExecutePilotWrapper.py )$ + + - repo: https://github.com/ikamensh/flynt/ + rev: "0.77" + hooks: + - id: flynt + exclude: | + (?x)^( + src/DIRAC/Resources/Computing/BatchSystems/[^/]+.py| + src/DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py| + tests/Integration/WorkloadManagementSystem/Test_GenerateAndExecutePilotWrapper.py + )$ + + - repo: https://github.com/asottile/pyupgrade + rev: v3.3.1 + hooks: + - id: pyupgrade + args: ["--py39-plus"] + exclude: | + (?x)^( + src/DIRAC/Resources/Computing/BatchSystems/[^/]+.py| + src/DIRAC/WorkloadManagementSystem/Utilities/PilotWrapper.py| + tests/Integration/WorkloadManagementSystem/Test_GenerateAndExecutePilotWrapper.py + )$ diff --git a/.pylintrc b/.pylintrc deleted file mode 100644 index a0f87f61f60..00000000000 --- a/.pylintrc +++ /dev/null @@ -1,215 +0,0 @@ -[MASTER] -# Specify a configuration file. -#rcfile= - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -# init-hook='import sys; sys.path.append("$WORKSPACE/DIRAC/WorkloadManagementSystem/PilotAgent")' - -# Add to the black list. It should be a base name, not a -# path. You may set this option multiple times. -ignore=.svn,.git,integration_tests.py - -# Pickle collected data for later comparisons. -persistent=yes - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins= - - -[MESSAGES CONTROL] - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time. - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). - -#R0903 = Too few public methods (%s/%s) -#C0326 = bad-whitespace -#I0011 = locally-disabling -disable=R0903,C0326,I0011,redefined-variable-type,c-extension-no-member - -[REPORTS] - -# Set the output format. Available formats are text, parseable, colorized, msvs -# (visual studio) and html -output-format=colorized - -# Put messages in a separate file for each module / package specified on the -# command line instead of printing them on stdout. Reports (if any) will be -# written in a file name "pylint_global.[txt|html]". -files-output=no - -# Tells whether to display a full report or only the messages -reports=no - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -[BASIC] - -# List of builtins function names that should not be used, separated by a comma -bad-functions=map,filter,apply,input,raw_input,has_key - -# Regular expression which should only match correct module names -# Added last option, to take into account script names -module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+)|([dirac_-][a-zA-Z_-]+)|(Test_[a-zA-Z_-]+))$ - -# Regular expression which should only match correct module level names -#const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__)|(g[A-Z][a-zA-Z0-9]*))$ -const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__)|(g[A-Z][a-zA-Z0-9]*)|[a-z_][A-Za-z0-9_]*)$ - -# Regular expression which should only match correct class names -class-rgx=[A-Z_][a-zA-Z0-9]+$ - -# Regular expression which should only match correct function names -function-rgx=(([a-z_][A-Za-z0-9_]{2,30}$)|(test_[A-Za-z0-9_]{2,50}$)) - -# Regular expression which should only match correct method names -method-rgx=[a-z_][A-Za-z0-9_]{2,30}$ - -# Regular expression which should only match correct instance attribute names -attr-rgx=[a-z_][A-Za-z0-9_]{2,30}$ - -# Regular expression which should only match correct argument names -argument-rgx=[a-z_][A-Za-z0-9_]{1,30}$ - -# Regular expression which should only match correct variable names -variable-rgx=[a-z_][A-Za-z0-9_]{1,30}$ - -# Regular expression which should only match correct list comprehension / -# generator expression variable names -inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ - -# Good variable names which should always be accepted, separated by a comma -good-names=e,i,j,k,x,ex,Run,_,S_OK,S_ERROR - -# Bad variable names which should always be refused, separated by a comma -bad-names=foo,bar,baz,toto,tutu,tata,spam,egg - -# Regular expression which should only match functions or classes name which do -# not require a docstring -no-docstring-rgx=__.*__ - - -[FORMAT] - -# Maximum number of characters on a single line. -max-line-length=130 - -# Maximum number of lines in a module -max-module-lines=1200 - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME,XXX,TODO - -[SIMILARITIES] - -# Minimum lines number of a similarity. -min-similarity-lines=4 - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -[TYPECHECK] - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# List of classes names for which member attributes should not be checked -# Skip some sqlalchemy objects which cause trouble -ignored-classes=SQLObject,Session - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E0201 when accessed. -generated-members=REQUEST,acl_users,aq_parent - -# List of ignored modules -# (useful for classes with attributes dynamically set). -ignored-modules = MySQLdb,numpy - -[VARIABLES] - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# A regular expression matching the beginning of the name of dummy variables -# (i.e. not used). -dummy-variables-rgx=_|dummy - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= - -[CLASSES] - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__,__new__,setUp - - -[DESIGN] - -# Maximum number of arguments for function / method -max-args=8 - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore -ignored-argument-names=_.* - -# Maximum number of locals for function / method body -max-locals=20 - -# Maximum number of return / yield for function / method body -max-returns=8 - -# Maximum number of branch for function / method body -max-branchs=15 - -# Maximum number of statements in function / method body -max-statements=50 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of attributes for a class (see R0902). -max-attributes=10 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=1 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 - -# Maximum number of nested blocks -max-nested-blocks=10 -[IMPORTS] - -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=regsub,string,TERMIOS,Bastion,rexec - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= diff --git a/.readthedocs.yaml b/.readthedocs.yaml index f841c92ea04..63d30735921 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,6 +1,6 @@ version: 2 build: - os: ubuntu-20.04 + os: ubuntu-24.04 tools: python: mambaforge-4.10 sphinx: diff --git a/AUTHORS.rst b/AUTHORS.rst index 82ee68848b9..d226042bc9d 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -2,14 +2,12 @@ Main contributors to the source code ------------------------------------ - Andrei Tsaregorodtsev - Federico Stagni -- Zoltan Mathe - Christophe Haen +- Chris Burr +- Alexandre Boyer - Philippe Charpentier - Andre Sailer -- Marko Petric - Simon Fayer -- Andrew McNab -- Wojciech Krzemien DIRAC consortium members ------------------------ @@ -17,6 +15,5 @@ DIRAC consortium members - CERN - European Organisation for Nuclear Research (Switzerland) - IHEP (China) - UM (Montpellier, France) -- PNNL (USA) - KEK (Japan) - Imperial College (UK) diff --git a/Core/scripts/dirac-install.py b/Core/scripts/dirac-install.py deleted file mode 100755 index ceeab08c728..00000000000 --- a/Core/scripts/dirac-install.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python -import os -import stat -import sys -import tempfile - -try: - # For Python 3.0 and later - from urllib.request import urlopen -except ImportError: - # Fall back to Python 2's urllib2 - from urllib2 import urlopen - -sys.stderr.write("#" * 100 + "\n") -sys.stderr.write("#" * 100 + "\n") -sys.stderr.write("#" * 100 + "\n") -sys.stderr.write("#" * 100 + "\n") -sys.stderr.write("#" * 100 + "\n") -sys.stderr.write("\n") -sys.stderr.write("Getting dirac-install from this location is no longer supported!\n") -sys.stderr.write("\n") -sys.stderr.write("Please update your scripts to use:\n") -sys.stderr.write(" https://raw.githubusercontent.com/DIRACGrid/management/master/dirac-install.py\n") -sys.stderr.write("\n") -sys.stderr.write("#" * 100 + "\n") -sys.stderr.write("#" * 100 + "\n") -sys.stderr.write("#" * 100 + "\n") -sys.stderr.write("#" * 100 + "\n") -sys.stderr.write("#" * 100 + "\n") - -if os.environ.get("DIRAC_DEPRECATED_FAIL", "No").lower() in ("y", "yes", "t", "true", "on", "1"): - raise RuntimeError("Failing as DIRAC_DEPRECATED_FAIL is set") - -# Download dirac-install.py -response = urlopen("https://raw.githubusercontent.com/DIRACGrid/management/master/dirac-install.py") -code = response.getcode() -if code > 200 or code >= 300: - raise RuntimeError("Failed to download dirac-install.py with code %s" % code) - -# Write dirac-install.py to a temporay file -tmpHandle, tmp = tempfile.mkstemp() -fp = os.fdopen(tmpHandle, "wb") -fp.write(response.read()) -fp.close() - -# Make the dirac-install.py temporay file executable -st = os.stat(tmp) -os.chmod(tmp, st.st_mode | stat.S_IEXEC) - -# Replace the current process with the actual dirac-install.py script -os.execv(tmp, sys.argv) diff --git a/README.rst b/README.rst index 6f82b2e967e..8d540053cce 100644 --- a/README.rst +++ b/README.rst @@ -18,26 +18,22 @@ DIRAC has been started by the `LHCb collaboration `__. -DIRAC 7.3 adds support for Python 3 based clients and servers (which are production level from DIRAC 8.0). -We recommend transitioning to Python 3 clients during using DIRAC 7.2 and Python 3 servers using DIRAC 7.3. +DIRAC 8.0 drops support for Python 2 based clients and servers. There are three available options for installation: diff --git a/SECURITY.md b/SECURITY.md index 184b7301e37..79605a83b3c 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -4,11 +4,10 @@ | Version | Supported | | ------- | ------------------ | -| >= 7.1 | :white_check_mark: | -| < 7.1 | :x: | +| >= 8.0 | :white_check_mark: | +| < 8.0 | :x: | ## Reporting a Vulnerability -Please report any suspected vulnerabilities by emailing fstagni-at-cern.ch and atsareg-at-in2p3.fr. +Please report any suspected vulnerabilities in https://github.com/DIRACGrid/DIRAC/security. We will respond to you within 2 working days. -If for some reason you do not recieve a response, please follow up via email to ensure we received your original message. diff --git a/consistency_check/README.md b/consistency_check/README.md new file mode 100644 index 00000000000..40e0f15d191 --- /dev/null +++ b/consistency_check/README.md @@ -0,0 +1,126 @@ +# Consistency check + +this script is here to help compare storage and DFC dumps. + +## What you need + +### SE definitions + +A CSV file containing its name and base path. Like + +``` +CSCS-DST;/pnfs/lcg.cscs.ch/lhcb +CSCS_MC-DST;/pnfs/lcg.cscs.ch/lhcb +``` + +You can obtain it with something like + +```python +from DIRAC import initialize +initialize() +from DIRAC import gConfig +from DIRAC.Resources.Storage.StorageElement import StorageElement + +for se in gConfig.getSections("/Resources/StorageElements")["Value"]: + print(f"{se};{list(StorageElement(se).storages.values())[0].basePath}") +``` + +### StorageElement dump + +This is typically provided by the site, and we expect just a flat list of the files + +``` +/pnfs/lcg.cscs.ch/lhcb/generated/2013-07-07/fileeed071eb-1aa0-4d00-8775-79624737224e +/pnfs/lcg.cscs.ch/lhcb/generated/2013-07-10/fileed08b040-196c-46d9-b4d6-37d80cba27eb +/pnfs/lcg.cscs.ch/lhcb/lhcb/test/SAM/testfile-put-LHCb-Disk-1494915199-61e6d085bb84.txt +``` + +### Catalog dump(s) + +Ideally, you should have two catalog dumps for the SE that you are concerned about: one before the SE dump, and one after. Having only one of the two only allows to get partial comparison + +You could get it with a script like + +```python +import sys +from datetime import datetime,timezone +from DIRAC import initialize +initialize() +from DIRAC import gConfig +from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient +dfc = FileCatalogClient() + +# Something like LCG.CERN.ch +site_name = sys.argv[1] + +ses = gConfig.getOption(f"/Resources/Sites/{site_name.split('.')[0]}/{site_name}/SE",[])["Value"] + +timestamp = datetime.now(tz=timezone.utc).strftime("%Y%m%d_%H%M%S") +output_file = f"{site_name}_dfc_{timestamp}.dump" +print(f"Getting FC dump for {ses} in {output_file}") +res = dfc.getSEDump(ses, output_file) +print(res) +``` + + +Or from a `BaseSE` + +```python +#!/usr/bin/env python3 + +import sys +from datetime import datetime,timezone +from DIRAC import initialize +initialize() +from DIRAC import gConfig +from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient +dfc = FileCatalogClient() + +# Something like RAL-ECHO +base_se_name = sys.argv[1] + +ses = [] +ses_data = gConfig.getOptionsDictRecursively(f"/Resources/StorageElements")["Value"] +for key, val in ses_data.items(): + try: + if val['BaseSE'] == base_se_name: + ses.append(key) + except (KeyError, TypeError): + pass + +timestamp = datetime.now(tz=timezone.utc).strftime("%Y%m%d_%H%M%S") +output_file = f"{base_se_name}_dfc_{timestamp}.dump" +print(f"Getting FC dump for {ses} in {output_file}") +res = dfc.getSEDump(ses, output_file) +print(res) +``` + +## How it works + +We look at the differences and the intersections between the dump of the old catalog, the new catalog, and the storage element. + +For example, you find dark data by looking at files that are in the SE dump, but not in any of the catalog dump. Lost data is data that is in both catalog dump, but not in the SE dump. + + +| Old FC | New FC | SE | Status | +|--------|--------|----|------------------| +| 0 | 0 | 1 | Dark data | +| 0 | 1 | 0 | Very new | +| 0 | 1 | 1 | New | +| 1 | 0 | 0 | Deleted | +| 1 | 0 | 1 | Recently deleted | +| 1 | 1 | 0 | Lost file | +| 1 | 1 | 1 | OK | + +## How to use + +Although you probably need DIRAC to be able to get the DFC dump or the SE config, you do not need DIRAC installed once you have all the `csv` files. +You will however need `pandas` and `typer` + + +The `consistency` script has 3 commands: +* `threeways`: do a proper comparison of 1 old DFC dump, one SE dump, one new DFC dump. Results are as good as it gets +* `possibly-dark-data`: Tries to find dark data but be careful of the result (see `help`). +* `possibly-lost-data`: Tries to find lost data but be careful of the result (see `help`). + +In any case, you should check the output with commands like `dirac-dms-replica-stats` or `dirac-dms-pfn-exists`. diff --git a/consistency_check/consistency.py b/consistency_check/consistency.py new file mode 100755 index 00000000000..3485304a29b --- /dev/null +++ b/consistency_check/consistency.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python +import pandas as pd +import typer +from pathlib import Path +from typer import colors +from typing import Annotated + + +RED = colors.RED +GREEN = colors.GREEN + +app = typer.Typer() + + +def load_se_definition(se_def_path): + return pd.read_csv(se_def_path, names=["seName", "basePath"], delimiter=";", index_col="seName") + + +def load_dfc_dump(dfc_dump_path, version): + fc_dump = pd.read_csv(dfc_dump_path, names=["seName", "lfn", "fc_cks", "size"], delimiter="|") + fc_dump["fc_cks"] = fc_dump["fc_cks"].str.lower().str.pad(8, fillchar="0") + fc_dump["version"] = version + return fc_dump + + +def load_se_dump(se_dump_path): + se_dump = pd.read_csv(se_dump_path, names=["pfn", "se_cks"], delimiter="|", index_col="pfn") + se_dump["se_cks"] = se_dump["se_cks"].str.lower().str.pad(8, fillchar="0") + se_dump["version"] = "se_dump" + assert not se_dump.index.duplicated().any(), f"Duplicated entries in SE dump {se_dump[se_dump.index.duplicated()]}" + + return se_dump + + +@app.command() +def possibly_lost_data( + fc_dump_file: Annotated[Path, typer.Option(help="DFC dump AFTER the SE dump")], + se_def_file: Annotated[Path, typer.Option(help="Definition of the SE path")], + se_dump_file: Annotated[Path, typer.Option(help="Dump of the SE")], + lost_file_output: Annotated[Path, typer.Option(help="Output file in which to dump lost")] = "lost.csv", +): + """ + DANGER: make a partial comparison of an SE dump and an FC dump to find lost data + Be careful because you can't trust the result: + * if the FC dump is more recent than the SE dump, you may get files that were added on the SE after the dump + * if the FC dump is older than the SE dump, the file may have been purposedly removed + """ + se_dump = load_se_dump(se_dump_file) + se_def = load_se_definition(se_def_file) + + # Compute the PFN for each LFN in the DFC dump + + fc_dump = load_dfc_dump(fc_dump_file, "fc") + fc_dump = pd.merge(fc_dump, se_def, on="seName") + fc_dump["pfn"] = fc_dump["basePath"] + fc_dump["lfn"] + fc_dump.set_index("pfn", inplace=True) + + # Lost files: in both FC dump but not in the SE + + lostData = fc_dump.index.difference(se_dump.index) + if len(lostData): + typer.secho(f"Found {len(lostData)} lost files, dumping them in {lost_file_output}", err=True, fg=RED) + lastDataDetail = fc_dump[fc_dump.index.isin(lostData)] + lastDataDetail.to_csv(lost_file_output) + else: + typer.secho("No dark data found", fg=GREEN) + + +@app.command() +def possibly_dark_data( + fc_dump_file: Annotated[Path, typer.Option(help="DFC dump")], + se_def_file: Annotated[Path, typer.Option(help="Definition of the SE path")], + se_dump_file: Annotated[Path, typer.Option(help="Dump of the SE")], + dark_file_output: Annotated[Path, typer.Option(help="Output file in which to dump dark data")] = "dark.csv", +): + """ + DANGER: make a partial comparison of an SE dump and an FC dump to find dark data. + Be careful because you can't trust the result: + * if the FC dump is more recent than the SE dump, you may get files that were already removed + * if the FC dump is older than the SE dump, you may find files that were added properly after the dump (DANGER) + """ + se_dump = load_se_dump(se_dump_file) + se_def = load_se_definition(se_def_file) + + # Compute the PFN for each LFN in the DFC dump + + fc_dump = load_dfc_dump(fc_dump_file, "fc") + fc_dump = pd.merge(fc_dump, se_def, on="seName") + fc_dump["pfn"] = fc_dump["basePath"] + fc_dump["lfn"] + fc_dump.set_index("pfn", inplace=True) + + # Dark data: in the SE dump but not in any of the FC dump + + typer.echo(f"Computing dark data") + # Find the dark data + darkData = se_dump.index.difference(fc_dump.index) + + if len(darkData): + typer.secho(f"Found {len(darkData)} dark data, dumping them in {dark_file_output}", err=True, fg=RED) + pd.DataFrame(index=darkData).to_csv(dark_file_output) + else: + typer.secho("No dark data found", fg=GREEN) + + +@app.command() +def compare_checksum( + fc_dump_file: Annotated[Path, typer.Option(help="DFC dump")], + se_def_file: Annotated[Path, typer.Option(help="Definition of the SE path")], + se_dump_file: Annotated[Path, typer.Option(help="Dump of the SE")], + checksum_output: Annotated[ + Path, typer.Option(help="Output file in which to dump checksum difference") + ] = "cks_diff.csv", +): + """ + Compare the checksums of a DFC and an SE dump. + Careful, sometimes the cks are not padded the same way + """ + se_dump = load_se_dump(se_dump_file) + se_def = load_se_definition(se_def_file) + + # Compute the PFN for each LFN in the DFC dump + + fc_dump = load_dfc_dump(fc_dump_file, "fc") + fc_dump = pd.merge(fc_dump, se_def, on="seName") + fc_dump["pfn"] = fc_dump["basePath"] + fc_dump["lfn"] + fc_dump.set_index("pfn", inplace=True) + + typer.echo(f"Computing checksum mismath") + # Find data in both SE and FC + in_both = se_dump.index.intersection(fc_dump.index) + # Make a single DF with both info, and only keep pfn in both + joined = pd.concat([fc_dump, se_dump], axis=1) + joined = joined[joined.index.isin(in_both)] + + # Filter on non matching checksum + non_matching = joined.loc[joined["fc_cks"] != joined["se_cks"]][["seName", "lfn", "fc_cks", "se_cks"]] + + if len(non_matching): + typer.secho( + f"Found {len(non_matching)} non matching checksum, dumping them in {checksum_output}", err=True, fg=RED + ) + non_matching.to_csv(checksum_output, index=False) + else: + typer.secho("No checksum mismatch found", fg=GREEN) + + +@app.command() +def threeway( + old_fc_dump_file: Annotated[Path, typer.Option(help="DFC dump BEFORE the SE dump")], + new_fc_dump_file: Annotated[Path, typer.Option(help="DFC dump AFTER the SE dump")], + se_def_file: Annotated[Path, typer.Option(help="Definition of the SE path")], + se_dump_file: Annotated[Path, typer.Option(help="Dump of the SE")], + lost_file_output: Annotated[Path, typer.Option(help="Output file in which to dump lost files")] = "lost.csv", + dark_file_output: Annotated[Path, typer.Option(help="Output file in which to dump dark data")] = "dark.csv", +): + """ + Make a full comparison of two FC dump and one SE dump + """ + se_dump = load_se_dump(se_dump_file) + se_def = load_se_definition(se_def_file) + + # Compute the PFN for each LFN in the DFC dump + old_fc_dump = load_dfc_dump(old_fc_dump_file, "old_fc") + old_fc_dump = pd.merge(old_fc_dump, se_def, on="seName") + old_fc_dump["pfn"] = old_fc_dump["basePath"] + old_fc_dump["lfn"] + old_fc_dump.set_index("pfn", inplace=True) + + new_fc_dump = load_dfc_dump(new_fc_dump_file, "new_fc") + new_fc_dump = pd.merge(new_fc_dump, se_def, on="seName") + new_fc_dump["pfn"] = new_fc_dump["basePath"] + new_fc_dump["lfn"] + new_fc_dump.set_index("pfn", inplace=True) + + # Dark data: in the SE dump but not in any of the FC dump + + typer.echo(f"Computing dark data") + # Find the dark data + darkData = se_dump.index.difference(old_fc_dump.index.union(new_fc_dump.index)) + + if len(darkData): + typer.secho(f"Found {len(darkData)} dark data, dumping them in {dark_file_output}", err=True, fg=RED) + pd.DataFrame(index=darkData).to_csv(dark_file_output) + else: + typer.secho("No dark data found", fg=GREEN) + + # Lost files: in both FC dump but not in the SE + + lostData = (old_fc_dump.index.intersection(new_fc_dump.index)).difference(se_dump.index) + if len(lostData): + typer.secho(f"Found {len(lostData)} lost files, dumping them in {lost_file_output}", err=True, fg=RED) + lastDataDetail = new_fc_dump[new_fc_dump.index.isin(lostData)] + lastDataDetail.to_csv(lost_file_output) + else: + typer.secho("No dark data found", fg=GREEN) + + +if __name__ == "__main__": + app() diff --git a/dashboards/AgentMonitoring/AgentMonitoring.json b/dashboards/AgentMonitoring/AgentMonitoring.json new file mode 100644 index 00000000000..2071a1be2aa --- /dev/null +++ b/dashboards/AgentMonitoring/AgentMonitoring.json @@ -0,0 +1,445 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 5631, + "iteration": 1685458325330, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "elasticsearch", + "uid": "4MD8EUwVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 15, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "max", + "mean", + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "AgentName", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "4MD8EUwVz" + }, + "metrics": [ + { + "field": "CpuPercentage", + "id": "1", + "settings": { + "script": "_value / 100" + }, + "type": "sum" + } + ], + "query": "AgentName: ($AgentName)", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "CpuUsage by Agent", + "type": "timeseries" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "4MD8EUwVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 15, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "max", + "mean", + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "AgentName", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "4MD8EUwVz" + }, + "metrics": [ + { + "field": "MemoryUsage", + "id": "1", + "type": "sum" + } + ], + "query": "AgentName: ($AgentName)", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "MemoryUsage by Agent", + "type": "timeseries" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "4MD8EUwVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "seconds", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "max", + "mean", + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "AgentName", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "4MD8EUwVz" + }, + "metrics": [ + { + "field": "CycleDuration", + "id": "1", + "settings": {}, + "type": "sum" + } + ], + "query": "AgentName: ($AgentName)", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "CycleDuration by Agent", + "type": "timeseries" + } + ], + "schemaVersion": 36, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "elasticsearch", + "uid": "4MD8EUwVz" + }, + "definition": "{\"find\": \"terms\", \"field\": \"AgentName\"}", + "hide": 0, + "includeAll": true, + "label": "Agent Name", + "multi": true, + "name": "AgentName", + "options": [], + "query": "{\"find\": \"terms\", \"field\": \"AgentName\"}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Agent Monitoring", + "uid": "81IaUUQVk", + "version": 3, + "weekStart": "" +} diff --git a/dashboards/AgentMonitoring/AgentMonitoringDashboard.ndjson b/dashboards/AgentMonitoring/AgentMonitoringDashboard.ndjson index 927572585d0..673dc5f693a 100644 --- a/dashboards/AgentMonitoring/AgentMonitoringDashboard.ndjson +++ b/dashboards/AgentMonitoring/AgentMonitoringDashboard.ndjson @@ -1,6 +1,6 @@ -{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"title":"agent title","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"agent title\",\"type\":\"markdown\",\"aggs\":[],\"params\":{\"fontSize\":15,\"openLinksInNewTab\":false,\"markdown\":\"# **Agent Monitoring Dashboard**\\n\\n- Here are shown the `CPU` and `Memory Usage` of Agents, and their `Cycle Duration`\"}}"},"id":"c75816a0-c79b-11ec-b67b-89dd8cb60ff9","migrationVersion":{"visualization":"7.10.0"},"references":[],"type":"visualization","updated_at":"2022-04-29T13:38:32.168Z","version":"WzIyMjQ2LDhd"} -{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"title":"CpuUsage by Agent (AgentMonitoring)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"CpuUsage by Agent (AgentMonitoring)\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#68BC00\",\"split_mode\":\"terms\",\"split_color_mode\":\"rainbow\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"sum\",\"field\":\"CpuPercentage\"},{\"id\":\"8d9622f0-c7a9-11ec-8da2-2f902edadcbf\",\"type\":\"math\",\"variables\":[{\"id\":\"8f4be940-c7a9-11ec-8da2-2f902edadcbf\",\"name\":\"a\",\"field\":\"61ca57f2-469d-11e7-af02-69e470af7417\"}],\"script\":\"params.a/100\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"bar\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"stacked\",\"label\":\"Cpu Percentage \",\"type\":\"timeseries\",\"terms_field\":\"AgentName\",\"value_template\":\"{{value}}%\",\"terms_size\":\"20\",\"terms_order_by\":\"61ca57f2-469d-11e7-af02-69e470af7417\"}],\"time_field\":null,\"index_pattern\":\"dirac-certification_agent_monitoring-*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"dirac-certification_wmshistory_index-*\",\"default_timefield\":\"timestamp\",\"isModelInvalid\":false}}"},"id":"0d63b080-c79e-11ec-aa10-d1ab7e9f6014","migrationVersion":{"visualization":"7.10.0"},"references":[],"type":"visualization","updated_at":"2022-04-29T10:46:44.591Z","version":"WzIxNzgzLDhd"} -{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"title":"MemoryUsage by Agent (AgentMonitoring)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"MemoryUsage by Agent (AgentMonitoring)\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#68BC00\",\"split_mode\":\"terms\",\"split_color_mode\":\"kibana\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"sum\",\"field\":\"MemoryUsage\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"stacked\",\"label\":\"Memory Usage\",\"type\":\"timeseries\",\"terms_field\":\"AgentName\",\"terms_size\":\"20\",\"value_template\":\"{{value}}MB\",\"terms_order_by\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"steps\":0}],\"time_field\":\"\",\"index_pattern\":\"dirac-certification_agent_monitoring-*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"dirac-certification_wmshistory_index-*\",\"default_timefield\":\"timestamp\",\"isModelInvalid\":false}}"},"id":"2f0ffd60-c79e-11ec-b508-cbe83a8a8723","migrationVersion":{"visualization":"7.10.0"},"references":[],"type":"visualization","updated_at":"2022-04-29T09:24:34.998Z","version":"WzIxNjY4LDhd"} -{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"title":"CycleDuration by Agent (AgentMonitoring)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"CycleDuration by Agent (AgentMonitoring)\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#68BC00\",\"split_mode\":\"terms\",\"split_color_mode\":\"kibana\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"sum\",\"field\":\"CycleDuration\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"stacked\",\"label\":\"Memory Usage\",\"type\":\"timeseries\",\"terms_field\":\"AgentName\",\"terms_size\":\"20\",\"value_template\":\"{{value}}s\",\"terms_order_by\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"steps\":0}],\"time_field\":\"\",\"index_pattern\":\"dirac-certification_agent_monitoring-*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"dirac-certification_wmshistory_index-*\",\"default_timefield\":\"timestamp\",\"isModelInvalid\":false}}"},"id":"ffc85f60-c7c1-11ec-aa10-d1ab7e9f6014","migrationVersion":{"visualization":"7.10.0"},"references":[],"type":"visualization","updated_at":"2022-04-29T13:40:57.558Z","version":"WzIyMjcwLDhd"} -{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}"},"optionsJSON":"{\"hidePanelTitles\":false,\"useMargins\":true}","panelsJSON":"[{\"version\":\"7.10.2\",\"gridData\":{\"x\":15,\"y\":0,\"w\":20,\"h\":8,\"i\":\"b7b5c660-8dfc-48f0-8f15-c636bac4f635\"},\"panelIndex\":\"b7b5c660-8dfc-48f0-8f15-c636bac4f635\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_0\"},{\"version\":\"7.10.2\",\"gridData\":{\"x\":0,\"y\":8,\"w\":24,\"h\":15,\"i\":\"7201d0bc-9746-4d2c-8230-a2a4ac291790\"},\"panelIndex\":\"7201d0bc-9746-4d2c-8230-a2a4ac291790\",\"embeddableConfig\":{},\"panelRefName\":\"panel_1\"},{\"version\":\"7.10.2\",\"gridData\":{\"x\":24,\"y\":8,\"w\":24,\"h\":15,\"i\":\"fb0d433e-b223-40e5-a51e-27c54d6713f3\"},\"panelIndex\":\"fb0d433e-b223-40e5-a51e-27c54d6713f3\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"7.10.2\",\"gridData\":{\"x\":13,\"y\":23,\"w\":24,\"h\":15,\"i\":\"a61113e4-32d4-4142-9aa8-f87a51c0a765\"},\"panelIndex\":\"a61113e4-32d4-4142-9aa8-f87a51c0a765\",\"embeddableConfig\":{},\"panelRefName\":\"panel_3\"}]","refreshInterval":{"pause":true,"value":0},"timeFrom":"now-24h","timeRestore":true,"timeTo":"now","title":"Agent Monitoring Dashboard","version":1},"id":"708821e0-c79f-11ec-b508-cbe83a8a8723","migrationVersion":{"dashboard":"7.9.3"},"references":[{"id":"c75816a0-c79b-11ec-b67b-89dd8cb60ff9","name":"panel_0","type":"visualization"},{"id":"0d63b080-c79e-11ec-aa10-d1ab7e9f6014","name":"panel_1","type":"visualization"},{"id":"2f0ffd60-c79e-11ec-b508-cbe83a8a8723","name":"panel_2","type":"visualization"},{"id":"ffc85f60-c7c1-11ec-aa10-d1ab7e9f6014","name":"panel_3","type":"visualization"}],"type":"dashboard","updated_at":"2022-04-29T13:43:17.454Z","version":"WzIyMjc1LDhd"} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"title":"agent title","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"agent title\",\"type\":\"markdown\",\"aggs\":[],\"params\":{\"fontSize\":15,\"openLinksInNewTab\":false,\"markdown\":\"# **Agent Monitoring Dashboard**\\n\\n- Here are shown the `CPU` and `Memory Usage` of Agents, and their `Cycle Duration`\"}}"},"id":"c75816a0-c79b-11ec-b67b-89dd8cb60ff9","migrationVersion":{"visualization":"7.10.0"},"references":[],"type":"visualization","updated_at":"2022-07-28T10:43:48.809Z","version":"WzIyOSwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"title":"CpuUsage by Agent (AgentMonitoring) (using environment agnostic index pattern))","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"CpuUsage by Agent (AgentMonitoring) (using environment agnostic index pattern))\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#68BC00\",\"split_mode\":\"terms\",\"split_color_mode\":\"rainbow\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"sum\",\"field\":\"CpuPercentage\"},{\"id\":\"8d9622f0-c7a9-11ec-8da2-2f902edadcbf\",\"type\":\"math\",\"variables\":[{\"id\":\"8f4be940-c7a9-11ec-8da2-2f902edadcbf\",\"name\":\"a\",\"field\":\"61ca57f2-469d-11e7-af02-69e470af7417\"}],\"script\":\"params.a/100\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"bar\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"stacked\",\"label\":\"Cpu Percentage \",\"type\":\"timeseries\",\"terms_field\":\"AgentName\",\"value_template\":\"{{value}}%\",\"terms_size\":\"200\",\"terms_order_by\":\"61ca57f2-469d-11e7-af02-69e470af7417\"}],\"time_field\":\"timestamp\",\"index_pattern\":\"*_agent_monitoring-*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"lhcb-certification_wmshistory_index-*\",\"default_timefield\":\"timestamp\",\"isModelInvalid\":false}}"},"id":"9bac3ff0-b2c2-11ed-baf6-bd7d09f916cf","migrationVersion":{"visualization":"7.10.0"},"references":[],"type":"visualization","updated_at":"2023-02-22T15:07:22.350Z","version":"WzQ3OCw0XQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"title":"MemoryUsage by Agent (AgentMonitoring) (using environment agnostic index pattern)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"MemoryUsage by Agent (AgentMonitoring) (using environment agnostic index pattern)\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#68BC00\",\"split_mode\":\"terms\",\"split_color_mode\":\"kibana\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"sum\",\"field\":\"MemoryUsage\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"stacked\",\"label\":\"Memory Usage\",\"type\":\"timeseries\",\"terms_field\":\"AgentName\",\"terms_size\":\"200\",\"value_template\":\"{{value}}MB\",\"terms_order_by\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"steps\":0}],\"time_field\":\"timestamp\",\"index_pattern\":\"*_agent_monitoring-*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"lhcb-certification_wmshistory_index-*\",\"default_timefield\":\"timestamp\",\"isModelInvalid\":false}}"},"id":"e905e2b0-b2c2-11ed-baf6-bd7d09f916cf","migrationVersion":{"visualization":"7.10.0"},"references":[],"type":"visualization","updated_at":"2023-02-22T15:09:32.123Z","version":"WzQ3OSw0XQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"title":"CycleDuration by Agent (AgentMonitoring) (using environment agnostic index pattern)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"CycleDuration by Agent (AgentMonitoring) (using environment agnostic index pattern)\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#68BC00\",\"split_mode\":\"terms\",\"split_color_mode\":\"kibana\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"sum\",\"field\":\"CycleDuration\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"stacked\",\"label\":\"Memory Usage\",\"type\":\"timeseries\",\"terms_field\":\"AgentName\",\"terms_size\":\"200\",\"value_template\":\"{{value}}s\",\"terms_order_by\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"steps\":0}],\"time_field\":\"timestamp\",\"index_pattern\":\"*_agent_monitoring-*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"lhcb-certification_wmshistory_index-*\",\"default_timefield\":\"timestamp\",\"isModelInvalid\":false}}"},"id":"278e56c0-b2c3-11ed-baf6-bd7d09f916cf","migrationVersion":{"visualization":"7.10.0"},"references":[],"type":"visualization","updated_at":"2023-02-22T15:11:17.036Z","version":"WzQ4MCw0XQ=="} +{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}"},"optionsJSON":"{\"hidePanelTitles\":false,\"useMargins\":true}","panelsJSON":"[{\"version\":\"2.2.1\",\"gridData\":{\"h\":8,\"i\":\"b7b5c660-8dfc-48f0-8f15-c636bac4f635\",\"w\":20,\"x\":15,\"y\":0},\"panelIndex\":\"b7b5c660-8dfc-48f0-8f15-c636bac4f635\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_0\"},{\"version\":\"2.2.1\",\"gridData\":{\"h\":15,\"i\":\"0bc8cbb1-e1ee-4825-9623-35bad2c63212\",\"w\":24,\"x\":0,\"y\":8},\"panelIndex\":\"0bc8cbb1-e1ee-4825-9623-35bad2c63212\",\"embeddableConfig\":{},\"panelRefName\":\"panel_1\"},{\"version\":\"2.2.1\",\"gridData\":{\"h\":15,\"i\":\"58e7e1c1-5901-4a55-afa8-82b2712dc7ce\",\"w\":24,\"x\":24,\"y\":8},\"panelIndex\":\"58e7e1c1-5901-4a55-afa8-82b2712dc7ce\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"2.2.1\",\"gridData\":{\"x\":13,\"y\":23,\"w\":24,\"h\":15,\"i\":\"a738300c-84f3-456b-a696-90de9ed9d4ac\"},\"panelIndex\":\"a738300c-84f3-456b-a696-90de9ed9d4ac\",\"embeddableConfig\":{},\"panelRefName\":\"panel_3\"}]","timeRestore":false,"title":"Agent Monitoring Dashboard (using environment agnostic index pattern)","version":1},"id":"d09cc280-b2c1-11ed-baf6-bd7d09f916cf","migrationVersion":{"dashboard":"7.9.3"},"references":[{"id":"c75816a0-c79b-11ec-b67b-89dd8cb60ff9","name":"panel_0","type":"visualization"},{"id":"9bac3ff0-b2c2-11ed-baf6-bd7d09f916cf","name":"panel_1","type":"visualization"},{"id":"e905e2b0-b2c2-11ed-baf6-bd7d09f916cf","name":"panel_2","type":"visualization"},{"id":"278e56c0-b2c3-11ed-baf6-bd7d09f916cf","name":"panel_3","type":"visualization"}],"type":"dashboard","updated_at":"2023-02-22T15:11:54.150Z","version":"WzQ4MSw0XQ=="} {"exportedCount":5,"missingRefCount":0,"missingReferences":[]} diff --git a/dashboards/DataOperation/grafana/DMSOverview.json b/dashboards/DataOperation/grafana/DMSOverview.json new file mode 100644 index 00000000000..d1a24a57453 --- /dev/null +++ b/dashboards/DataOperation/grafana/DMSOverview.json @@ -0,0 +1,4855 @@ +{ + "__inputs": [ + { + "name": "DS_OPENSEARCH", + "label": "OpenSearch", + "description": "", + "type": "datasource", + "pluginId": "grafana-opensearch-datasource", + "pluginName": "OpenSearch" + }, + { + "name": "DS_EXPRESSION", + "label": "Expression", + "description": "", + "type": "datasource", + "pluginId": "__expr__" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "datasource", + "id": "__expr__", + "version": "1.0.0" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.1.6" + }, + { + "type": "datasource", + "id": "grafana-opensearch-datasource", + "name": "OpenSearch", + "version": "2.0.1" + }, + { + "type": "panel", + "id": "piechart", + "name": "Pie chart", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Overview of the ongoing Data Management operations", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 50, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Destination", + "id": "3", + "settings": { + "min_doc_count": "0", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": true, + "metrics": [ + { + "field": "TransferSize", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:Successful AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user} AND Destination:${destination}", + "queryType": "lucene", + "refId": "TransferSize", + "timeField": "timestamp" + }, + { + "alias": "", + "bucketAggs": [ + { + "field": "Destination", + "id": "3", + "settings": { + "min_doc_count": "0", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": true, + "metrics": [ + { + "field": "TransferTime", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:Successful AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user} AND Destination:${destination}", + "queryType": "lucene", + "refId": "TransferTime", + "timeField": "timestamp" + }, + { + "datasource": { + "type": "__expr__", + "uid": "${DS_EXPRESSION}" + }, + "expression": "$TransferSize/$TransferTime", + "hide": false, + "refId": "A", + "type": "math" + } + ], + "title": "Panel Title", + "transformations": [], + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 12, + "panels": [], + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "Final status of se.putFile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 0, + "y": 9 + }, + "id": 4, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "FinalStatus", + "id": "2", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "20" + }, + "type": "terms" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND Destination:${destination} AND ExecutionSite:${executionSite} AND FinalStatus:${finalStatus} AND Protocol:${protocol} AND User:${user}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Data upload final status", + "type": "piechart" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "Final status of se.putFile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 4, + "y": 9 + }, + "id": 16, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Protocol", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:Failed AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Failed Data upload by Protocol", + "type": "piechart" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "Final status of se.putFile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 8, + "y": 9 + }, + "id": 19, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Protocol", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:Successful AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Successful Data upload by Protocol", + "type": "piechart" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "Final status of se.putFile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 12, + "y": 9 + }, + "id": 22, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "User", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:Successful AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Successful Data upload by User", + "type": "piechart" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 16, + "y": 9 + }, + "id": 23, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "User", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:Failed AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Failed Data upload by User", + "type": "piechart" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 52, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Destination", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "time_series", + "hide": false, + "metrics": [ + { + "field": "TransferSize", + "hide": true, + "id": "1", + "type": "sum" + }, + { + "field": "1", + "id": "4", + "type": "cumulative_sum" + } + ], + "query": "OperationType:se.putFile AND Destination:${destination} AND ExecutionSite:${executionSite} AND FinalStatus:${finalStatus} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Cumulative uploaded data", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 53, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "ExecutionSite", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "time_series", + "hide": false, + "metrics": [ + { + "field": "TransferSize", + "hide": true, + "id": "1", + "type": "sum" + }, + { + "field": "1", + "id": "4", + "type": "cumulative_sum" + } + ], + "query": "OperationType:se.putFile AND Destination:${destination} AND ExecutionSite:${executionSite} AND FinalStatus:${finalStatus} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Cumulative uploaded data", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "Final status of se.putFile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 0, + "y": 24 + }, + "id": 39, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "FinalStatus", + "id": "2", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "20" + }, + "type": "terms" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.getFile AND Destination:${destination} AND ExecutionSite:${executionSite} AND FinalStatus:${finalStatus} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Data download final status", + "type": "piechart" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 4, + "y": 24 + }, + "id": 40, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Protocol", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.getFile AND FinalStatus:Failed AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Failed Data Download by Protocol", + "type": "piechart" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 8, + "y": 24 + }, + "id": 45, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Protocol", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.getFile AND FinalStatus:Successful AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Successful Data download by Protocol", + "type": "piechart" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 12, + "y": 24 + }, + "id": 42, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "User", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.getFile AND FinalStatus:Successful AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user}AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Successful Data download by User", + "type": "piechart" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 16, + "y": 24 + }, + "id": 43, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "User", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.getFile AND FinalStatus:Failed AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Failed Data download by User", + "type": "piechart" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 54, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Source", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "time_series", + "hide": false, + "metrics": [ + { + "field": "TransferSize", + "hide": true, + "id": "1", + "type": "sum" + }, + { + "field": "1", + "id": "4", + "type": "cumulative_sum" + } + ], + "query": "OperationType:se.getFile AND ExecutionSite:${executionSite} AND FinalStatus:${finalStatus} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Cumulative downloaded data (source SE)", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 55, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "ExecutionSite", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "time_series", + "hide": false, + "metrics": [ + { + "field": "TransferSize", + "hide": true, + "id": "1", + "type": "sum" + }, + { + "field": "1", + "id": "4", + "type": "cumulative_sum" + } + ], + "query": "OperationType:se.getFile AND ExecutionSite:${executionSite} AND FinalStatus:${finalStatus} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Cumulative downloaded data (execution site)", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "Final status of se.putFile", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 0, + "y": 39 + }, + "id": 44, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "FinalStatus", + "id": "2", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "20" + }, + "type": "terms" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.removeFile AND Destination:${destination} AND ExecutionSite:${executionSite} AND FinalStatus:${finalStatus} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Removal final status", + "type": "piechart" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 4, + "y": 39 + }, + "id": 46, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Protocol", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.removeFile AND FinalStatus:Failed AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Failed removal by Protocol", + "type": "piechart" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 8, + "y": 39 + }, + "id": 41, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Protocol", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.removeFile AND FinalStatus:Successful AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Successful removal by Protocol", + "type": "piechart" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 12, + "y": 39 + }, + "id": 47, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "User", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.removeFile AND FinalStatus:Successful AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user}AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Successful removal by User", + "type": "piechart" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 16, + "y": 39 + }, + "id": 48, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true, + "values": [] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "User", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.removeFile AND FinalStatus:Failed AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Failed removal by User", + "type": "piechart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 46 + }, + "id": 25, + "panels": [], + "title": "Data download", + "type": "row" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "asc" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "FinalStatus", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.getFile AND Destination:${destination} AND ExecutionSite:${executionSite} AND FinalStatus:${finalStatus} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": " Data download by status", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 29, + "options": { + "legend": { + "calcs": [ + "max", + "min", + "mean" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "id": "1", + "settings": { + "size": "500" + }, + "type": "raw_data" + } + ], + "query": "OperationType:se.getFile AND FinalStatus:Successful AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Successful download throughput", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "Throughput", + "binary": { + "left": "TransferSize", + "operator": "/", + "reducer": "sum", + "right": "TransferTime" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 56 + }, + "id": 31, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Source", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.getFile AND FinalStatus:Successful AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Successful download by source", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 56 + }, + "id": 33, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Source", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.getFile AND FinalStatus:Failed AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Failed download by source", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 64 + }, + "id": 34, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Protocol", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.getFile AND FinalStatus:Successful AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Successful download by protocol", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 64 + }, + "id": 36, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Protocol", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.getFile AND FinalStatus:Failed AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Failed download by Protocol", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 72 + }, + "id": 32, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "User", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.getFile AND FinalStatus:Successful AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Successful download by User", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 72 + }, + "id": 35, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "User", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.getFile AND FinalStatus:Failed AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Failed download by user", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 80 + }, + "id": 37, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "ExecutionSite", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.getFile AND FinalStatus:Successful AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Successful download by ExecutionSite", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 80 + }, + "id": 38, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "ExecutionSite", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.getFile AND FinalStatus:Failed AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Failed download by ExecutionSite", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 88 + }, + "id": 6, + "panels": [], + "title": "Data Upload", + "type": "row" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 89 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "asc" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "FinalStatus", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND Destination:${destination} AND ExecutionSite:${executionSite} AND FinalStatus:${finalStatus} AND Protocol:${protocol} AND User:${user}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Data Upload by Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 89 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "max", + "min", + "mean" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "id": "1", + "settings": { + "size": "500" + }, + "type": "raw_data" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:Successful AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Successful upload throughput", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "Throughput", + "binary": { + "left": "TransferSize", + "operator": "/", + "reducer": "sum", + "right": "TransferTime" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 98 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Destination", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:Successful AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Successful upload by destination", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 98 + }, + "id": 20, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Destination", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:Failed AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Failed upload by destination", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 106 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Protocol", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "20" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:Successful AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Successful upload by protocol", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 106 + }, + "id": 15, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Protocol", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "20" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:Failed AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Failed upload by protocol", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 114 + }, + "id": 17, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "User", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "20" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:Successful AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Successful upload by User", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 114 + }, + "id": 18, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "User", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "20" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:Failed AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Failed upload by User", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 122 + }, + "id": 21, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "ExecutionSite", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "20" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:Successful AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Failed upload by ExecutionSite", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 122 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "ExecutionSite", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "20" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:Failed AND Destination:${destination} AND ExecutionSite:${executionSite} AND Protocol:${protocol} AND User:${user}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Failed upload by ExecutionSite", + "type": "timeseries" + } + ], + "refresh": false, + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": "*", + "current": {}, + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "definition": "{\"find\": \"terms\", \"field\": \"Destination\" , \"size\":10000}", + "hide": 0, + "includeAll": true, + "label": "Destination", + "multi": true, + "name": "destination", + "options": [], + "query": "{\"find\": \"terms\", \"field\": \"Destination\" , \"size\":10000}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": "*", + "current": {}, + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "definition": "{\"find\": \"terms\", \"field\": \"ExecutionSite\" , \"size\":10000}", + "hide": 0, + "includeAll": true, + "label": "ExecutionSite", + "multi": true, + "name": "executionSite", + "options": [], + "query": "{\"find\": \"terms\", \"field\": \"ExecutionSite\" , \"size\":10000}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": "*", + "current": {}, + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "definition": "{\"find\": \"terms\", \"field\": \"User\" , \"size\":10000}", + "hide": 0, + "includeAll": true, + "label": "User", + "multi": true, + "name": "user", + "options": [], + "query": "{\"find\": \"terms\", \"field\": \"User\" , \"size\":10000}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": "*", + "current": {}, + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "definition": "{\"find\": \"terms\", \"field\": \"Protocol\" , \"size\":10000 , \"query\":\"OperationType:se*\"}", + "description": "All the protocols used by the StorageElement", + "hide": 0, + "includeAll": true, + "label": "Protocol", + "multi": true, + "name": "protocol", + "options": [], + "query": "{\"find\": \"terms\", \"field\": \"Protocol\" , \"size\":10000 , \"query\":\"OperationType:se*\"}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": "*", + "current": {}, + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "definition": "{\"find\": \"terms\", \"field\": \"Source\" , \"size\":10000}", + "hide": 0, + "includeAll": true, + "label": "Source", + "multi": true, + "name": "source", + "options": [], + "query": "{\"find\": \"terms\", \"field\": \"Source\" , \"size\":10000}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": "*", + "current": {}, + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "definition": "{\"find\": \"terms\", \"field\": \"FinalStatus\" , \"size\":10000}", + "hide": 0, + "includeAll": true, + "label": "Status", + "multi": true, + "name": "finalStatus", + "options": [], + "query": "{\"find\": \"terms\", \"field\": \"FinalStatus\" , \"size\":10000}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": "*", + "current": {}, + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "definition": "{\"find\": \"terms\", \"field\": \"OperationType\" , \"size\":10000}", + "hide": 0, + "includeAll": true, + "label": "Operation", + "multi": true, + "name": "operationType", + "options": [], + "query": "{\"find\": \"terms\", \"field\": \"OperationType\" , \"size\":10000}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Data Management overview", + "uid": "Ix04HM4Vz", + "version": 32, + "weekStart": "" +} diff --git a/dashboards/DataOperation/grafana/Tier1DMS.json b/dashboards/DataOperation/grafana/Tier1DMS.json new file mode 100644 index 00000000000..ccb8b378cbc --- /dev/null +++ b/dashboards/DataOperation/grafana/Tier1DMS.json @@ -0,0 +1,1905 @@ +{ + "__inputs": [ + { + "name": "DS_OPENSEARCH", + "label": "OpenSearch", + "description": "", + "type": "datasource", + "pluginId": "grafana-opensearch-datasource", + "pluginName": "OpenSearch" + }, + { + "name": "DS_EXPRESSION", + "label": "Expression", + "description": "", + "type": "datasource", + "pluginId": "__expr__" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "datasource", + "id": "__expr__", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "flant-statusmap-panel", + "name": "Statusmap", + "version": "0.5.1" + }, + { + "type": "panel", + "id": "gauge", + "name": "Gauge", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.1.6" + }, + { + "type": "datasource", + "id": "grafana-opensearch-datasource", + "name": "OpenSearch", + "version": "2.0.1" + }, + { + "type": "panel", + "id": "heatmap", + "name": "Heatmap", + "version": "" + }, + { + "type": "panel", + "id": "piechart", + "name": "Pie chart", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + }, + { + "type": "panel", + "id": "vonage-status-panel", + "name": "Status Panel", + "version": "1.0.11" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 451, + "panels": [], + "title": "Upload from site overwiew", + "type": "row" + }, + { + "clusterName": "", + "colorMode": "Panel", + "colors": { + "crit": "rgba(245, 54, 54, 0.9)", + "disable": "rgba(128, 128, 128, 0.9)", + "ok": "rgba(50, 128, 45, 0.9)", + "warn": "rgba(237, 129, 40, 0.9)" + }, + "cornerRadius": 0, + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "displayName": "", + "flipCard": true, + "flipTime": 5, + "fontFormat": "Regular", + "gridPos": { + "h": 8, + "w": 3.4285714285714284, + "x": 0, + "y": 1 + }, + "id": 525, + "isAutoScrollOnOverflow": false, + "isGrayOnNoData": false, + "isHideAlertsOnDisable": false, + "isIgnoreOKColors": false, + "maxPerRow": 12, + "pluginVersion": "9.1.6", + "repeat": "site", + "repeatDirection": "h", + "targets": [ + { + "$$hashKey": "object:24", + "aggregation": "Avg", + "alias": "Status", + "bucketAggs": [ + { + "field": "timestamp", + "id": "5", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "crit": 0.8, + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "decimals": 2, + "displayAliasType": "Always", + "displayType": "Regular", + "displayValueWithAlias": "When Alias Displayed", + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "hide": true, + "id": "1", + "type": "sum" + }, + { + "field": "TransferOK", + "hide": true, + "id": "6", + "type": "sum" + }, + { + "id": "7", + "pipelineVariables": [ + { + "name": "all", + "pipelineAgg": "1" + }, + { + "name": "ok", + "pipelineAgg": "6" + } + ], + "settings": { + "script": "params.ok / params.all" + }, + "type": "bucket_script" + } + ], + "query": "OperationType:se.putFile AND Destination:${destination} AND ExecutionSite:${site} AND FinalStatus:${finalStatus} AND Protocol:${protocol} AND User:${user}", + "queryType": "lucene", + "refId": "TransferTotal", + "timeField": "timestamp", + "units": "percentunit", + "valueHandler": "Number Threshold", + "warn": 0.95 + } + ], + "title": "Upload status from ${site}", + "type": "vonage-status-panel" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 2, + "panels": [], + "repeat": "site", + "repeatDirection": "h", + "title": "${site}", + "type": "row" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Destination", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:Successful AND ExecutionSite:${site} AND Protocol:${protocol} AND User:${user} AND Destination:${destination}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Successful upload from ${site}", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Destination", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:Failed AND Destination:${destination} AND ExecutionSite:${site} AND Protocol:${protocol} AND User:${user}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Failed upload from ${site}", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 24, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Source", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.getFile AND FinalStatus:Successful AND ExecutionSite:${site} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Successful download from ${site}", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 25, + "options": { + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Source", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.getFile AND FinalStatus:Failed AND ExecutionSite:${site} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Failed download from ${site}", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-RdYlGr" + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 70 + }, + { + "color": "red", + "value": 85 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 100, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Protocol", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "FinalStatus", + "id": "4", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.getFile AND FinalStatus:Successful AND ExecutionSite:${site} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Download by protocol at ${site}", + "type": "gauge" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-RdYlGr" + }, + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 70 + }, + { + "color": "red", + "value": 85 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 101, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "sum" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Protocol", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "FinalStatus", + "id": "4", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND ExecutionSite:${site} AND Protocol:${protocol} AND User:${user} AND Destination:${destination} AND FinalStatus:${finalStatus}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Upload by protocol at ${site}", + "type": "gauge" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 4.8, + "x": 0, + "y": 34 + }, + "id": 156, + "maxPerRow": 8, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "percent" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "repeat": "protocol", + "repeatDirection": "h", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "FinalStatus", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND ExecutionSite:${site} AND Protocol:${protocol} AND User:${user} AND Destination:${destination} AND FinalStatus:${finalStatus}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "${protocol} upload performance at ${site}", + "type": "piechart" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 4.8, + "x": 0, + "y": 42 + }, + "id": 245, + "maxPerRow": 8, + "options": { + "displayLabels": [ + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "percent" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.6", + "repeat": "protocol", + "repeatDirection": "h", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "FinalStatus", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferTotal", + "id": "1", + "type": "sum" + } + ], + "query": "OperationType:se.getFile AND ExecutionSite:${site} AND Protocol:${protocol} AND User:${user} AND Source:${source} AND FinalStatus:${finalStatus}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "${protocol} download performance at ${site}", + "type": "piechart" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 50 + }, + "id": 57, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "hide": false, + "id": "1", + "settings": { + "size": "500" + }, + "type": "raw_data" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:Successful AND ExecutionSite:${site} AND Protocol:${protocol} AND User:${user} AND Destination:${destination}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Upload throughput from ${site}", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "Throughput", + "binary": { + "left": "TransferSize", + "operator": "/", + "reducer": "sum", + "right": "TransferTime" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": false + } + }, + { + "id": "groupBy", + "options": { + "fields": { + "Destination": { + "aggregations": [], + "operation": "groupby" + }, + "Source": { + "aggregations": [] + }, + "Throughput": { + "aggregations": [ + "last" + ], + "operation": "aggregate" + } + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 50 + }, + "id": 56, + "options": { + "footer": { + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "hide": false, + "id": "1", + "settings": { + "size": "500" + }, + "type": "raw_data" + } + ], + "query": "OperationType:se.getFile AND FinalStatus:Successful AND ExecutionSite:${site} AND Protocol:${protocol} AND User:${user} AND Source:${source}", + "queryType": "lucene", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Download throughput from ${site}", + "transformations": [ + { + "id": "calculateField", + "options": { + "alias": "Throughput", + "binary": { + "left": "TransferSize", + "operator": "/", + "reducer": "sum", + "right": "TransferTime" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + }, + "replaceFields": false + } + }, + { + "id": "groupBy", + "options": { + "fields": { + "Source": { + "aggregations": [], + "operation": "groupby" + }, + "Throughput": { + "aggregations": [ + "last" + ], + "operation": "aggregate" + } + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 58 + }, + "id": 377, + "options": { + "calculate": false, + "cellGap": 1, + "cellValues": { + "unit": "percentunit" + }, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "max": 1, + "min": 0, + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlGn", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": -1 + }, + "legend": { + "show": false + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "show": true, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": true + } + }, + "pluginVersion": "9.1.6", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Destination", + "id": "6", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "1", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": true, + "metrics": [ + { + "field": "TransferOK", + "id": "4", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:${finalStatus} AND ExecutionSite:${site} AND Protocol:${protocol} AND User:${user} AND Destination:${destination}", + "queryType": "lucene", + "refId": "TransferOK", + "timeField": "timestamp" + }, + { + "alias": "", + "bucketAggs": [ + { + "field": "Destination", + "id": "6", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "1", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": true, + "metrics": [ + { + "field": "TransferTotal", + "id": "5", + "type": "sum" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:${finalStatus} AND ExecutionSite:${site} AND Protocol:${protocol} AND User:${user}", + "queryType": "lucene", + "refId": "TransferTotal", + "timeField": "timestamp" + }, + { + "datasource": { + "type": "__expr__", + "uid": "${DS_EXPRESSION}" + }, + "expression": "1-$TransferOK/$TransferTotal", + "hide": false, + "refId": "To", + "type": "math" + } + ], + "title": "Upload from ${site}", + "transformations": [], + "type": "heatmap" + }, + { + "cards": { + "cardHSpacing": 2, + "cardMinWidth": 5, + "cardVSpacing": 2 + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateGnYlRd", + "defaultColor": "#757575", + "exponent": 0.5, + "mode": "spectrum", + "thresholds": [] + }, + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 58 + }, + "hideBranding": false, + "highlightCards": true, + "id": 612, + "legend": { + "show": true + }, + "nullPointMode": "as empty", + "pageSize": 15, + "pluginVersion": "9.1.6", + "seriesFilterIndex": -1, + "statusmap": { + "ConfigVersion": "v1" + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Destination", + "id": "6", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "9", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "format": "table", + "hide": false, + "metrics": [ + { + "field": "TransferOK", + "hide": true, + "id": "4", + "settings": { + "missing": "-1" + }, + "type": "sum" + }, + { + "field": "TransferTotal", + "hide": true, + "id": "7", + "type": "sum" + }, + { + "id": "8", + "pipelineVariables": [ + { + "name": "ok", + "pipelineAgg": "4" + }, + { + "name": "all", + "pipelineAgg": "7" + } + ], + "settings": { + "script": "params.ok / params.all" + }, + "type": "bucket_script" + } + ], + "query": "OperationType:se.putFile AND FinalStatus:${finalStatus} AND ExecutionSite:${site} AND Protocol:${protocol} AND User:${user} AND Destination:${destination}", + "queryType": "lucene", + "refId": "TransferOK", + "timeField": "timestamp" + } + ], + "title": "Upload from ${site}", + "tooltip": { + "extraInfo": "", + "freezeOnClick": true, + "items": [], + "show": true, + "showExtraInfo": false, + "showItems": false + }, + "transformations": [], + "type": "flant-statusmap-panel", + "useMax": true, + "usingPagination": false, + "xAxis": { + "show": true + }, + "yAxis": { + "maxWidth": -1, + "minWidth": -1, + "show": true + }, + "yAxisSort": "metrics", + "yLabel": { + "delimiter": "", + "labelTemplate": "", + "usingSplitLabel": false + } + } + ], + "refresh": false, + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": {}, + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "definition": "{\"find\": \"terms\", \"field\": \"Destination\" , \"size\":10000}", + "hide": 0, + "includeAll": true, + "label": "Destination", + "multi": true, + "name": "destination", + "options": [], + "query": "{\"find\": \"terms\", \"field\": \"Destination\" , \"size\":10000}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "definition": "{\"find\": \"terms\", \"field\": \"ExecutionSite\" , \"size\":10000}", + "hide": 0, + "includeAll": true, + "label": "ExecutionSite", + "multi": true, + "name": "executionSite", + "options": [], + "query": "{\"find\": \"terms\", \"field\": \"ExecutionSite\" , \"size\":10000}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "definition": "{\"find\": \"terms\", \"field\": \"User\" , \"size\":10000}", + "hide": 0, + "includeAll": true, + "label": "User", + "multi": true, + "name": "user", + "options": [], + "query": "{\"find\": \"terms\", \"field\": \"User\" , \"size\":10000}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "definition": "{\"find\": \"terms\", \"field\": \"Protocol\" , \"size\":10000 , \"query\":\"OperationType:se*\"}", + "hide": 0, + "includeAll": true, + "label": "Protocol", + "multi": true, + "name": "protocol", + "options": [], + "query": "{\"find\": \"terms\", \"field\": \"Protocol\" , \"size\":10000 , \"query\":\"OperationType:se*\"}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "definition": "{\"find\": \"terms\", \"field\": \"Source\" , \"size\":10000}", + "hide": 0, + "includeAll": true, + "label": "Source", + "multi": true, + "name": "source", + "options": [], + "query": "{\"find\": \"terms\", \"field\": \"Source\" , \"size\":10000}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "definition": "{\"find\": \"terms\", \"field\": \"FinalStatus\" , \"size\":10000}", + "hide": 0, + "includeAll": true, + "label": "Status", + "multi": true, + "name": "finalStatus", + "options": [], + "query": "{\"find\": \"terms\", \"field\": \"FinalStatus\" , \"size\":10000}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "grafana-opensearch-datasource", + "uid": "${DS_OPENSEARCH}" + }, + "definition": "{\"find\": \"terms\", \"field\": \"OperationType\" , \"size\":10000}", + "hide": 0, + "includeAll": true, + "label": "Operation", + "multi": true, + "name": "operationType", + "options": [], + "query": "{\"find\": \"terms\", \"field\": \"OperationType\" , \"size\":10000}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "allValue": "*", + "current": { + "selected": false, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "hide": 2, + "includeAll": true, + "label": "Site", + "multi": true, + "name": "site", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "LCG.CERN.cern", + "value": "LCG.CERN.cern" + }, + { + "selected": false, + "text": "LCG.CNAF.it", + "value": "LCG.CNAF.it" + }, + { + "selected": false, + "text": "LCG.IN2P3.fr", + "value": "LCG.IN2P3.fr" + }, + { + "selected": false, + "text": "LCG.GRIDKA.de", + "value": "LCG.GRIDKA.de" + }, + { + "selected": false, + "text": "LCG.PIC.es", + "value": "LCG.PIC.es" + }, + { + "selected": false, + "text": "LCG.RAL.uk", + "value": "LCG.RAL.uk" + }, + { + "selected": false, + "text": "LCG.RRCKI.ru", + "value": "LCG.RRCKI.ru" + } + ], + "query": "LCG.CERN.cern,LCG.CNAF.it,LCG.IN2P3.fr,LCG.GRIDKA.de,LCG.PIC.es,LCG.RAL.uk,LCG.RRCKI.ru", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-7d", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Tier1 DMS", + "uid": "XVEc6G4Vk", + "version": 55, + "weekStart": "" +} diff --git a/dashboards/GrafanaDemo/README.md b/dashboards/GrafanaDemo/README.md new file mode 100644 index 00000000000..26d9aed9fc9 --- /dev/null +++ b/dashboards/GrafanaDemo/README.md @@ -0,0 +1,25 @@ +# Grafana Demo + +A running example of this dashboard can be found on the LHCb grafana organisation in the Dev Area with the name `Demo Dashboard`. + +## What plots does the dashboard contain? +- Production Jobs By Final Minor Status (MySQL) +- User Jobs By Final Minor Status (MySQL) +- Job CPU Efficiency By Job Type (MySQL) +- User Jobs By Job Type (OpenSearch) +- User Jobs By Final Minor Status (MySQL) +- Pilots By Status (MySQL) + +## How to import this dashboard into another Grafana installation? +- Create Grafana data sources for the MySQL database and OpenSearch cluster (both the DIRAC certification versions). +- Create a new dashboard and in 'Dashboard settings' under 'JSON model' replace the JSON representation of the dashboard with this one. +- In the JSON file, replace the datasource uid's with the ones for the data sources in your grafana installation. You can find the UID of a data source: + - with the [API](https://grafana.com/docs/grafana/latest/developers/http_api/data_source/) + - by creating a new visualisation using this data source and looking into the JSON model of the visualisation (right click the title > Inspect > Panel JSON) + +``` +"datasource": { + "type": "mysql", + "uid": "R3A-SHw7z" <- CHANGE THIS VALUE TO THE ONE FOR YOUR INSTALLATION + }, +``` diff --git a/dashboards/GrafanaDemo/demo.json b/dashboards/GrafanaDemo/demo.json new file mode 100644 index 00000000000..9167ac6bd19 --- /dev/null +++ b/dashboards/GrafanaDemo/demo.json @@ -0,0 +1,697 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 5448, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "mysql", + "uid": "R3A-SHw7z" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 16, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "last", + "max", + "min", + "mean" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "mysql", + "uid": "R3A-SHw7z" + }, + "format": "time_series", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT\nac_key_Job_FinalMinorStatus.value,\nac_bucket_Job.startTime as time,\nSUM(ac_bucket_Job.entriesInBucket) / ac_bucket_Job.bucketLength * 3600 as metric\nFROM\nac_bucket_Job,\nac_key_Job_JobType,\nac_key_Job_FinalMinorStatus\nWHERE ac_bucket_Job.startTime >= $__unixEpochFrom()\nAND ac_bucket_Job.startTime <= $__unixEpochTo()\nAND ( ac_key_Job_JobType.value in (\"10\", \"DataReconstruction\", \"DataReprocessing\", \"DataStripping\", \"DataSwimming\", \"Hospital\", \"MCReconstruction\", \"MCReprocessing\", \"MCSimulation\", \"MCStripping\", \"Merge\", \"MergeMDF\" ,\"private\", \"production\", \"reconstruction\", \"sam\", \"test\", \"unknown\", \"WGProduction\") )\nAND ac_bucket_Job.FinalMinorStatus = ac_key_Job_FinalMinorStatus.id\nAND ac_bucket_Job.JobType = ac_key_Job_JobType.id\nGROUP BY\nstartTime,\nac_key_Job_FinalMinorStatus.value,\nbucketlength", + "refId": "A" + } + ], + "title": "Production Jobs By Final Minor Status (MySQL)", + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "metric ", + "renamePattern": "" + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "mysql", + "uid": "R3A-SHw7z" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 16, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "last", + "max", + "min", + "mean" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "mysql", + "uid": "R3A-SHw7z" + }, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n`ac_key_Job_FinalMinorStatus`.`value`,\n`ac_bucket_Job`.`startTime` as time,\nSUM(`ac_bucket_Job`.`entriesInBucket`) / `ac_bucket_Job`.`bucketLength` * 3600 as metric\nFROM\n`ac_bucket_Job`,\n`ac_key_Job_JobType`,\n`ac_key_Job_FinalMinorStatus`\nWHERE `ac_bucket_Job`.`startTime` >= $__unixEpochFrom()\nAND `ac_bucket_Job`.`startTime` <= $__unixEpochTo()\nAND ( `ac_key_Job_JobType`.`value` = \"User\")\nAND `ac_bucket_Job`.`FinalMinorStatus` = `ac_key_Job_FinalMinorStatus`.`id`\nAND `ac_bucket_Job`.`JobType` = `ac_key_Job_JobType`.`id`\nGROUP BY\nstartTime,\n`ac_key_Job_FinalMinorStatus`.value,\nbucketlength", + "refId": "A" + } + ], + "title": "User Jobs By Final Minor Status (MySQL)", + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "metric ", + "renamePattern": "" + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "mysql", + "uid": "R3A-SHw7z" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-RdYlGr" + }, + "custom": { + "fillOpacity": 100, + "lineWidth": 1 + }, + "mappings": [], + "max": 100, + "min": 0, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 16, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 5, + "options": { + "colWidth": 1, + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "rowHeight": 0.95, + "showValue": "auto", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "mysql", + "uid": "R3A-SHw7z" + }, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n`ac_key_Job_JobType`.`value`,\n`ac_bucket_Job`.`startTime` as time,\n100 * SUM(`ac_bucket_Job`.`CPUTime`) / SUM(`ac_bucket_Job`.`ExecTime`) / `ac_bucket_Job`.`bucketLength` * 3600 as metric\nFROM\n`ac_bucket_Job`,\n`ac_key_Job_JobType`\nWHERE\n`ac_bucket_Job`.`startTime` >= $__unixEpochFrom()\nAND `ac_bucket_Job`.`startTime` <= $__unixEpochTo()\nAND `ac_bucket_Job`.`JobType` = `ac_key_Job_JobType`.`id`\nGROUP BY\ntime,\n`ac_key_Job_JobType`.Value,\nbucketlength", + "refId": "A" + } + ], + "title": "Job CPU Efficiency By JobType (MySQL)", + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "metric ", + "renamePattern": "" + } + } + ], + "type": "status-history" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "iEgMolO4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 16, + "w": 8, + "x": 0, + "y": 16 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "last", + "max", + "min", + "mean" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "JobType", + "id": "2", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_count", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "3", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "iEgMolO4k" + }, + "metrics": [ + { + "id": "1", + "type": "count" + } + ], + "query": "Status: Done AND JobType: User", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "User Jobs By JobType (ElasticSearch)", + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "metric ", + "renamePattern": "" + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "mysql", + "uid": "R3A-SHw7z" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 16, + "w": 8, + "x": 8, + "y": 16 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "last", + "max", + "min", + "mean" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "mysql", + "uid": "R3A-SHw7z" + }, + "format": "time_series", + "rawQuery": true, + "rawSql": "SELECT\n`ac_key_DataOperation_Protocol`.`value`,\n`ac_bucket_DataOperation`.`startTime` as time,\n(SUM(`ac_bucket_DataOperation`.`TransferTotal`) - SUM(`ac_bucket_DataOperation`.`TransferOK`) ) / `ac_bucket_DataOperation`.`bucketLength` * 3600 AS metric\nFROM\n`ac_bucket_DataOperation`,\n`ac_key_DataOperation_Protocol`,\n`ac_key_DataOperation_FinalStatus`\nWHERE `ac_bucket_DataOperation`.`startTime` >= $__unixEpochFrom()\nAND `ac_bucket_DataOperation`.`startTime` <= $__unixEpochTo()\nAND ( `ac_key_DataOperation_FinalStatus`.`value` = \"Failed\" )\nAND `ac_bucket_DataOperation`.`Protocol` =\n`ac_key_DataOperation_Protocol`.`id`\nAND `ac_bucket_DataOperation`.`FinalStatus` = `ac_key_DataOperation_FinalStatus`.`id`\nGROUP BY\ntime,\n`ac_key_DataOperation_Protocol`.Value,\nbucketlength", + "refId": "A" + } + ], + "title": "User Jobs By Final Minor Status (MySQL)", + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "metric ", + "renamePattern": "" + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "mysql", + "uid": "R3A-SHw7z" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 16, + "w": 8, + "x": 16, + "y": 16 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "last", + "max", + "min", + "mean" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "mysql", + "uid": "R3A-SHw7z" + }, + "format": "time_series", + "group": [], + "hide": false, + "rawQuery": true, + "rawSql": "SELECT\n`ac_key_Pilot_GridStatus`.`value`,\n`ac_bucket_Pilot`.`startTime` as time,\nSUM(`ac_bucket_Pilot`.`entriesInBucket`) / `ac_bucket_Pilot`.`bucketLength` * 3600 as metric\nFROM\n`ac_bucket_Pilot`,\n`ac_key_Pilot_GridStatus`\nWHERE\n`ac_bucket_Pilot`.`startTime` >= $__unixEpochFrom()\nAND `ac_bucket_Pilot`.`startTime` <= $__unixEpochTo()\nAND `ac_bucket_Pilot`.`GridStatus` = `ac_key_Pilot_GridStatus`.`id`\nGROUP BY\nstartTime,\n`ac_key_Pilot_GridStatus`.Value,\nbucketlength", + "refId": "A", + "select": [ + [ + { + "params": [ + "CPUTime" + ], + "type": "column" + } + ], + [ + { + "params": [ + "DiskSpace" + ], + "type": "column" + } + ] + ], + "table": "ac_type_Job", + "timeColumn": "startTime", + "timeColumnType": "int", + "where": [ + { + "name": "$__unixEpochFilter", + "params": [], + "type": "macro" + } + ] + } + ], + "title": "Pilots By Status (MySQL)", + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "metric ", + "renamePattern": "" + } + } + ], + "type": "timeseries" + } + ], + "refresh": false, + "schemaVersion": 36, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "2023-03-02T00:00:00.000Z", + "to": "2023-03-02T23:59:59.000Z" + }, + "timepicker": {}, + "timezone": "", + "title": "Demo Dashboard", + "uid": "2lMZDWa4k", + "version": 6, + "weekStart": "" +} diff --git a/dashboards/ElasticJobParameters/JobParametersDashboard.ndjson b/dashboards/JobParameters/JobParametersDashboard.ndjson similarity index 100% rename from dashboards/ElasticJobParameters/JobParametersDashboard.ndjson rename to dashboards/JobParameters/JobParametersDashboard.ndjson diff --git a/dashboards/PilotSubmissions/PilotSubmissions.json b/dashboards/PilotSubmissions/PilotSubmissions.json new file mode 100644 index 00000000000..b3478308262 --- /dev/null +++ b/dashboards/PilotSubmissions/PilotSubmissions.json @@ -0,0 +1,1117 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 5602, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "elasticsearch", + "uid": "_YDJGQQVk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "max", + "mean", + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Site", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "1" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "_YDJGQQVk" + }, + "metrics": [ + { + "field": "NumTotal", + "id": "1", + "type": "sum" + } + ], + "query": "", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Pilot Submissions by Site", + "type": "timeseries" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "_YDJGQQVk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "max", + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "$$hashKey": "object:177", + "aggregation": "Last", + "alias": "", + "bucketAggs": [ + { + "field": "Site", + "id": "8", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "6", + "size": "50" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "_YDJGQQVk" + }, + "decimals": 2, + "displayAliasType": "Warning / Critical", + "displayType": "Regular", + "displayValueWithAlias": "Never", + "metrics": [ + { + "field": "NumSucceeded", + "hide": true, + "id": "5", + "type": "sum" + }, + { + "field": "NumTotal", + "hide": true, + "id": "6", + "type": "sum" + }, + { + "id": "7", + "pipelineVariables": [ + { + "name": "var1", + "pipelineAgg": "5" + }, + { + "name": "var2", + "pipelineAgg": "6" + } + ], + "settings": { + "script": "params.var1 /params.var2 * 100" + }, + "type": "bucket_script" + } + ], + "query": "", + "refId": "A", + "timeField": "timestamp", + "units": "none", + "valueHandler": "Number Threshold" + } + ], + "title": "Efficiency of Pilot Submissions by Site", + "type": "timeseries" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "_YDJGQQVk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "max", + "mean", + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "HostName", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "1m", + "min_doc_count": "1" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "_YDJGQQVk" + }, + "metrics": [ + { + "field": "NumTotal", + "id": "1", + "type": "sum" + } + ], + "query": "", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Pilot Submissions by HostName", + "type": "timeseries" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "_YDJGQQVk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepBefore", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "noValue": "0", + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 75 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "max", + "last" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.5.21", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "HostName", + "id": "8", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "_YDJGQQVk" + }, + "metrics": [ + { + "field": "NumSucceeded", + "hide": true, + "id": "5", + "type": "sum" + }, + { + "field": "NumTotal", + "hide": true, + "id": "6", + "type": "sum" + }, + { + "id": "7", + "pipelineVariables": [ + { + "name": "var1", + "pipelineAgg": "5" + }, + { + "name": "var2", + "pipelineAgg": "6" + } + ], + "settings": { + "script": "params.var1 /params.var2 * 100" + }, + "type": "bucket_script" + } + ], + "query": "", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Efficiency of Pilot Submissions by HostName", + "type": "timeseries" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "_YDJGQQVk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "max", + "mean", + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "CE", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "1" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "_YDJGQQVk" + }, + "metrics": [ + { + "field": "NumTotal", + "id": "1", + "type": "sum" + } + ], + "query": "", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Pilot Submissions by CE", + "type": "timeseries" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "_YDJGQQVk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 9, + "options": { + "legend": { + "calcs": [ + "max", + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "CE", + "id": "8", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "6", + "size": "50" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "_YDJGQQVk" + }, + "metrics": [ + { + "field": "NumSucceeded", + "hide": true, + "id": "5", + "type": "sum" + }, + { + "field": "NumTotal", + "hide": true, + "id": "6", + "type": "sum" + }, + { + "id": "7", + "pipelineVariables": [ + { + "name": "var1", + "pipelineAgg": "5" + }, + { + "name": "var2", + "pipelineAgg": "6" + } + ], + "settings": { + "script": "params.var1 /params.var2 * 100" + }, + "type": "bucket_script" + } + ], + "query": "", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Efficiency of Pilot submissions by CE", + "type": "timeseries" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "_YDJGQQVk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "max", + "mean", + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "SiteDirector", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "1" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "_YDJGQQVk" + }, + "metrics": [ + { + "field": "NumTotal", + "id": "1", + "type": "sum" + } + ], + "query": "", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Pilot Submissions by SiteDirector", + "type": "timeseries" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "_YDJGQQVk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "min": 0, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "max", + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "SiteDirector", + "id": "8", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "_YDJGQQVk" + }, + "metrics": [ + { + "field": "NumSucceeded", + "hide": true, + "id": "5", + "type": "sum" + }, + { + "field": "NumTotal", + "hide": true, + "id": "6", + "type": "sum" + }, + { + "id": "7", + "pipelineVariables": [ + { + "name": "var1", + "pipelineAgg": "5" + }, + { + "name": "var2", + "pipelineAgg": "6" + } + ], + "settings": { + "script": "params.var1 /params.var2 * 100" + }, + "type": "bucket_script" + } + ], + "query": "", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Efficiency of Pilot submissions by SiteDirector", + "type": "timeseries" + } + ], + "refresh": false, + "schemaVersion": 36, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Pilot Submissions", + "uid": "3C1QagU4z", + "version": 6, + "weekStart": "" +} diff --git a/dashboards/PilotSubmissions/PilotSubmissionsDashboard.ndjson b/dashboards/PilotSubmissions/PilotSubmissionsDashboard.ndjson index 708213a4fd2..dbfce33c009 100644 --- a/dashboards/PilotSubmissions/PilotSubmissionsDashboard.ndjson +++ b/dashboards/PilotSubmissions/PilotSubmissionsDashboard.ndjson @@ -1,11 +1,12 @@ -{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"title":"Pilot Submissions","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Pilot Submissions\",\"type\":\"markdown\",\"aggs\":[],\"params\":{\"fontSize\":15,\"openLinksInNewTab\":false,\"markdown\":\"# Pilot Submissions Dashboard\\n- Here are shown the number of pilot submission and the submission efficiency by `Site`, `HostName`, `CE` and `Queue\\n`\"}}"},"id":"8dbbb480-b3ee-11ec-aa10-d1ab7e9f6014","migrationVersion":{"visualization":"7.10.0"},"references":[],"type":"visualization","updated_at":"2022-04-22T16:33:24.084Z","version":"WzIwMjIyLDhd"} -{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Pilot Submissions by Site","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Pilot Submissions by Site\",\"type\":\"area\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"NumTotal\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"timestamp\",\"timeRange\":{\"from\":\"2022-03-24T08:18:58.278Z\",\"to\":\"2022-03-25T23:30:00.000Z\"},\"useNormalizedEsInterval\":true,\"scaleMetricValues\":true,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"Site\",\"orderBy\":\"_key\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"}],\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"filter\":true,\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"category\"}],\"grid\":{\"categoryLines\":false},\"labels\":{},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Sum of NumTotal\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"mode\":\"stacked\",\"show\":true,\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"thresholdLine\":{\"color\":\"#E7664C\",\"show\":false,\"style\":\"full\",\"value\":10,\"width\":1},\"times\":[],\"type\":\"area\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":200},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Number of Submissions \"},\"type\":\"value\"}]}}"},"id":"71449090-a45f-11ec-8f72-459c28757f0a","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"ef627470-8f2d-11ec-a890-8913ef6e85ae","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2022-04-20T08:23:49.790Z","version":"WzE3MzEyLDhd"} -{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"title":"Submission Efficiency by Site","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Submission Efficiency by Site\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#68BC00\",\"split_mode\":\"terms\",\"split_color_mode\":\"kibana\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"sum\",\"field\":\"numSucceeded\"},{\"id\":\"77a6e660-c258-11ec-8ab5-1131e085094c\",\"type\":\"sum\",\"field\":\"NumTotal\"},{\"id\":\"8149c9d0-c258-11ec-8ab5-1131e085094c\",\"type\":\"math\",\"variables\":[{\"id\":\"85275220-c258-11ec-8ab5-1131e085094c\",\"name\":\"a\",\"field\":\"61ca57f2-469d-11e7-af02-69e470af7417\"},{\"id\":\"88157160-c258-11ec-8ab5-1131e085094c\",\"name\":\"b\",\"field\":\"77a6e660-c258-11ec-8ab5-1131e085094c\"}],\"script\":\"params.a/params.b\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"percent\",\"chart_type\":\"bar\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Efficiency of Submission\",\"type\":\"timeseries\",\"terms_field\":\"Site\"}],\"time_field\":\"\",\"index_pattern\":\"dirac-certification_pilotsubmission*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"dirac-certification_wmshistory_index-*\",\"default_timefield\":\"timestamp\",\"isModelInvalid\":false}}"},"id":"dc53e900-c258-11ec-b508-cbe83a8a8723","migrationVersion":{"visualization":"7.10.0"},"references":[],"type":"visualization","updated_at":"2022-04-22T16:25:45.104Z","version":"WzIwMTAwLDhd"} -{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Pilot Submissions by HostName","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Pilot Submissions by HostName\",\"type\":\"area\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"NumTotal\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"timestamp\",\"timeRange\":{\"from\":\"now-30d\",\"to\":\"now\"},\"useNormalizedEsInterval\":true,\"scaleMetricValues\":true,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"HostName\",\"orderBy\":\"_key\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"}],\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"filter\":true,\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"category\"}],\"grid\":{\"categoryLines\":false},\"labels\":{},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Sum of NumTotal\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"mode\":\"stacked\",\"show\":true,\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"thresholdLine\":{\"color\":\"#E7664C\",\"show\":false,\"style\":\"full\",\"value\":10,\"width\":1},\"times\":[],\"type\":\"area\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":200},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Number of Submissions \"},\"type\":\"value\"}]}}"},"id":"d903a200-c257-11ec-b67b-89dd8cb60ff9","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"ef627470-8f2d-11ec-a890-8913ef6e85ae","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2022-04-22T16:18:30.048Z","version":"WzE5OTQ5LDhd"} -{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"title":"Submission Efficiency by HostName","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Submission Efficiency by HostName\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"rgba(211,96,134,1)\",\"split_mode\":\"terms\",\"split_color_mode\":\"kibana\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"sum\",\"field\":\"numSucceeded\"},{\"id\":\"77a6e660-c258-11ec-8ab5-1131e085094c\",\"type\":\"sum\",\"field\":\"NumTotal\"},{\"id\":\"8149c9d0-c258-11ec-8ab5-1131e085094c\",\"type\":\"math\",\"variables\":[{\"id\":\"85275220-c258-11ec-8ab5-1131e085094c\",\"name\":\"a\",\"field\":\"61ca57f2-469d-11e7-af02-69e470af7417\"},{\"id\":\"88157160-c258-11ec-8ab5-1131e085094c\",\"name\":\"b\",\"field\":\"77a6e660-c258-11ec-8ab5-1131e085094c\"}],\"script\":\"params.a/params.b\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"percent\",\"chart_type\":\"bar\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Efficiency of Submission\",\"type\":\"timeseries\",\"terms_field\":\"HostName\"}],\"time_field\":\"\",\"index_pattern\":\"dirac-certification_pilotsubmission*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"dirac-certification_wmshistory_index-*\",\"default_timefield\":\"timestamp\",\"isModelInvalid\":false}}"},"id":"25661e10-c259-11ec-b67b-89dd8cb60ff9","migrationVersion":{"visualization":"7.10.0"},"references":[],"type":"visualization","updated_at":"2022-04-22T16:28:34.309Z","version":"WzIwMTU1LDhd"} -{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Pilot Submissions by CE","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Pilot Submissions by CE\",\"type\":\"area\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"NumTotal\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"timestamp\",\"timeRange\":{\"from\":\"now-30d\",\"to\":\"now\"},\"useNormalizedEsInterval\":true,\"scaleMetricValues\":true,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"CE\",\"orderBy\":\"_key\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"}],\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"filter\":true,\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"category\"}],\"grid\":{\"categoryLines\":false},\"labels\":{},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Sum of NumTotal\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"mode\":\"stacked\",\"show\":true,\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"thresholdLine\":{\"color\":\"#E7664C\",\"show\":false,\"style\":\"full\",\"value\":10,\"width\":1},\"times\":[],\"type\":\"area\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":200},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Number of Submissions \"},\"type\":\"value\"}]}}"},"id":"f1a10e10-c257-11ec-b67b-89dd8cb60ff9","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"ef627470-8f2d-11ec-a890-8913ef6e85ae","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2022-04-22T16:19:11.345Z","version":"WzE5OTYzLDhd"} -{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"title":"Submission Efficiency by CE","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Submission Efficiency by CE\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#68BC00\",\"split_mode\":\"terms\",\"split_color_mode\":\"kibana\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"sum\",\"field\":\"numSucceeded\"},{\"id\":\"77a6e660-c258-11ec-8ab5-1131e085094c\",\"type\":\"sum\",\"field\":\"NumTotal\"},{\"id\":\"8149c9d0-c258-11ec-8ab5-1131e085094c\",\"type\":\"math\",\"variables\":[{\"id\":\"85275220-c258-11ec-8ab5-1131e085094c\",\"name\":\"a\",\"field\":\"61ca57f2-469d-11e7-af02-69e470af7417\"},{\"id\":\"88157160-c258-11ec-8ab5-1131e085094c\",\"name\":\"b\",\"field\":\"77a6e660-c258-11ec-8ab5-1131e085094c\"}],\"script\":\"params.a/params.b\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"percent\",\"chart_type\":\"bar\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Efficiency of Submission\",\"type\":\"timeseries\",\"terms_field\":\"CE\"}],\"time_field\":\"\",\"index_pattern\":\"dirac-certification_pilotsubmission*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"dirac-certification_wmshistory_index-*\",\"default_timefield\":\"timestamp\",\"isModelInvalid\":false}}"},"id":"37f89a80-c259-11ec-b67b-89dd8cb60ff9","migrationVersion":{"visualization":"7.10.0"},"references":[],"type":"visualization","updated_at":"2022-04-22T16:28:18.856Z","version":"WzIwMTQyLDhd"} -{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Pilot Submissions by SiteDirector","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Pilot Submissions by SiteDirector\",\"type\":\"area\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"NumTotal\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"timestamp\",\"timeRange\":{\"from\":\"now-30d\",\"to\":\"now\"},\"useNormalizedEsInterval\":true,\"scaleMetricValues\":true,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"SiteDirector\",\"orderBy\":\"_key\",\"order\":\"desc\",\"size\":100,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"}],\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"filter\":true,\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{},\"type\":\"category\"}],\"grid\":{\"categoryLines\":false},\"labels\":{},\"legendPosition\":\"right\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Sum of NumTotal\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"lineWidth\":2,\"mode\":\"stacked\",\"show\":true,\"showCircles\":true,\"type\":\"histogram\",\"valueAxis\":\"ValueAxis-1\"}],\"thresholdLine\":{\"color\":\"#E7664C\",\"show\":false,\"style\":\"full\",\"value\":10,\"width\":1},\"times\":[],\"type\":\"area\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":200},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Number of Submissions \"},\"type\":\"value\"}]}}"},"id":"202acff0-c258-11ec-aa10-d1ab7e9f6014","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"ef627470-8f2d-11ec-a890-8913ef6e85ae","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2022-04-22T16:20:29.423Z","version":"WzE5OTgyLDhd"} -{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"title":"Submission Efficiency by SiteDirector","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Submission Efficiency by SiteDirector\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"rgba(214,191,87,1)\",\"split_mode\":\"terms\",\"split_color_mode\":\"rainbow\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"sum\",\"field\":\"numSucceeded\"},{\"id\":\"77a6e660-c258-11ec-8ab5-1131e085094c\",\"type\":\"sum\",\"field\":\"NumTotal\"},{\"id\":\"8149c9d0-c258-11ec-8ab5-1131e085094c\",\"type\":\"math\",\"variables\":[{\"id\":\"85275220-c258-11ec-8ab5-1131e085094c\",\"name\":\"a\",\"field\":\"61ca57f2-469d-11e7-af02-69e470af7417\"},{\"id\":\"88157160-c258-11ec-8ab5-1131e085094c\",\"name\":\"b\",\"field\":\"77a6e660-c258-11ec-8ab5-1131e085094c\"}],\"script\":\"params.a/params.b\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"percent\",\"chart_type\":\"bar\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"none\",\"label\":\"Efficiency of Submission\",\"type\":\"timeseries\",\"terms_field\":\"SiteDirector\"}],\"time_field\":\"\",\"index_pattern\":\"dirac-certification_pilotsubmission*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"dirac-certification_wmshistory_index-*\",\"default_timefield\":\"timestamp\",\"isModelInvalid\":false}}"},"id":"6d82cf90-c259-11ec-b67b-89dd8cb60ff9","migrationVersion":{"visualization":"7.10.0"},"references":[],"type":"visualization","updated_at":"2022-04-22T16:29:48.681Z","version":"WzIwMTc0LDhd"} -{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}"},"optionsJSON":"{\"hidePanelTitles\":false,\"useMargins\":true}","panelsJSON":"[{\"version\":\"7.10.2\",\"gridData\":{\"x\":13,\"y\":0,\"w\":27,\"h\":7,\"i\":\"cc6aad17-246d-4146-84c8-4d7902b353b3\"},\"panelIndex\":\"cc6aad17-246d-4146-84c8-4d7902b353b3\",\"embeddableConfig\":{},\"panelRefName\":\"panel_0\"},{\"version\":\"7.10.2\",\"gridData\":{\"x\":0,\"y\":7,\"w\":24,\"h\":15,\"i\":\"88cb4f0d-51f0-4bd3-9a15-51cc0bd2d0dd\"},\"panelIndex\":\"88cb4f0d-51f0-4bd3-9a15-51cc0bd2d0dd\",\"embeddableConfig\":{\"hidePanelTitles\":false},\"panelRefName\":\"panel_1\"},{\"version\":\"7.10.2\",\"gridData\":{\"x\":24,\"y\":7,\"w\":24,\"h\":15,\"i\":\"11f4a7ac-b01d-4307-ae6e-7050a85d8ddd\"},\"panelIndex\":\"11f4a7ac-b01d-4307-ae6e-7050a85d8ddd\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"7.10.2\",\"gridData\":{\"x\":0,\"y\":22,\"w\":24,\"h\":15,\"i\":\"0e78674b-8317-4ca8-b99f-5e3b142d20d4\"},\"panelIndex\":\"0e78674b-8317-4ca8-b99f-5e3b142d20d4\",\"embeddableConfig\":{},\"panelRefName\":\"panel_3\"},{\"version\":\"7.10.2\",\"gridData\":{\"x\":24,\"y\":22,\"w\":24,\"h\":15,\"i\":\"590e328f-87f0-4415-8186-f1f55d0a374e\"},\"panelIndex\":\"590e328f-87f0-4415-8186-f1f55d0a374e\",\"embeddableConfig\":{},\"panelRefName\":\"panel_4\"},{\"version\":\"7.10.2\",\"gridData\":{\"x\":0,\"y\":37,\"w\":24,\"h\":15,\"i\":\"e682c108-af97-4aa8-8c5e-60f5d72abc61\"},\"panelIndex\":\"e682c108-af97-4aa8-8c5e-60f5d72abc61\",\"embeddableConfig\":{},\"panelRefName\":\"panel_5\"},{\"version\":\"7.10.2\",\"gridData\":{\"x\":24,\"y\":37,\"w\":24,\"h\":15,\"i\":\"5c908bcb-84ba-49b7-a4d0-b8025ab5017e\"},\"panelIndex\":\"5c908bcb-84ba-49b7-a4d0-b8025ab5017e\",\"embeddableConfig\":{},\"panelRefName\":\"panel_6\"},{\"version\":\"7.10.2\",\"gridData\":{\"x\":0,\"y\":52,\"w\":24,\"h\":15,\"i\":\"bd406ec3-af25-4488-a1b4-1f695b48830f\"},\"panelIndex\":\"bd406ec3-af25-4488-a1b4-1f695b48830f\",\"embeddableConfig\":{},\"panelRefName\":\"panel_7\"},{\"version\":\"7.10.2\",\"gridData\":{\"x\":24,\"y\":52,\"w\":24,\"h\":15,\"i\":\"784ba8fd-15c5-4b18-b2ac-995dc6fa256c\"},\"panelIndex\":\"784ba8fd-15c5-4b18-b2ac-995dc6fa256c\",\"embeddableConfig\":{},\"panelRefName\":\"panel_8\"}]","timeRestore":false,"title":"Pilot Submissions Dashboard","version":1},"id":"b6919620-bfd0-11ec-b508-cbe83a8a8723","migrationVersion":{"dashboard":"7.9.3"},"references":[{"id":"8dbbb480-b3ee-11ec-aa10-d1ab7e9f6014","name":"panel_0","type":"visualization"},{"id":"71449090-a45f-11ec-8f72-459c28757f0a","name":"panel_1","type":"visualization"},{"id":"dc53e900-c258-11ec-b508-cbe83a8a8723","name":"panel_2","type":"visualization"},{"id":"d903a200-c257-11ec-b67b-89dd8cb60ff9","name":"panel_3","type":"visualization"},{"id":"25661e10-c259-11ec-b67b-89dd8cb60ff9","name":"panel_4","type":"visualization"},{"id":"f1a10e10-c257-11ec-b67b-89dd8cb60ff9","name":"panel_5","type":"visualization"},{"id":"37f89a80-c259-11ec-b67b-89dd8cb60ff9","name":"panel_6","type":"visualization"},{"id":"202acff0-c258-11ec-aa10-d1ab7e9f6014","name":"panel_7","type":"visualization"},{"id":"6d82cf90-c259-11ec-b67b-89dd8cb60ff9","name":"panel_8","type":"visualization"}],"type":"dashboard","updated_at":"2022-04-22T16:35:32.869Z","version":"WzIwMjQyLDhd"} -{"exportedCount":10,"missingRefCount":0,"missingReferences":[]} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"title":"Pilot Submissions","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Pilot Submissions\",\"type\":\"markdown\",\"aggs\":[],\"params\":{\"fontSize\":15,\"openLinksInNewTab\":false,\"markdown\":\"# Pilot Submissions Dashboard\\n- Here are shown the number of pilot submission and the submission efficiency by `Site`, `HostName`, `CE` and `Queue\\n`\"}}"},"id":"8dbbb480-b3ee-11ec-aa10-d1ab7e9f6014","migrationVersion":{"visualization":"7.10.0"},"references":[],"type":"visualization","updated_at":"2023-01-30T14:03:42.417Z","version":"WzQzNyw0XQ=="} +{"attributes":{"fields":"[{\"count\":0,\"name\":\"CE\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"HostName\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"NumSucceeded\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"NumTotal\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"Queue\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"Site\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"SiteDirector\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"Status\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_score\",\"type\":\"number\",\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_type\",\"type\":\"string\",\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"script\":\"doc[\\\"NumSucceeded\\\"].value / doc[\\\"NumTotal\\\"].value\",\"lang\":\"painless\",\"name\":\"Ratio\",\"type\":\"number\",\"scripted\":true,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false}]","timeFieldName":"timestamp","title":"*_pilotsubmission_monitoring-index-*"},"id":"6b97a110-a93a-11ed-baf6-bd7d09f916cf","migrationVersion":{"index-pattern":"7.6.0"},"references":[],"type":"index-pattern","updated_at":"2023-02-10T12:00:47.678Z","version":"WzQ1MCw0XQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Pilot Submission By Site (using environment agnostic index pattern)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Pilot Submission By Site (using environment agnostic index pattern)\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"NumTotal\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"timestamp\",\"timeRange\":{\"from\":\"now-6M\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"Site\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":500,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Sum of NumTotal\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Sum of NumTotal\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}"},"id":"e0239fb0-b28c-11ed-baf6-bd7d09f916cf","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"6b97a110-a93a-11ed-baf6-bd7d09f916cf","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-02-22T08:42:44.394Z","version":"WzQ1Nyw0XQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"efficiency of pilot submission by site (using environment agnostic index pattern)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"efficiency of pilot submission by site (using environment agnostic index pattern)\",\"type\":\"heatmap\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"avg\",\"params\":{\"field\":\"Ratio\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"timestamp\",\"timeRange\":{\"from\":\"now-6M\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"Site\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"}],\"params\":{\"type\":\"heatmap\",\"addTooltip\":true,\"addLegend\":true,\"enableHover\":false,\"legendPosition\":\"right\",\"times\":[],\"colorsNumber\":10,\"colorSchema\":\"Green to Red\",\"setColorRange\":false,\"colorsRange\":[],\"invertColors\":true,\"percentageMode\":false,\"valueAxes\":[{\"show\":false,\"id\":\"ValueAxis-1\",\"type\":\"value\",\"scale\":{\"type\":\"linear\",\"defaultYExtents\":false},\"labels\":{\"show\":false,\"rotate\":0,\"overwriteColor\":false,\"color\":\"black\"}}]}}"},"id":"9fd2cb70-a93b-11ed-baf6-bd7d09f916cf","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"6b97a110-a93a-11ed-baf6-bd7d09f916cf","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-02-22T08:30:15.915Z","version":"WzQ1Miw0XQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"efficiency of pilot submission by hostname (using environment agnostic index pattern)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"efficiency of pilot submission by hostname (using environment agnostic index pattern)\",\"type\":\"heatmap\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"avg\",\"params\":{\"field\":\"Ratio\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"timestamp\",\"timeRange\":{\"from\":\"now-6M\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"HostName\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"}],\"params\":{\"type\":\"heatmap\",\"addTooltip\":true,\"addLegend\":true,\"enableHover\":false,\"legendPosition\":\"right\",\"times\":[],\"colorsNumber\":10,\"colorSchema\":\"Green to Red\",\"setColorRange\":false,\"colorsRange\":[],\"invertColors\":true,\"percentageMode\":false,\"valueAxes\":[{\"show\":false,\"id\":\"ValueAxis-1\",\"type\":\"value\",\"scale\":{\"type\":\"linear\",\"defaultYExtents\":false},\"labels\":{\"show\":false,\"rotate\":0,\"overwriteColor\":false,\"color\":\"black\"}}]}}"},"id":"6676e6a0-b28b-11ed-baf6-bd7d09f916cf","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"6b97a110-a93a-11ed-baf6-bd7d09f916cf","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-02-22T08:32:10.762Z","version":"WzQ1NSw0XQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Pilot Submission By HostName (using environment agnostic index pattern)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Pilot Submission By HostName (using environment agnostic index pattern)\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"NumTotal\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"timestamp\",\"timeRange\":{\"from\":\"now-6M\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"HostName\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":500,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Sum of NumTotal\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Sum of NumTotal\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}"},"id":"0e0513a0-b28d-11ed-baf6-bd7d09f916cf","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"6b97a110-a93a-11ed-baf6-bd7d09f916cf","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-02-22T08:44:01.370Z","version":"WzQ2MCw0XQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"efficiency of pilot submission by CE (using environment agnostic index pattern)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"efficiency of pilot submission by CE (using environment agnostic index pattern)\",\"type\":\"heatmap\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"avg\",\"params\":{\"field\":\"Ratio\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"timestamp\",\"timeRange\":{\"from\":\"now-6M\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"CE\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"}],\"params\":{\"type\":\"heatmap\",\"addTooltip\":true,\"addLegend\":true,\"enableHover\":false,\"legendPosition\":\"right\",\"times\":[],\"colorsNumber\":10,\"colorSchema\":\"Green to Red\",\"setColorRange\":false,\"colorsRange\":[],\"invertColors\":true,\"percentageMode\":false,\"valueAxes\":[{\"show\":false,\"id\":\"ValueAxis-1\",\"type\":\"value\",\"scale\":{\"type\":\"linear\",\"defaultYExtents\":false},\"labels\":{\"show\":false,\"rotate\":0,\"overwriteColor\":false,\"color\":\"black\"}}]}}"},"id":"4abe7ef0-b28b-11ed-baf6-bd7d09f916cf","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"6b97a110-a93a-11ed-baf6-bd7d09f916cf","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-02-22T08:31:52.190Z","version":"WzQ1NCw0XQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Pilot Submission By CE (using environment agnostic index pattern)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Pilot Submission By CE (using environment agnostic index pattern)\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"NumTotal\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"timestamp\",\"timeRange\":{\"from\":\"now-6M\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"CE\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":500,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Sum of NumTotal\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Sum of NumTotal\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}"},"id":"28533cf0-b28d-11ed-baf6-bd7d09f916cf","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"6b97a110-a93a-11ed-baf6-bd7d09f916cf","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-02-22T08:44:45.503Z","version":"WzQ2MSw0XQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"efficiency of pilot submission by sitedirector (using environment agnostic index pattern)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"efficiency of pilot submission by sitedirector (using environment agnostic index pattern)\",\"type\":\"heatmap\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"avg\",\"params\":{\"field\":\"Ratio\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"timestamp\",\"timeRange\":{\"from\":\"now-6M\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"SiteDirector\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":100,\"otherBucket\":true,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"}],\"params\":{\"type\":\"heatmap\",\"addTooltip\":true,\"addLegend\":true,\"enableHover\":false,\"legendPosition\":\"right\",\"times\":[],\"colorsNumber\":10,\"colorSchema\":\"Green to Red\",\"setColorRange\":false,\"colorsRange\":[],\"invertColors\":true,\"percentageMode\":false,\"valueAxes\":[{\"show\":false,\"id\":\"ValueAxis-1\",\"type\":\"value\",\"scale\":{\"type\":\"linear\",\"defaultYExtents\":false},\"labels\":{\"show\":false,\"rotate\":0,\"overwriteColor\":false,\"color\":\"black\"}}]}}"},"id":"766c7110-b28b-11ed-baf6-bd7d09f916cf","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"6b97a110-a93a-11ed-baf6-bd7d09f916cf","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-02-22T08:32:37.537Z","version":"WzQ1Niw0XQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Pilot Submission By SiteDirector (using environment agnostic index pattern)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Pilot Submission By SiteDirector (using environment agnostic index pattern)\",\"type\":\"histogram\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"NumTotal\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"timestamp\",\"timeRange\":{\"from\":\"now-6M\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"SiteDirector\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":500,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"}],\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Sum of NumTotal\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Sum of NumTotal\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{\"show\":false},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}"},"id":"e9bb1170-b28c-11ed-baf6-bd7d09f916cf","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"6b97a110-a93a-11ed-baf6-bd7d09f916cf","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-02-22T08:43:21.193Z","version":"WzQ1OSw0XQ=="} +{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}"},"optionsJSON":"{\"hidePanelTitles\":false,\"useMargins\":true}","panelsJSON":"[{\"version\":\"2.2.1\",\"gridData\":{\"h\":7,\"i\":\"cc6aad17-246d-4146-84c8-4d7902b353b3\",\"w\":27,\"x\":13,\"y\":0},\"panelIndex\":\"cc6aad17-246d-4146-84c8-4d7902b353b3\",\"embeddableConfig\":{},\"panelRefName\":\"panel_0\"},{\"version\":\"2.2.1\",\"gridData\":{\"x\":0,\"y\":7,\"w\":24,\"h\":15,\"i\":\"93747900-f3d7-435e-9fc7-f5795550a088\"},\"panelIndex\":\"93747900-f3d7-435e-9fc7-f5795550a088\",\"embeddableConfig\":{},\"panelRefName\":\"panel_1\"},{\"version\":\"2.2.1\",\"gridData\":{\"x\":24,\"y\":7,\"w\":24,\"h\":15,\"i\":\"7899363d-0a53-47ac-9740-abe14c8e3751\"},\"panelIndex\":\"7899363d-0a53-47ac-9740-abe14c8e3751\",\"embeddableConfig\":{\"vis\":null},\"panelRefName\":\"panel_2\"},{\"version\":\"2.2.1\",\"gridData\":{\"x\":24,\"y\":22,\"w\":24,\"h\":15,\"i\":\"ed5aea9f-84a2-4ecb-b3b5-66446378920e\"},\"panelIndex\":\"ed5aea9f-84a2-4ecb-b3b5-66446378920e\",\"embeddableConfig\":{\"vis\":null},\"panelRefName\":\"panel_3\"},{\"version\":\"2.2.1\",\"gridData\":{\"x\":0,\"y\":22,\"w\":24,\"h\":15,\"i\":\"dc063303-4fe6-4bb7-a1c3-c89d92030b83\"},\"panelIndex\":\"dc063303-4fe6-4bb7-a1c3-c89d92030b83\",\"embeddableConfig\":{},\"panelRefName\":\"panel_4\"},{\"version\":\"2.2.1\",\"gridData\":{\"x\":24,\"y\":37,\"w\":24,\"h\":15,\"i\":\"d3cfd430-4d9b-498e-9ed7-8b2e1e4d34da\"},\"panelIndex\":\"d3cfd430-4d9b-498e-9ed7-8b2e1e4d34da\",\"embeddableConfig\":{\"vis\":null},\"panelRefName\":\"panel_5\"},{\"version\":\"2.2.1\",\"gridData\":{\"x\":0,\"y\":37,\"w\":24,\"h\":15,\"i\":\"2417a2ae-6915-4252-83a2-1cb961030ff2\"},\"panelIndex\":\"2417a2ae-6915-4252-83a2-1cb961030ff2\",\"embeddableConfig\":{},\"panelRefName\":\"panel_6\"},{\"version\":\"2.2.1\",\"gridData\":{\"x\":24,\"y\":52,\"w\":24,\"h\":15,\"i\":\"97a2f080-92fb-4531-a077-53b79ef5613b\"},\"panelIndex\":\"97a2f080-92fb-4531-a077-53b79ef5613b\",\"embeddableConfig\":{\"vis\":null},\"panelRefName\":\"panel_7\"},{\"version\":\"2.2.1\",\"gridData\":{\"x\":0,\"y\":52,\"w\":24,\"h\":15,\"i\":\"20d1af82-679e-4397-9b05-5d5e5673ebd3\"},\"panelIndex\":\"20d1af82-679e-4397-9b05-5d5e5673ebd3\",\"embeddableConfig\":{},\"panelRefName\":\"panel_8\"}]","timeRestore":false,"title":"Pilot Submissions Dashboard (using environment agnostic index pattern)","version":1},"id":"39924c70-b28f-11ed-baf6-bd7d09f916cf","migrationVersion":{"dashboard":"7.9.3"},"references":[{"id":"8dbbb480-b3ee-11ec-aa10-d1ab7e9f6014","name":"panel_0","type":"visualization"},{"id":"e0239fb0-b28c-11ed-baf6-bd7d09f916cf","name":"panel_1","type":"visualization"},{"id":"9fd2cb70-a93b-11ed-baf6-bd7d09f916cf","name":"panel_2","type":"visualization"},{"id":"6676e6a0-b28b-11ed-baf6-bd7d09f916cf","name":"panel_3","type":"visualization"},{"id":"0e0513a0-b28d-11ed-baf6-bd7d09f916cf","name":"panel_4","type":"visualization"},{"id":"4abe7ef0-b28b-11ed-baf6-bd7d09f916cf","name":"panel_5","type":"visualization"},{"id":"28533cf0-b28d-11ed-baf6-bd7d09f916cf","name":"panel_6","type":"visualization"},{"id":"766c7110-b28b-11ed-baf6-bd7d09f916cf","name":"panel_7","type":"visualization"},{"id":"e9bb1170-b28c-11ed-baf6-bd7d09f916cf","name":"panel_8","type":"visualization"}],"type":"dashboard","updated_at":"2023-02-22T08:59:33.431Z","version":"WzQ2Miw0XQ=="} +{"exportedCount":11,"missingRefCount":0,"missingReferences":[]} diff --git a/dashboards/PilotsHistory/PilotsHistory.json b/dashboards/PilotsHistory/PilotsHistory.json new file mode 100644 index 00000000000..0a98a723c2d --- /dev/null +++ b/dashboards/PilotsHistory/PilotsHistory.json @@ -0,0 +1,548 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 5601, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "elasticsearch", + "uid": "CN0tVww4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "si:" + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "max", + "mean", + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "GridSite", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "1", + "offset": "10m" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "CN0tVww4k" + }, + "metrics": [ + { + "field": "NumOfPilots", + "id": "1", + "settings": {}, + "type": "sum" + } + ], + "query": "", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Number of Pilots by GridSite", + "type": "timeseries" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "CN0tVww4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "si:" + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "max", + "mean", + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Status", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "1", + "offset": "10m" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "CN0tVww4k" + }, + "metrics": [ + { + "field": "NumOfPilots", + "id": "1", + "settings": {}, + "type": "sum" + } + ], + "query": "", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Number of Pilots by Status", + "type": "timeseries" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "CN0tVww4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "si:" + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "max", + "mean", + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "GridType", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "1", + "offset": "10m" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "CN0tVww4k" + }, + "metrics": [ + { + "field": "NumOfPilots", + "id": "1", + "settings": {}, + "type": "sum" + } + ], + "query": "", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Number of Pilots by GridType", + "type": "timeseries" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "CN0tVww4k" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "si:" + }, + "overrides": [] + }, + "gridPos": { + "h": 14, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "max", + "mean", + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "TaskQueueID", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "1", + "offset": "10m" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "CN0tVww4k" + }, + "metrics": [ + { + "field": "NumOfPilots", + "id": "1", + "settings": {}, + "type": "sum" + } + ], + "query": "", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Number of Pilots by TaskQueueID", + "type": "timeseries" + } + ], + "schemaVersion": 36, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Pilots History", + "uid": "C9Jgbg8Vz", + "version": 7, + "weekStart": "" +} diff --git a/dashboards/PilotsHistory/PilotsHistory.ndjson b/dashboards/PilotsHistory/PilotsHistory.ndjson index e6350e7fe3e..efd420e05bd 100644 --- a/dashboards/PilotsHistory/PilotsHistory.ndjson +++ b/dashboards/PilotsHistory/PilotsHistory.ndjson @@ -1,7 +1,8 @@ -{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Pilots by Status (History)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Pilots by Status (History)\",\"type\":\"area\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"NumOfPilots\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"timestamp\",\"timeRange\":{\"from\":\"now-7d\",\"to\":\"now\"},\"useNormalizedEsInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"Status\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":5,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"}],\"params\":{\"type\":\"area\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Sum of NumOfPilots\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"area\",\"mode\":\"stacked\",\"data\":{\"label\":\"Sum of NumOfPilots\",\"id\":\"1\"},\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true,\"interpolate\":\"linear\",\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"},\"labels\":{}}}"},"id":"ca3878b0-c255-11ec-aa10-d1ab7e9f6014","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"c966f710-b650-11ec-aa10-d1ab7e9f6014","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2022-04-22T16:03:46.235Z","version":"WzE5ODYzLDhd"} -{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Pilots by GridType (History)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Pilots by GridType (History)\",\"type\":\"area\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"NumOfPilots\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"timestamp\",\"timeRange\":{\"from\":\"now-7d\",\"to\":\"now\"},\"useNormalizedEsInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"GridType\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":5,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"}],\"params\":{\"type\":\"area\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Sum of NumOfPilots\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"area\",\"mode\":\"stacked\",\"data\":{\"label\":\"Sum of NumOfPilots\",\"id\":\"1\"},\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true,\"interpolate\":\"linear\",\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"},\"labels\":{}}}"},"id":"dfffe160-c255-11ec-b67b-89dd8cb60ff9","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"c966f710-b650-11ec-aa10-d1ab7e9f6014","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2022-04-22T16:04:22.774Z","version":"WzE5ODcyLDhd"} -{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"title":"PilotsHistory Title","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"PilotsHistory Title\",\"type\":\"markdown\",\"aggs\":[],\"params\":{\"fontSize\":15,\"openLinksInNewTab\":false,\"markdown\":\"# Pilots History Dashboard\\n- PilotsHistory grouped by `Site`, `Type`, `Status` and `TaskQueueID`.\"}}"},"id":"2807d260-c256-11ec-aa10-d1ab7e9f6014","migrationVersion":{"visualization":"7.10.0"},"references":[],"type":"visualization","updated_at":"2022-04-22T16:06:53.410Z","version":"WzE5OTAxLDhd"} -{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Pilots by TaskQueueID (History)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Pilots by TaskQueueID (History)\",\"type\":\"area\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"NumOfPilots\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"timestamp\",\"timeRange\":{\"from\":\"now-7d\",\"to\":\"now\"},\"useNormalizedEsInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"TaskQueueID\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":5,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"}],\"params\":{\"type\":\"area\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Sum of NumOfPilots\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"area\",\"mode\":\"stacked\",\"data\":{\"label\":\"Sum of NumOfPilots\",\"id\":\"1\"},\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true,\"interpolate\":\"linear\",\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"},\"labels\":{}}}"},"id":"ed0291a0-c255-11ec-aa10-d1ab7e9f6014","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"c966f710-b650-11ec-aa10-d1ab7e9f6014","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2022-04-22T16:04:44.602Z","version":"WzE5ODc5LDhd"} -{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Pilots by GridSite (History)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Pilots by GridSite (History)\",\"type\":\"area\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"NumOfPilots\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"timestamp\",\"timeRange\":{\"from\":\"now-7d\",\"to\":\"now\"},\"useNormalizedEsInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"GridSite\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":5,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"}],\"params\":{\"type\":\"area\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Sum of NumOfPilots\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"area\",\"mode\":\"stacked\",\"data\":{\"label\":\"Sum of NumOfPilots\",\"id\":\"1\"},\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true,\"interpolate\":\"linear\",\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"},\"labels\":{}}}"},"id":"9f2d0820-c255-11ec-aa10-d1ab7e9f6014","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"c966f710-b650-11ec-aa10-d1ab7e9f6014","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2022-04-22T16:02:34.018Z","version":"WzE5ODU3LDhd"} -{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}"},"optionsJSON":"{\"hidePanelTitles\":false,\"useMargins\":true}","panelsJSON":"[{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":7,\"i\":\"f845a506-b6b2-4017-9cad-70be31b5d7c9\",\"w\":21,\"x\":15,\"y\":0},\"panelIndex\":\"f845a506-b6b2-4017-9cad-70be31b5d7c9\",\"version\":\"7.10.2\",\"panelRefName\":\"panel_0\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":15,\"i\":\"99d151f0-859d-4e23-a2c3-1f5609ad78d2\",\"w\":24,\"x\":0,\"y\":7},\"panelIndex\":\"99d151f0-859d-4e23-a2c3-1f5609ad78d2\",\"version\":\"7.10.2\",\"panelRefName\":\"panel_1\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":15,\"i\":\"9297d626-3d56-4310-a456-80dc98da7e1e\",\"w\":24,\"x\":24,\"y\":7},\"panelIndex\":\"9297d626-3d56-4310-a456-80dc98da7e1e\",\"version\":\"7.10.2\",\"panelRefName\":\"panel_2\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":15,\"i\":\"94dc2321-0277-4f85-8535-702837c3d17b\",\"w\":24,\"x\":0,\"y\":22},\"panelIndex\":\"94dc2321-0277-4f85-8535-702837c3d17b\",\"version\":\"7.10.2\",\"panelRefName\":\"panel_3\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":15,\"i\":\"824a4855-c088-4ce9-91a7-66dad234b352\",\"w\":24,\"x\":24,\"y\":22},\"panelIndex\":\"824a4855-c088-4ce9-91a7-66dad234b352\",\"version\":\"7.10.2\",\"panelRefName\":\"panel_4\"}]","timeRestore":false,"title":"PilotsHistory Dashboard","version":1},"id":"7d086180-c256-11ec-aa10-d1ab7e9f6014","migrationVersion":{"dashboard":"7.9.3"},"references":[{"id":"2807d260-c256-11ec-aa10-d1ab7e9f6014","name":"panel_0","type":"visualization"},{"id":"9f2d0820-c255-11ec-aa10-d1ab7e9f6014","name":"panel_1","type":"visualization"},{"id":"ca3878b0-c255-11ec-aa10-d1ab7e9f6014","name":"panel_2","type":"visualization"},{"id":"dfffe160-c255-11ec-b67b-89dd8cb60ff9","name":"panel_3","type":"visualization"},{"id":"ed0291a0-c255-11ec-aa10-d1ab7e9f6014","name":"panel_4","type":"visualization"}],"type":"dashboard","updated_at":"2022-04-22T16:08:46.232Z","version":"WzE5OTEwLDhd"} -{"exportedCount":6,"missingRefCount":0,"missingReferences":[]} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"title":"PilotsHistory Title","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"PilotsHistory Title\",\"type\":\"markdown\",\"aggs\":[],\"params\":{\"fontSize\":15,\"openLinksInNewTab\":false,\"markdown\":\"# Pilots History Dashboard\\n- PilotsHistory grouped by `Site`, `Type`, `Status` and `TaskQueueID`.\"}}"},"id":"2807d260-c256-11ec-aa10-d1ab7e9f6014","migrationVersion":{"visualization":"7.10.0"},"references":[],"type":"visualization","updated_at":"2022-07-28T10:46:44.421Z","version":"WzI3NywxXQ=="} +{"attributes":{"fields":"[{\"count\":0,\"name\":\"GridSite\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"GridType\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"NumOfPilots\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"Status\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"TaskQueueID\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_score\",\"type\":\"number\",\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_type\",\"type\":\"string\",\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]","timeFieldName":"timestamp","title":"*_pilotshistory_index-*"},"id":"7346c540-b2c6-11ed-baf6-bd7d09f916cf","migrationVersion":{"index-pattern":"7.6.0"},"references":[],"type":"index-pattern","updated_at":"2023-02-22T15:34:52.563Z","version":"WzQ4Myw0XQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Pilots By GridSite (History) (using environment agnostic index pattern)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Pilots By GridSite (History) (using environment agnostic index pattern)\",\"type\":\"line\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"NumOfPilots\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"timestamp\",\"timeRange\":{\"from\":\"now-7d\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"GridSite\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":500,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"}],\"params\":{\"type\":\"line\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Sum of NumOfPilots\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"area\",\"mode\":\"stacked\",\"data\":{\"label\":\"Sum of NumOfPilots\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"interpolate\":\"step-after\",\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}"},"id":"dcc32a40-b2c6-11ed-baf6-bd7d09f916cf","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"7346c540-b2c6-11ed-baf6-bd7d09f916cf","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-02-22T15:41:15.410Z","version":"WzQ4OSw0XQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Pilots By Status (History) (using environment agnostic index pattern)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Pilots By Status (History) (using environment agnostic index pattern)\",\"type\":\"line\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"NumOfPilots\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"timestamp\",\"timeRange\":{\"from\":\"now-7d\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"Status\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":500,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"}],\"params\":{\"type\":\"line\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Sum of NumOfPilots\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"area\",\"mode\":\"stacked\",\"data\":{\"label\":\"Sum of NumOfPilots\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"interpolate\":\"step-after\",\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}"},"id":"fcfe0cd0-b2c6-11ed-baf6-bd7d09f916cf","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"7346c540-b2c6-11ed-baf6-bd7d09f916cf","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-02-22T15:41:39.502Z","version":"WzQ5MCw0XQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Pilots By TaskQueueID (History) (using environment agnostic index pattern)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Pilots By TaskQueueID (History) (using environment agnostic index pattern)\",\"type\":\"line\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"NumOfPilots\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"timestamp\",\"timeRange\":{\"from\":\"now-7d\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"TaskQueueID\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":500,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"}],\"params\":{\"type\":\"line\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Sum of NumOfPilots\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"area\",\"mode\":\"stacked\",\"data\":{\"label\":\"Sum of NumOfPilots\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"interpolate\":\"step-after\",\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}"},"id":"13d86c20-b2c7-11ed-baf6-bd7d09f916cf","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"7346c540-b2c6-11ed-baf6-bd7d09f916cf","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-02-22T15:42:36.985Z","version":"WzQ5Miw0XQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Pilots By GridType (History) (using environment agnostic index pattern)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Pilots By GridType (History) (using environment agnostic index pattern)\",\"type\":\"line\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"NumOfPilots\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"timestamp\",\"timeRange\":{\"from\":\"now-7d\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"GridType\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":500,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"}],\"params\":{\"type\":\"line\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Sum of NumOfPilots\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"area\",\"mode\":\"stacked\",\"data\":{\"label\":\"Sum of NumOfPilots\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"interpolate\":\"step-after\",\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"labels\":{},\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"}}}"},"id":"ee345830-b2c6-11ed-baf6-bd7d09f916cf","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"7346c540-b2c6-11ed-baf6-bd7d09f916cf","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-02-22T15:42:10.293Z","version":"WzQ5MSw0XQ=="} +{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}"},"optionsJSON":"{\"hidePanelTitles\":false,\"useMargins\":true}","panelsJSON":"[{\"version\":\"2.2.1\",\"gridData\":{\"h\":7,\"i\":\"f845a506-b6b2-4017-9cad-70be31b5d7c9\",\"w\":21,\"x\":15,\"y\":0},\"panelIndex\":\"f845a506-b6b2-4017-9cad-70be31b5d7c9\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_0\"},{\"version\":\"2.2.1\",\"gridData\":{\"x\":0,\"y\":7,\"w\":24,\"h\":15,\"i\":\"68001476-8cbe-4673-896c-4fd72d5655c2\"},\"panelIndex\":\"68001476-8cbe-4673-896c-4fd72d5655c2\",\"embeddableConfig\":{},\"panelRefName\":\"panel_1\"},{\"version\":\"2.2.1\",\"gridData\":{\"x\":24,\"y\":7,\"w\":24,\"h\":15,\"i\":\"b113ffcd-cb74-493a-8d37-d32ba1ce91aa\"},\"panelIndex\":\"b113ffcd-cb74-493a-8d37-d32ba1ce91aa\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"2.2.1\",\"gridData\":{\"x\":24,\"y\":22,\"w\":24,\"h\":15,\"i\":\"b21a717d-2c0b-4ca6-81c3-56a972f7581e\"},\"panelIndex\":\"b21a717d-2c0b-4ca6-81c3-56a972f7581e\",\"embeddableConfig\":{},\"panelRefName\":\"panel_3\"},{\"version\":\"2.2.1\",\"gridData\":{\"x\":0,\"y\":22,\"w\":24,\"h\":15,\"i\":\"e6aaea89-a5af-4291-9167-9d95ea968dda\"},\"panelIndex\":\"e6aaea89-a5af-4291-9167-9d95ea968dda\",\"embeddableConfig\":{},\"panelRefName\":\"panel_4\"}]","timeRestore":false,"title":"PilotsHistory Dashboard (using environment agnostic index pattern)","version":1},"id":"85bf7820-b2c6-11ed-baf6-bd7d09f916cf","migrationVersion":{"dashboard":"7.9.3"},"references":[{"id":"2807d260-c256-11ec-aa10-d1ab7e9f6014","name":"panel_0","type":"visualization"},{"id":"dcc32a40-b2c6-11ed-baf6-bd7d09f916cf","name":"panel_1","type":"visualization"},{"id":"fcfe0cd0-b2c6-11ed-baf6-bd7d09f916cf","name":"panel_2","type":"visualization"},{"id":"13d86c20-b2c7-11ed-baf6-bd7d09f916cf","name":"panel_3","type":"visualization"},{"id":"ee345830-b2c6-11ed-baf6-bd7d09f916cf","name":"panel_4","type":"visualization"}],"type":"dashboard","updated_at":"2023-02-22T15:47:10.089Z","version":"WzQ5Myw0XQ=="} +{"exportedCount":7,"missingRefCount":0,"missingReferences":[]} diff --git a/dashboards/ServiceMonitoring/ServiceMonitoring.json b/dashboards/ServiceMonitoring/ServiceMonitoring.json new file mode 100644 index 00000000000..4c3c9bc6905 --- /dev/null +++ b/dashboards/ServiceMonitoring/ServiceMonitoring.json @@ -0,0 +1,957 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 5632, + "iteration": 1685458469408, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "elasticsearch", + "uid": "MM7X_8wVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 15, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "max", + "mean", + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "ServiceName", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "MM7X_8wVz" + }, + "metrics": [ + { + "field": "CpuPercentage", + "id": "1", + "settings": { + "script": "_value / 100" + }, + "type": "sum" + } + ], + "query": "ServiceName: ($ServiceName)", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "CpuUsage by ServiceName", + "type": "timeseries" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "MM7X_8wVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decmbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 15, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "max", + "mean", + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "ServiceName", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "MM7X_8wVz" + }, + "metrics": [ + { + "field": "MemoryUsage", + "id": "1", + "settings": {}, + "type": "sum" + } + ], + "query": "ServiceName: ($ServiceName)", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "MemoryUsage by ServiceName", + "type": "timeseries" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "MM7X_8wVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "Max FD", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 15, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "max", + "mean", + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "ServiceName", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "MM7X_8wVz" + }, + "metrics": [ + { + "field": "MaxFD", + "id": "1", + "type": "max" + } + ], + "query": "ServiceName: ($ServiceName)", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Max FD by ServiceName", + "type": "timeseries" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "MM7X_8wVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "Running Threads", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 15, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "max", + "mean", + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "ServiceName", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "MM7X_8wVz" + }, + "metrics": [ + { + "field": "RunningThreads", + "id": "1", + "settings": {}, + "type": "sum" + } + ], + "query": "ServiceName: ($ServiceName)", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "RunningThreads by ServiceName", + "type": "timeseries" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "MM7X_8wVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "Respone time in seconds", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 15, + "w": 12, + "x": 0, + "y": 30 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "max", + "mean", + "last" + ], + "displayMode": "table", + "placement": "bottom", + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "ServiceName", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "MM7X_8wVz" + }, + "metrics": [ + { + "field": "ResponseTime", + "id": "1", + "settings": {}, + "type": "sum" + } + ], + "query": "ServiceName: ($ServiceName)", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "ResponeTime by ServiceName", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 45 + }, + "id": 10, + "panels": [], + "title": "Monitoring of Queries", + "type": "row" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "MM7X_8wVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "Active queries", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 15, + "w": 12, + "x": 0, + "y": 46 + }, + "id": 7, + "options": { + "legend": { + "calcs": [ + "max", + "mean", + "last" + ], + "displayMode": "table", + "placement": "bottom", + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "ServiceName", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "MM7X_8wVz" + }, + "metrics": [ + { + "field": "ActiveQueries", + "id": "1", + "settings": {}, + "type": "sum" + } + ], + "query": "ServiceName: ($ServiceName)", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Active Queries by ServiceName", + "type": "timeseries" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "MM7X_8wVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "Pending queries", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 15, + "w": 12, + "x": 12, + "y": 46 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "max", + "mean", + "last" + ], + "displayMode": "table", + "placement": "bottom", + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "ServiceName", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "MM7X_8wVz" + }, + "metrics": [ + { + "field": "PendingQueries", + "id": "1", + "settings": {}, + "type": "sum" + } + ], + "query": "ServiceName: ($ServiceName)", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Pending Queries by ServiceName", + "type": "timeseries" + } + ], + "refresh": false, + "schemaVersion": 36, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "elasticsearch", + "uid": "MM7X_8wVz" + }, + "definition": "{\"find\": \"terms\", \"field\": \"ServiceName\"}", + "hide": 0, + "includeAll": true, + "label": "Service Name", + "multi": true, + "name": "ServiceName", + "options": [], + "query": "{\"find\": \"terms\", \"field\": \"ServiceName\"}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Service Monitoring", + "uid": "9s0Ru8wVk", + "version": 6, + "weekStart": "" +} diff --git a/dashboards/ServiceMonitoring/ServiceMonitoringDashboard.ndjson b/dashboards/ServiceMonitoring/ServiceMonitoringDashboard.ndjson index 3a46a1d37b5..19ad6caf99e 100644 --- a/dashboards/ServiceMonitoring/ServiceMonitoringDashboard.ndjson +++ b/dashboards/ServiceMonitoring/ServiceMonitoringDashboard.ndjson @@ -1,12 +1,12 @@ -{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"title":"Title (ServiceMonitoring)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Title (ServiceMonitoring)\",\"type\":\"markdown\",\"aggs\":[],\"params\":{\"fontSize\":15,\"openLinksInNewTab\":false,\"markdown\":\"# **Service Monitoring Dashboard**\\n- Here are shown the `CPU` and `Memory Usage` of Services.\\n- Also `MaxFD`, `Running Threads`, `Response Time`, and `Queries` (active and pending)\\n\"}}"},"id":"c17da9b0-c797-11ec-aa10-d1ab7e9f6014","migrationVersion":{"visualization":"7.10.0"},"references":[],"type":"visualization","updated_at":"2022-04-29T13:36:37.810Z","version":"WzIyMjMyLDhd"} -{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"title":"CpuUsage by Service (ServiceMonitoring)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"CpuUsage by Service (ServiceMonitoring)\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#68BC00\",\"split_mode\":\"terms\",\"split_color_mode\":\"rainbow\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"sum\",\"field\":\"CpuPercentage\"},{\"id\":\"09eb0e10-c7aa-11ec-94e8-61a1e6d20a8a\",\"type\":\"math\",\"variables\":[{\"id\":\"0cf5b600-c7aa-11ec-94e8-61a1e6d20a8a\",\"name\":\"a\",\"field\":\"61ca57f2-469d-11e7-af02-69e470af7417\"}],\"script\":\"params.a/100\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"bar\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"stacked\",\"label\":\"Cpu Percentage \",\"type\":\"timeseries\",\"terms_field\":\"ServiceName\",\"value_template\":\"{{value}}%\",\"terms_size\":\"100\",\"terms_order_by\":\"61ca57f2-469d-11e7-af02-69e470af7417\"}],\"time_field\":\"timestamp\",\"index_pattern\":\"dirac-certification_service_monitoring-*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"dirac-certification_wmshistory_index-*\",\"default_timefield\":\"timestamp\",\"isModelInvalid\":false}}"},"id":"29349890-c6f6-11ec-aa10-d1ab7e9f6014","migrationVersion":{"visualization":"7.10.0"},"references":[],"type":"visualization","updated_at":"2022-04-29T13:34:19.323Z","version":"WzIyMTM0LDhd"} -{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"title":"MemoryUsage by Service (ServiceMonitoring)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"MemoryUsage by Service (ServiceMonitoring)\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#68BC00\",\"split_mode\":\"terms\",\"split_color_mode\":\"kibana\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"sum\",\"field\":\"MemoryUsage\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"stacked\",\"label\":\"Memory Usage\",\"type\":\"timeseries\",\"terms_field\":\"ServiceName\",\"terms_size\":\"20\",\"value_template\":\"{{value}}MB\",\"terms_order_by\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"steps\":0}],\"time_field\":\"\",\"index_pattern\":\"dirac-certification_service_monitoring-*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"dirac-certification_wmshistory_index-*\",\"default_timefield\":\"timestamp\",\"isModelInvalid\":false}}"},"id":"beff2750-c6f6-11ec-b508-cbe83a8a8723","migrationVersion":{"visualization":"7.10.0"},"references":[],"type":"visualization","updated_at":"2022-04-29T09:22:07.064Z","version":"WzIxNjI1LDhd"} -{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Max FD by Service","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Max FD by Service\",\"type\":\"area\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"MaxFD\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"timestamp\",\"timeRange\":{\"from\":\"now-24h\",\"to\":\"now\"},\"useNormalizedEsInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"ServiceName\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":20,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"}],\"params\":{\"type\":\"area\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"MaxFD\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"area\",\"mode\":\"stacked\",\"data\":{\"label\":\"Sum of MaxFD\",\"id\":\"1\"},\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true,\"interpolate\":\"linear\",\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"},\"labels\":{}}}"},"id":"a1736220-c799-11ec-b508-cbe83a8a8723","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"4c868650-c6ce-11ec-b67b-89dd8cb60ff9","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2022-04-29T13:34:40.227Z","version":"WzIyMTQzLDhd"} -{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Running Threads by Service","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Running Threads by Service\",\"type\":\"area\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"RunningThreads\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"timestamp\",\"timeRange\":{\"from\":\"now-24h\",\"to\":\"now\"},\"useNormalizedEsInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"ServiceName\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":20,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"}],\"params\":{\"type\":\"area\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Running Threads\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"area\",\"mode\":\"stacked\",\"data\":{\"label\":\"Sum of RunningThreads\",\"id\":\"1\"},\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true,\"interpolate\":\"linear\",\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"},\"labels\":{}}}"},"id":"3311ac50-c79a-11ec-b508-cbe83a8a8723","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"4c868650-c6ce-11ec-b67b-89dd8cb60ff9","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2022-04-29T13:35:51.011Z","version":"WzIyMjEyLDhd"} -{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Response Time by Service","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Response Time by Service\",\"type\":\"area\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"ResponseTime\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"timestamp\",\"timeRange\":{\"from\":\"now-24h\",\"to\":\"now\"},\"useNormalizedEsInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"ServiceName\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":20,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"}],\"params\":{\"type\":\"area\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Response Time in seconds\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"area\",\"mode\":\"stacked\",\"data\":{\"label\":\"Sum of ResponseTime\",\"id\":\"1\"},\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true,\"interpolate\":\"linear\",\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"},\"labels\":{}}}"},"id":"1bc921e0-c79a-11ec-b67b-89dd8cb60ff9","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"4c868650-c6ce-11ec-b67b-89dd8cb60ff9","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2022-04-29T13:35:25.649Z","version":"WzIyMTY0LDhd"} -{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"title":"queries title (ServiceMonitoring)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"queries title (ServiceMonitoring)\",\"type\":\"markdown\",\"aggs\":[],\"params\":{\"fontSize\":12,\"openLinksInNewTab\":false,\"markdown\":\"## Monitoring of Queries \\n\"}}"},"id":"6c7523f0-c79a-11ec-b508-cbe83a8a8723","migrationVersion":{"visualization":"7.10.0"},"references":[],"type":"visualization","updated_at":"2022-04-29T08:57:40.014Z","version":"WzIxNTMyLDhd"} -{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Active Queries by Service ","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Active Queries by Service \",\"type\":\"area\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"ActiveQueries\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"timestamp\",\"timeRange\":{\"from\":\"now-24h\",\"to\":\"now\"},\"useNormalizedEsInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"ServiceName\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":20,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"}],\"params\":{\"type\":\"area\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Active Queries\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"area\",\"mode\":\"stacked\",\"data\":{\"label\":\"Sum of ActiveQueries\",\"id\":\"1\"},\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true,\"interpolate\":\"linear\",\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"},\"labels\":{}}}"},"id":"fe6f34a0-c798-11ec-aa10-d1ab7e9f6014","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"4c868650-c6ce-11ec-b67b-89dd8cb60ff9","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2022-04-29T08:47:25.930Z","version":"WzIxNDE3LDhd"} -{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Pending Queries by Service","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Pending Queries by Service\",\"type\":\"area\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"PendingQueries\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"timestamp\",\"timeRange\":{\"from\":\"now-24h\",\"to\":\"now\"},\"useNormalizedEsInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"ServiceName\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":20,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"}],\"params\":{\"type\":\"area\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Pending Queries\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"area\",\"mode\":\"stacked\",\"data\":{\"label\":\"Sum of PendingQueries\",\"id\":\"1\"},\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true,\"interpolate\":\"linear\",\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"},\"labels\":{}}}"},"id":"073186a0-c79a-11ec-b67b-89dd8cb60ff9","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"4c868650-c6ce-11ec-b67b-89dd8cb60ff9","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2022-04-29T13:36:10.382Z","version":"WzIyMjIyLDhd"} -{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}"},"optionsJSON":"{\"hidePanelTitles\":false,\"useMargins\":true}","panelsJSON":"[{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":9,\"i\":\"3fd63247-82ff-413a-af80-af68dca057c6\",\"w\":20,\"x\":15,\"y\":0},\"panelIndex\":\"3fd63247-82ff-413a-af80-af68dca057c6\",\"version\":\"7.10.2\",\"panelRefName\":\"panel_0\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":15,\"i\":\"74c25948-8fc3-43e4-8bca-c7f0298bfc7f\",\"w\":24,\"x\":0,\"y\":9},\"panelIndex\":\"74c25948-8fc3-43e4-8bca-c7f0298bfc7f\",\"version\":\"7.10.2\",\"panelRefName\":\"panel_1\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":15,\"i\":\"2b1a74fa-9f8f-4b9f-8e4e-64bdf3228b76\",\"w\":24,\"x\":24,\"y\":9},\"panelIndex\":\"2b1a74fa-9f8f-4b9f-8e4e-64bdf3228b76\",\"version\":\"7.10.2\",\"panelRefName\":\"panel_2\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":15,\"i\":\"8b455f4e-f939-4c35-a44b-9d8edee19a7b\",\"w\":24,\"x\":0,\"y\":24},\"panelIndex\":\"8b455f4e-f939-4c35-a44b-9d8edee19a7b\",\"version\":\"7.10.2\",\"panelRefName\":\"panel_3\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":15,\"i\":\"72a1f15f-df7c-497f-b0dc-06c4a1c44052\",\"w\":24,\"x\":24,\"y\":24},\"panelIndex\":\"72a1f15f-df7c-497f-b0dc-06c4a1c44052\",\"version\":\"7.10.2\",\"panelRefName\":\"panel_4\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":15,\"i\":\"64157742-5ff5-4986-8d3f-526f9e9ce759\",\"w\":24,\"x\":13,\"y\":39},\"panelIndex\":\"64157742-5ff5-4986-8d3f-526f9e9ce759\",\"version\":\"7.10.2\",\"panelRefName\":\"panel_5\"},{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":4,\"i\":\"1021bd77-a329-47ba-9c47-ac363a405c6f\",\"w\":9,\"x\":19,\"y\":54},\"panelIndex\":\"1021bd77-a329-47ba-9c47-ac363a405c6f\",\"version\":\"7.10.2\",\"panelRefName\":\"panel_6\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":15,\"i\":\"950cd33b-da3b-492a-9154-c8da2f5e41a9\",\"w\":24,\"x\":0,\"y\":58},\"panelIndex\":\"950cd33b-da3b-492a-9154-c8da2f5e41a9\",\"version\":\"7.10.2\",\"panelRefName\":\"panel_7\"},{\"embeddableConfig\":{},\"gridData\":{\"h\":15,\"i\":\"80647d0c-331f-46d3-97a3-d3d292ef5fa1\",\"w\":24,\"x\":24,\"y\":58},\"panelIndex\":\"80647d0c-331f-46d3-97a3-d3d292ef5fa1\",\"version\":\"7.10.2\",\"panelRefName\":\"panel_8\"}]","refreshInterval":{"pause":true,"value":0},"timeFrom":"now-24h","timeRestore":true,"timeTo":"now","title":"Service Monitoring Dashboard","version":1},"id":"e1439580-c6f6-11ec-b508-cbe83a8a8723","migrationVersion":{"dashboard":"7.9.3"},"references":[{"id":"c17da9b0-c797-11ec-aa10-d1ab7e9f6014","name":"panel_0","type":"visualization"},{"id":"29349890-c6f6-11ec-aa10-d1ab7e9f6014","name":"panel_1","type":"visualization"},{"id":"beff2750-c6f6-11ec-b508-cbe83a8a8723","name":"panel_2","type":"visualization"},{"id":"a1736220-c799-11ec-b508-cbe83a8a8723","name":"panel_3","type":"visualization"},{"id":"3311ac50-c79a-11ec-b508-cbe83a8a8723","name":"panel_4","type":"visualization"},{"id":"1bc921e0-c79a-11ec-b67b-89dd8cb60ff9","name":"panel_5","type":"visualization"},{"id":"6c7523f0-c79a-11ec-b508-cbe83a8a8723","name":"panel_6","type":"visualization"},{"id":"fe6f34a0-c798-11ec-aa10-d1ab7e9f6014","name":"panel_7","type":"visualization"},{"id":"073186a0-c79a-11ec-b67b-89dd8cb60ff9","name":"panel_8","type":"visualization"}],"type":"dashboard","updated_at":"2022-04-29T13:44:16.911Z","version":"WzIyMjg1LDhd"} -{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[{\"meta\":{\"negate\":false,\"disabled\":false,\"alias\":null,\"type\":\"phrase\",\"key\":\"message.keyword\",\"value\":\"Executing action\",\"params\":{\"query\":\"Executing action\",\"type\":\"phrase\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match\":{\"message.keyword\":{\"query\":\"Executing action\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}}],\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Service queries per host","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Service queries per host\",\"type\":\"histogram\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"hostname.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":15,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"componentname.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}]}"},"id":"7d982120-4a28-11e9-92db-b12061b7ddd0","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"40fb1d70-5999-11ec-94f9-b5f7798e60d5","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"40fb1d70-5999-11ec-94f9-b5f7798e60d5","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"},{"id":"40fb1d70-5999-11ec-94f9-b5f7798e60d5","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"40fb1d70-5999-11ec-94f9-b5f7798e60d5","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"}],"type":"visualization","updated_at":"2021-12-10T09:12:27.676Z","version":"WzUwNzcsNF0="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"title":"Title (ServiceMonitoring)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Title (ServiceMonitoring)\",\"type\":\"markdown\",\"aggs\":[],\"params\":{\"fontSize\":15,\"openLinksInNewTab\":false,\"markdown\":\"# **Service Monitoring Dashboard**\\n- Here are shown the `CPU` and `Memory Usage` of Services.\\n- Also `MaxFD`, `Running Threads`, `Response Time`, and `Queries` (active and pending)\\n\"}}"},"id":"c17da9b0-c797-11ec-aa10-d1ab7e9f6014","migrationVersion":{"visualization":"7.10.0"},"references":[],"type":"visualization","updated_at":"2022-07-28T10:37:48.444Z","version":"WzE3NiwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"title":"queries title (ServiceMonitoring)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"queries title (ServiceMonitoring)\",\"type\":\"markdown\",\"aggs\":[],\"params\":{\"fontSize\":12,\"openLinksInNewTab\":false,\"markdown\":\"## Monitoring of Queries \\n\"}}"},"id":"6c7523f0-c79a-11ec-b508-cbe83a8a8723","migrationVersion":{"visualization":"7.10.0"},"references":[],"type":"visualization","updated_at":"2022-07-28T10:37:48.444Z","version":"WzE4MiwxXQ=="} +{"attributes":{"fields":"[{\"count\":0,\"name\":\"ActiveQueries\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"Connections\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"CpuPercentage\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"Host\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"Location\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"MaxFD\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"MemoryUsage\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"PendingQueries\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"Queries\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"ResponseTime\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"RunningThreads\",\"type\":\"number\",\"esTypes\":[\"long\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"ServiceName\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"Status\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_score\",\"type\":\"number\",\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_type\",\"type\":\"string\",\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true}]","timeFieldName":"timestamp","title":"*_service_monitoring-*"},"id":"15d86330-b2b9-11ed-baf6-bd7d09f916cf","migrationVersion":{"index-pattern":"7.6.0"},"references":[],"type":"index-pattern","updated_at":"2023-02-22T13:59:12.354Z","version":"WzQ2Myw0XQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Pending Queries By Service (using environment agnostic index pattern)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Pending Queries By Service (using environment agnostic index pattern)\",\"type\":\"area\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"PendingQueries\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"timestamp\",\"timeRange\":{\"from\":\"now-24h\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"ServiceName\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":500,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"}],\"params\":{\"type\":\"area\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Sum of PendingQueries\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"area\",\"mode\":\"stacked\",\"data\":{\"label\":\"Sum of PendingQueries\",\"id\":\"1\"},\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true,\"interpolate\":\"linear\",\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"},\"labels\":{}}}"},"id":"4a4640e0-b2bb-11ed-baf6-bd7d09f916cf","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"15d86330-b2b9-11ed-baf6-bd7d09f916cf","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-02-22T14:16:37.720Z","version":"WzQ2OSw0XQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Active Queries By Service (using environment agnostic index pattern)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Active Queries By Service (using environment agnostic index pattern)\",\"type\":\"area\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"ActiveQueries\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"timestamp\",\"timeRange\":{\"from\":\"now-24h\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"ServiceName\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":500,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"}],\"params\":{\"type\":\"area\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Sum of ActiveQueries\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"area\",\"mode\":\"stacked\",\"data\":{\"label\":\"Sum of ActiveQueries\",\"id\":\"1\"},\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true,\"interpolate\":\"linear\",\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"},\"labels\":{}}}"},"id":"c6ff9320-b2bb-11ed-baf6-bd7d09f916cf","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"15d86330-b2b9-11ed-baf6-bd7d09f916cf","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-02-22T14:18:28.562Z","version":"WzQ3MCw0XQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Response Time By Service (using environment agnostic index pattern)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Response Time By Service (using environment agnostic index pattern)\",\"type\":\"area\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"ResponseTime\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"timestamp\",\"timeRange\":{\"from\":\"now-24h\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"ServiceName\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":500,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"}],\"params\":{\"type\":\"area\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Sum of ResponseTime\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"area\",\"mode\":\"stacked\",\"data\":{\"label\":\"Sum of ResponseTime\",\"id\":\"1\"},\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true,\"interpolate\":\"linear\",\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"},\"labels\":{}}}"},"id":"dbd4d3a0-b2bb-11ed-baf6-bd7d09f916cf","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"15d86330-b2b9-11ed-baf6-bd7d09f916cf","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-02-22T14:19:03.514Z","version":"WzQ3MSw0XQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Running Threads By Service (using environment agnostic index pattern)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Running Threads By Service (using environment agnostic index pattern)\",\"type\":\"area\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"RunningThreads\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"timestamp\",\"timeRange\":{\"from\":\"now-24h\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"ServiceName\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":500,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"}],\"params\":{\"type\":\"area\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Sum of RunningThreads\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"area\",\"mode\":\"stacked\",\"data\":{\"label\":\"Sum of RunningThreads\",\"id\":\"1\"},\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true,\"interpolate\":\"linear\",\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"},\"labels\":{}}}"},"id":"36ef6300-b2bb-11ed-baf6-bd7d09f916cf","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"15d86330-b2b9-11ed-baf6-bd7d09f916cf","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-02-22T14:14:26.864Z","version":"WzQ2Niw0XQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Max FD By Service (using environment agnostic index pattern)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Max FD By Service (using environment agnostic index pattern)\",\"type\":\"area\",\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"sum\",\"params\":{\"field\":\"MaxFD\"},\"schema\":\"metric\"},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"params\":{\"field\":\"timestamp\",\"timeRange\":{\"from\":\"now-24h\",\"to\":\"now\"},\"useNormalizedOpenSearchInterval\":true,\"scaleMetricValues\":false,\"interval\":\"auto\",\"drop_partials\":false,\"min_doc_count\":1,\"extended_bounds\":{}},\"schema\":\"segment\"},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"params\":{\"field\":\"ServiceName\",\"orderBy\":\"1\",\"order\":\"desc\",\"size\":500,\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\"},\"schema\":\"group\"}],\"params\":{\"type\":\"area\",\"grid\":{\"categoryLines\":false},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"filter\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Sum of MaxFD\"}}],\"seriesParams\":[{\"show\":true,\"type\":\"area\",\"mode\":\"stacked\",\"data\":{\"label\":\"Sum of MaxFD\",\"id\":\"1\"},\"drawLinesBetweenPoints\":true,\"lineWidth\":2,\"showCircles\":true,\"interpolate\":\"linear\",\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false,\"thresholdLine\":{\"show\":false,\"value\":10,\"width\":1,\"style\":\"full\",\"color\":\"#E7664C\"},\"labels\":{}}}"},"id":"1613fab0-b2bb-11ed-baf6-bd7d09f916cf","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"15d86330-b2b9-11ed-baf6-bd7d09f916cf","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-02-22T14:13:31.739Z","version":"WzQ2NSw0XQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"title":"CpuUsage by Service (ServiceMonitoring) (using environment agnostic index pattern)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"CpuUsage by Service (ServiceMonitoring) (using environment agnostic index pattern)\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#68BC00\",\"split_mode\":\"terms\",\"split_color_mode\":\"rainbow\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"sum\",\"field\":\"CpuPercentage\"},{\"id\":\"09eb0e10-c7aa-11ec-94e8-61a1e6d20a8a\",\"type\":\"math\",\"variables\":[{\"id\":\"0cf5b600-c7aa-11ec-94e8-61a1e6d20a8a\",\"name\":\"a\",\"field\":\"61ca57f2-469d-11e7-af02-69e470af7417\"}],\"script\":\"params.a/100\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"bar\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"stacked\",\"label\":\"Cpu Percentage \",\"type\":\"timeseries\",\"terms_field\":\"ServiceName\",\"value_template\":\"{{value}}%\",\"terms_size\":\"100\",\"terms_order_by\":\"61ca57f2-469d-11e7-af02-69e470af7417\"}],\"time_field\":\"timestamp\",\"index_pattern\":\"*_service_monitoring-*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"lhcb-certification_wmshistory_index-*\",\"default_timefield\":\"timestamp\",\"isModelInvalid\":false}}"},"id":"91649fc0-b2bc-11ed-baf6-bd7d09f916cf","migrationVersion":{"visualization":"7.10.0"},"references":[],"type":"visualization","updated_at":"2023-02-22T14:24:08.124Z","version":"WzQ3Myw0XQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"title":"MemoryUsage by Service (ServiceMonitoring) (using environment agnostic index pattern)","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"MemoryUsage by Service (ServiceMonitoring) (using environment agnostic index pattern)\",\"type\":\"metrics\",\"aggs\":[],\"params\":{\"id\":\"61ca57f0-469d-11e7-af02-69e470af7417\",\"type\":\"timeseries\",\"series\":[{\"id\":\"61ca57f1-469d-11e7-af02-69e470af7417\",\"color\":\"#68BC00\",\"split_mode\":\"terms\",\"split_color_mode\":\"kibana\",\"metrics\":[{\"id\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"type\":\"sum\",\"field\":\"MemoryUsage\"}],\"separate_axis\":0,\"axis_position\":\"right\",\"formatter\":\"number\",\"chart_type\":\"line\",\"line_width\":1,\"point_size\":1,\"fill\":0.5,\"stacked\":\"stacked\",\"label\":\"Memory Usage\",\"type\":\"timeseries\",\"terms_field\":\"ServiceName\",\"terms_size\":\"200\",\"value_template\":\"{{value}}MB\",\"terms_order_by\":\"61ca57f2-469d-11e7-af02-69e470af7417\",\"steps\":0}],\"time_field\":\"timestamp\",\"index_pattern\":\"*_service_monitoring-*\",\"interval\":\"\",\"axis_position\":\"left\",\"axis_formatter\":\"number\",\"axis_scale\":\"normal\",\"show_legend\":1,\"show_grid\":1,\"tooltip_mode\":\"show_all\",\"default_index_pattern\":\"lhcb-certification_wmshistory_index-*\",\"default_timefield\":\"timestamp\",\"isModelInvalid\":false}}"},"id":"7bb44c10-b2bd-11ed-baf6-bd7d09f916cf","migrationVersion":{"visualization":"7.10.0"},"references":[],"type":"visualization","updated_at":"2023-02-22T14:30:50.600Z","version":"WzQ3NSw0XQ=="} +{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}"},"optionsJSON":"{\"hidePanelTitles\":false,\"useMargins\":true}","panelsJSON":"[{\"version\":\"2.2.1\",\"gridData\":{\"h\":9,\"i\":\"3fd63247-82ff-413a-af80-af68dca057c6\",\"w\":20,\"x\":15,\"y\":0},\"panelIndex\":\"3fd63247-82ff-413a-af80-af68dca057c6\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_0\"},{\"version\":\"2.2.1\",\"gridData\":{\"h\":4,\"i\":\"1021bd77-a329-47ba-9c47-ac363a405c6f\",\"w\":9,\"x\":19,\"y\":54},\"panelIndex\":\"1021bd77-a329-47ba-9c47-ac363a405c6f\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_1\"},{\"version\":\"2.2.1\",\"gridData\":{\"h\":15,\"i\":\"8e61c1fd-9adc-4c1b-917b-7984c5b2a981\",\"w\":24,\"x\":24,\"y\":58},\"panelIndex\":\"8e61c1fd-9adc-4c1b-917b-7984c5b2a981\",\"embeddableConfig\":{},\"panelRefName\":\"panel_2\"},{\"version\":\"2.2.1\",\"gridData\":{\"h\":15,\"i\":\"7f9c6ef9-7f31-4bbb-ae9a-87e1747dd25c\",\"w\":24,\"x\":0,\"y\":58},\"panelIndex\":\"7f9c6ef9-7f31-4bbb-ae9a-87e1747dd25c\",\"embeddableConfig\":{},\"panelRefName\":\"panel_3\"},{\"version\":\"2.2.1\",\"gridData\":{\"h\":15,\"i\":\"900e1048-3533-479b-98a4-956e785b6fe4\",\"w\":24,\"x\":13,\"y\":39},\"panelIndex\":\"900e1048-3533-479b-98a4-956e785b6fe4\",\"embeddableConfig\":{},\"panelRefName\":\"panel_4\"},{\"version\":\"2.2.1\",\"gridData\":{\"h\":15,\"i\":\"b32d11d3-d50e-4a96-ad28-b374640753e2\",\"w\":24,\"x\":24,\"y\":24},\"panelIndex\":\"b32d11d3-d50e-4a96-ad28-b374640753e2\",\"embeddableConfig\":{},\"panelRefName\":\"panel_5\"},{\"version\":\"2.2.1\",\"gridData\":{\"h\":15,\"i\":\"8fc5f301-74da-40ba-b38e-e5c40942bd4b\",\"w\":24,\"x\":0,\"y\":24},\"panelIndex\":\"8fc5f301-74da-40ba-b38e-e5c40942bd4b\",\"embeddableConfig\":{},\"panelRefName\":\"panel_6\"},{\"version\":\"2.2.1\",\"gridData\":{\"h\":15,\"i\":\"612149d8-6fa6-4b65-bbee-c9c79b393f4a\",\"w\":24,\"x\":0,\"y\":9},\"panelIndex\":\"612149d8-6fa6-4b65-bbee-c9c79b393f4a\",\"embeddableConfig\":{},\"panelRefName\":\"panel_7\"},{\"version\":\"2.2.1\",\"gridData\":{\"x\":24,\"y\":9,\"w\":24,\"h\":15,\"i\":\"af998ab6-a286-4ba0-b5cd-388fac49eff7\"},\"panelIndex\":\"af998ab6-a286-4ba0-b5cd-388fac49eff7\",\"embeddableConfig\":{},\"panelRefName\":\"panel_8\"}]","timeRestore":false,"title":"Service Monitoring Dashboard (using environment agnostic index pattern)","version":1},"id":"94fc3050-b2ba-11ed-baf6-bd7d09f916cf","migrationVersion":{"dashboard":"7.9.3"},"references":[{"id":"c17da9b0-c797-11ec-aa10-d1ab7e9f6014","name":"panel_0","type":"visualization"},{"id":"6c7523f0-c79a-11ec-b508-cbe83a8a8723","name":"panel_1","type":"visualization"},{"id":"4a4640e0-b2bb-11ed-baf6-bd7d09f916cf","name":"panel_2","type":"visualization"},{"id":"c6ff9320-b2bb-11ed-baf6-bd7d09f916cf","name":"panel_3","type":"visualization"},{"id":"dbd4d3a0-b2bb-11ed-baf6-bd7d09f916cf","name":"panel_4","type":"visualization"},{"id":"36ef6300-b2bb-11ed-baf6-bd7d09f916cf","name":"panel_5","type":"visualization"},{"id":"1613fab0-b2bb-11ed-baf6-bd7d09f916cf","name":"panel_6","type":"visualization"},{"id":"91649fc0-b2bc-11ed-baf6-bd7d09f916cf","name":"panel_7","type":"visualization"},{"id":"7bb44c10-b2bd-11ed-baf6-bd7d09f916cf","name":"panel_8","type":"visualization"}],"type":"dashboard","updated_at":"2023-02-22T14:32:40.006Z","version":"WzQ3Niw0XQ=="} {"exportedCount":11,"missingRefCount":0,"missingReferences":[]} diff --git a/dashboards/WMS/WMSHistory.json b/dashboards/WMS/WMSHistory.json new file mode 100644 index 00000000000..155ea4dc957 --- /dev/null +++ b/dashboards/WMS/WMSHistory.json @@ -0,0 +1,1141 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 5633, + "iteration": 1685458517587, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "elasticsearch", + "uid": "yhaSzQQVk" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 16, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 8, + "options": { + "displayLabels": [ + "name", + "value" + ], + "legend": { + "displayMode": "hidden", + "placement": "bottom" + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "none", + "sort": "desc" + } + }, + "pluginVersion": "8.5.21", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Status", + "id": "9", + "settings": { + "min_doc_count": "1", + "missing": "Other", + "order": "desc", + "orderBy": "_term", + "size": "20" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "10", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "yhaSzQQVk" + }, + "metrics": [ + { + "field": "Jobs", + "id": "1", + "settings": {}, + "type": "sum" + } + ], + "query": "", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Jobs by Status", + "type": "piechart" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "yhaSzQQVk" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "Jobs", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 16, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 69, + "options": { + "legend": { + "calcs": [ + "max", + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "8.5.21", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Status", + "id": "9", + "settings": { + "min_doc_count": "1", + "missing": "Other", + "order": "desc", + "orderBy": "_term", + "size": "20" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "10", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "yhaSzQQVk" + }, + "metrics": [ + { + "field": "Jobs", + "id": "1", + "settings": {}, + "type": "sum" + } + ], + "query": "", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Jobs by Status", + "type": "timeseries" + }, + { + "collapsed": true, + "datasource": { + "type": "influxdb", + "uid": "000009465" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 2, + "panels": [ + { + "datasource": { + "type": "elasticsearch", + "uid": "yhaSzQQVk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 44, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "/^Sum Jobs$/", + "values": false + }, + "text": { + "valueSize": 200 + }, + "textMode": "auto" + }, + "pluginVersion": "8.5.21", + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "15m" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "yhaSzQQVk" + }, + "metrics": [ + { + "field": "Jobs", + "hide": false, + "id": "1", + "type": "sum" + }, + { + "field": "1", + "hide": true, + "id": "3", + "pipelineAgg": "1", + "settings": { + "model": "simple", + "window": "5" + }, + "type": "moving_avg" + } + ], + "query": "Status: $status", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "Number of $status Jobs", + "type": "stat" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "yhaSzQQVk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 0, + "y": 20 + }, + "id": 10, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean", + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "User", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "yhaSzQQVk" + }, + "metrics": [ + { + "field": "Jobs", + "id": "1", + "type": "sum" + } + ], + "query": "Status: ${status:lucene}", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "$status Jobs by All Users", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "yhaSzQQVk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 8, + "y": 20 + }, + "id": 37, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean", + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "JobGroup", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "yhaSzQQVk" + }, + "metrics": [ + { + "field": "Jobs", + "id": "1", + "type": "sum" + } + ], + "query": "Status: ${status:lucene}", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "$status Jobs by JobGroup", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "yhaSzQQVk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 16, + "y": 20 + }, + "id": 38, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean", + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "JobSplitType", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "yhaSzQQVk" + }, + "metrics": [ + { + "field": "Jobs", + "id": "1", + "type": "sum" + } + ], + "query": "Status: ${status:lucene}", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "$status Jobs by JobSplitType", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "yhaSzQQVk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 0, + "y": 31 + }, + "id": 39, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean", + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "UserGroup", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "yhaSzQQVk" + }, + "metrics": [ + { + "field": "Jobs", + "id": "1", + "type": "sum" + } + ], + "query": "Status: ${status:lucene}", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "$status Jobs by UserGroup", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "yhaSzQQVk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 8, + "y": 31 + }, + "id": 40, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean", + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "Site", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "yhaSzQQVk" + }, + "metrics": [ + { + "field": "Jobs", + "id": "1", + "type": "sum" + } + ], + "query": "Status: ${status:lucene}", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "$status Jobs by Site", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "yhaSzQQVk" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 8, + "x": 16, + "y": 31 + }, + "id": 41, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean", + "last" + ], + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "MinorStatus", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "1", + "size": "0" + }, + "type": "terms" + }, + { + "field": "timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "yhaSzQQVk" + }, + "metrics": [ + { + "field": "Jobs", + "id": "1", + "type": "sum" + } + ], + "query": "Status: ${status:lucene}", + "refId": "A", + "timeField": "timestamp" + } + ], + "title": "$status Jobs by MinorStatus", + "transformations": [], + "type": "timeseries" + } + ], + "repeat": "status", + "title": "$status Jobs", + "type": "row" + } + ], + "schemaVersion": 36, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "allValue": "*", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "elasticsearch", + "uid": "yhaSzQQVk" + }, + "definition": "{\"find\":\"terms\", \"field\": \"Status\"}", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "status", + "options": [], + "query": "{\"find\":\"terms\", \"field\": \"Status\"}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "WMS History", + "uid": "tk15RQw4z", + "version": 11, + "weekStart": "" +} diff --git a/dashboards/tornadoLogs/grafana/DIRAC_Metrics.json b/dashboards/tornadoLogs/grafana/DIRAC_Metrics.json new file mode 100644 index 00000000000..9c7a27130da --- /dev/null +++ b/dashboards/tornadoLogs/grafana/DIRAC_Metrics.json @@ -0,0 +1,573 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 56, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "elasticsearch", + "uid": "BcHFbIBVz" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "hostname.keyword", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "10s", + "min_doc_count": "0", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "BcHFbIBVz" + }, + "metrics": [ + { + "field": "cpu_p", + "id": "1", + "type": "sum" + } + ], + "query": "", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "Dirac cpu load", + "type": "timeseries" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "BcHFbIBVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "hostname.keyword", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "10s", + "min_doc_count": "0", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "BcHFbIBVz" + }, + "hide": false, + "metrics": [ + { + "field": "Mem.used", + "hide": true, + "id": "1", + "type": "sum" + }, + { + "field": "Mem.total", + "hide": true, + "id": "4", + "type": "sum" + }, + { + "hide": false, + "id": "5", + "pipelineVariables": [ + { + "name": "var1", + "pipelineAgg": "1" + }, + { + "name": "var2", + "pipelineAgg": "4" + } + ], + "settings": { + "script": "params.var1/params.var2" + }, + "type": "bucket_script" + } + ], + "query": "", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "Dirac memory usage", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "BcHFbIBVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binbps" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "hostname.keyword", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "10s", + "min_doc_count": "0", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "BcHFbIBVz" + }, + "hide": false, + "metrics": [ + { + "field": "read_size", + "hide": false, + "id": "1", + "type": "sum" + } + ], + "query": "", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "Dirac disk throughput (read)", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "BcHFbIBVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binbps" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "hostname.keyword", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "10s", + "min_doc_count": "0", + "timeZone": "utc", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "BcHFbIBVz" + }, + "hide": false, + "metrics": [ + { + "field": "write_size", + "hide": false, + "id": "1", + "type": "sum" + } + ], + "query": "", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "Dirac disk throughput (write)", + "transformations": [], + "type": "timeseries" + } + ], + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "DIRAC Metrics", + "uid": "dPzqOdE4k", + "version": 30, + "weekStart": "" +} diff --git a/dashboards/tornadoLogs/grafana/Dirac_Logs_ALL.json b/dashboards/tornadoLogs/grafana/Dirac_Logs_ALL.json new file mode 100644 index 00000000000..c80cb72cfe7 --- /dev/null +++ b/dashboards/tornadoLogs/grafana/Dirac_Logs_ALL.json @@ -0,0 +1,368 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 54, + "links": [], + "liveNow": true, + "panels": [ + { + "datasource": { + "type": "elasticsearch", + "uid": "BcHFbIBVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed" + }, + "custom": { + "align": "auto", + "displayMode": "color-text", + "filterable": false, + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "componentname" + }, + "properties": [ + { + "id": "custom.width", + "value": 136 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "@timestamp" + }, + "properties": [ + { + "id": "custom.width", + "value": 168 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "_id" + }, + "properties": [ + { + "id": "custom.width", + "value": 57 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "_index" + }, + "properties": [ + { + "id": "custom.width", + "value": 81 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "_type" + }, + "properties": [ + { + "id": "custom.width", + "value": 58 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "asctime" + }, + "properties": [ + { + "id": "custom.width", + "value": 80 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "customname" + }, + "properties": [ + { + "id": "custom.width", + "value": 289 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "highlight" + }, + "properties": [ + { + "id": "custom.width", + "value": 78 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "levelname" + }, + "properties": [ + { + "id": "custom.width", + "value": 102 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "varmessage" + }, + "properties": [ + { + "id": "custom.width", + "value": 708 + } + ] + } + ] + }, + "gridPos": { + "h": 23, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 4, + "options": { + "footer": { + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "@timestamp" + } + ] + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "alias": "", + "bucketAggs": [], + "datasource": { + "type": "elasticsearch", + "uid": "BcHFbIBVz" + }, + "metrics": [ + { + "id": "1", + "settings": { + "size": "1000" + }, + "type": "raw_data" + } + ], + "query": "componentname:$componentname AND customname:$customname AND levelname:$levelname", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "Logs Table", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "_id": true, + "_index": true, + "_type": true, + "asctime": true, + "highlight": true, + "sort": true, + "tornadoComponent": true + }, + "indexByName": { + "@timestamp": 0, + "_id": 1, + "_index": 2, + "_type": 3, + "asctime": 4, + "componentname": 5, + "customname": 6, + "highlight": 7, + "hostname": 12, + "levelname": 8, + "message": 9, + "sort": 10, + "varmessage": 11 + }, + "renameByName": {} + } + } + ], + "type": "table" + } + ], + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "elasticsearch", + "uid": "BcHFbIBVz" + }, + "definition": "{\"find\":\"terms\", \"field\":\"componentname.keyword\",\"query\":\"*\"}", + "hide": 0, + "includeAll": true, + "label": "", + "multi": true, + "name": "componentname", + "options": [], + "query": "{\"find\":\"terms\", \"field\":\"componentname.keyword\",\"query\":\"*\"}", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "elasticsearch", + "uid": "BcHFbIBVz" + }, + "definition": "{\"find\":\"terms\", \"field\":\"customname.keyword\",\"query\":\"*\"}", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "customname", + "options": [], + "query": "{\"find\":\"terms\", \"field\":\"customname.keyword\",\"query\":\"*\"}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "elasticsearch", + "uid": "BcHFbIBVz" + }, + "definition": "{\"find\":\"terms\", \"field\":\"levelname.keyword\",\"query\":\"*\"}", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "levelname", + "options": [], + "query": "{\"find\":\"terms\", \"field\":\"levelname.keyword\",\"query\":\"*\"}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Dirac Logs ALL", + "uid": "SXvsbSBVk", + "version": 63, + "weekStart": "" +} diff --git a/dashboards/tornadoLogs/grafana/Dirac_Logs_DEV.json b/dashboards/tornadoLogs/grafana/Dirac_Logs_DEV.json new file mode 100644 index 00000000000..94f5f51e274 --- /dev/null +++ b/dashboards/tornadoLogs/grafana/Dirac_Logs_DEV.json @@ -0,0 +1,325 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 57, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "elasticsearch", + "uid": "BcHFbIBVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed" + }, + "custom": { + "align": "auto", + "displayMode": "color-text", + "filterable": false, + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "componentname" + }, + "properties": [ + { + "id": "custom.width", + "value": 136 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "@timestamp" + }, + "properties": [ + { + "id": "custom.width", + "value": 168 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "_id" + }, + "properties": [ + { + "id": "custom.width", + "value": 57 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "_index" + }, + "properties": [ + { + "id": "custom.width", + "value": 81 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "_type" + }, + "properties": [ + { + "id": "custom.width", + "value": 58 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "asctime" + }, + "properties": [ + { + "id": "custom.width", + "value": 80 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "customname" + }, + "properties": [ + { + "id": "custom.width", + "value": 289 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "highlight" + }, + "properties": [ + { + "id": "custom.width", + "value": 78 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "levelname" + }, + "properties": [ + { + "id": "custom.width", + "value": 102 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "varmessage" + }, + "properties": [ + { + "id": "custom.width", + "value": 708 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "message" + }, + "properties": [ + { + "id": "custom.width", + "value": 1445 + } + ] + } + ] + }, + "gridPos": { + "h": 23, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 4, + "options": { + "footer": { + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "@timestamp" + } + ] + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "alias": "", + "bucketAggs": [], + "datasource": { + "type": "elasticsearch", + "uid": "BcHFbIBVz" + }, + "metrics": [ + { + "id": "1", + "settings": { + "size": "1000" + }, + "type": "raw_data" + } + ], + "query": "levelname:$levelname", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "Logs Table", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "_id": true, + "_index": true, + "_type": true, + "asctime": true, + "highlight": true, + "sort": true, + "tornadoComponent": true + }, + "indexByName": { + "@timestamp": 0, + "_id": 1, + "_index": 2, + "_type": 3, + "asctime": 4, + "componentname": 5, + "customname": 6, + "highlight": 7, + "hostname": 12, + "levelname": 8, + "message": 9, + "sort": 10, + "varmessage": 11 + }, + "renameByName": {} + } + } + ], + "type": "table" + } + ], + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "DEV" + ], + "value": [ + "DEV" + ] + }, + "datasource": { + "type": "elasticsearch", + "uid": "BcHFbIBVz" + }, + "definition": "{\"find\":\"terms\", \"field\":\"levelname.keyword\",\"query\":\"*\"}", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "levelname", + "options": [], + "query": "{\"find\":\"terms\", \"field\":\"levelname.keyword\",\"query\":\"*\"}", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Dirac Logs DEV", + "uid": "SXvsbSBVm", + "version": 5, + "weekStart": "" +} diff --git a/dashboards/tornadoLogs/grafana/Dirac_Logs_SECURITY.json b/dashboards/tornadoLogs/grafana/Dirac_Logs_SECURITY.json new file mode 100644 index 00000000000..37a51bc75c2 --- /dev/null +++ b/dashboards/tornadoLogs/grafana/Dirac_Logs_SECURITY.json @@ -0,0 +1,318 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 58, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "elasticsearch", + "uid": "BcHFbIBVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed" + }, + "custom": { + "align": "auto", + "displayMode": "color-text", + "filterable": false, + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "componentname" + }, + "properties": [ + { + "id": "custom.width", + "value": 136 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "@timestamp" + }, + "properties": [ + { + "id": "custom.width", + "value": 168 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "_id" + }, + "properties": [ + { + "id": "custom.width", + "value": 57 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "_index" + }, + "properties": [ + { + "id": "custom.width", + "value": 81 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "_type" + }, + "properties": [ + { + "id": "custom.width", + "value": 58 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "asctime" + }, + "properties": [ + { + "id": "custom.width", + "value": 80 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "customname" + }, + "properties": [ + { + "id": "custom.width", + "value": 289 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "highlight" + }, + "properties": [ + { + "id": "custom.width", + "value": 78 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "levelname" + }, + "properties": [ + { + "id": "custom.width", + "value": 102 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "varmessage" + }, + "properties": [ + { + "id": "custom.width", + "value": 708 + } + ] + } + ] + }, + "gridPos": { + "h": 23, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 4, + "options": { + "footer": { + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "@timestamp" + } + ] + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "alias": "", + "bucketAggs": [], + "datasource": { + "type": "elasticsearch", + "uid": "BcHFbIBVz" + }, + "metrics": [ + { + "id": "1", + "settings": { + "size": "1000" + }, + "type": "raw_data" + } + ], + "query": "message:$sec_message", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "Logs Table", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "_id": true, + "_index": true, + "_type": true, + "asctime": true, + "highlight": true, + "sort": true, + "tornadoComponent": true + }, + "indexByName": { + "@timestamp": 0, + "_id": 1, + "_index": 2, + "_type": 3, + "asctime": 4, + "componentname": 5, + "customname": 6, + "highlight": 7, + "hostname": 12, + "levelname": 8, + "message": 9, + "sort": 10, + "varmessage": 11 + }, + "renameByName": {} + } + } + ], + "type": "table" + } + ], + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "Incoming request" + ], + "value": [ + "Incoming request" + ] + }, + "hide": 0, + "includeAll": true, + "label": "Security message", + "multi": true, + "name": "sec_message", + "options": [ + { + "selected": false, + "text": "All", + "value": "$__all" + }, + { + "selected": true, + "text": "Incoming request", + "value": "Incoming request" + } + ], + "query": "Incoming request", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Dirac Logs SECURITY", + "uid": "SXvsbSBVn", + "version": 6, + "weekStart": "" +} diff --git a/dashboards/tornadoLogs/kibana/Dirac_Logs_Stats.ndjson b/dashboards/tornadoLogs/kibana/Dirac_Logs_Stats.ndjson new file mode 100644 index 00000000000..7a0055539fb --- /dev/null +++ b/dashboards/tornadoLogs/kibana/Dirac_Logs_Stats.ndjson @@ -0,0 +1,13 @@ +{"attributes":{"fields":"[{\"count\":0,\"name\":\"@timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_score\",\"type\":\"number\",\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"name\":\"asctime\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"asctime.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"asctime\"}}},{\"count\":0,\"name\":\"componentname\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"componentname.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"componentname\"}}},{\"count\":0,\"name\":\"customname\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"customname.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"customname\"}}},{\"count\":0,\"name\":\"hostname\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"hostname.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"hostname\"}}},{\"count\":0,\"name\":\"levelname\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"levelname.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"levelname\"}}},{\"count\":0,\"name\":\"message\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"message.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"message\"}}},{\"count\":0,\"name\":\"tornadoComponent\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"tornadoComponent.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"tornadoComponent\"}}},{\"count\":0,\"name\":\"varmessage\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"varmessage.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"varmessage\"}}}]","timeFieldName":"@timestamp","title":"dirac-test-log-*"},"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","migrationVersion":{"index-pattern":"7.6.0"},"references":[],"type":"index-pattern","updated_at":"2023-04-25T09:04:35.970Z","version":"WzI4OSwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Messages fixed part per LogLevel","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Messages fixed part per LogLevel\",\"type\":\"histogram\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Number of messages\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Number of messages\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Number of messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"levelname.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"LogLevel\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"message.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}]}"},"id":"f4e760b0-41ae-11e9-92db-b12061b7ddd0","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-04-18T09:59:22.902Z","version":"WzIzOCwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Messages per LogLevel","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Messages per LogLevel\",\"type\":\"area\",\"params\":{\"type\":\"area\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Messages\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"area\",\"mode\":\"stacked\",\"data\":{\"label\":\"Number of messages per log level\",\"id\":\"1\"},\"drawLinesBetweenPoints\":true,\"showCircles\":true,\"interpolate\":\"linear\",\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Number of messages per log level\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"levelname.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}]}"},"id":"b18a4830-41b0-11e9-92db-b12061b7ddd0","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-04-18T09:59:22.902Z","version":"WzI0MywxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"query\":\"levelname.keyword:\\\"ERROR\\\"\",\"language\":\"lucene\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Number of error messages","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Number of error messages\",\"type\":\"area\",\"params\":{\"type\":\"area\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Number of error messages\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"area\",\"mode\":\"stacked\",\"data\":{\"label\":\"Number of error messages\",\"id\":\"1\"},\"drawLinesBetweenPoints\":true,\"showCircles\":true,\"interpolate\":\"linear\",\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Number of error messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"message.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}]}"},"id":"1a1f8180-41b1-11e9-97b4-bda41e0e0b3d","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-04-18T09:59:22.902Z","version":"WzIzNywxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Number of messages per components","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Number of messages per components\",\"type\":\"area\",\"params\":{\"type\":\"area\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"area\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"drawLinesBetweenPoints\":true,\"showCircles\":true,\"interpolate\":\"linear\",\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"componentname.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}]}"},"id":"1e7a01d0-466e-11e9-92db-b12061b7ddd0","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-04-18T09:59:22.902Z","version":"WzI0MiwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[{\"meta\":{\"negate\":false,\"disabled\":false,\"alias\":null,\"type\":\"phrase\",\"key\":\"message.keyword\",\"value\":\"Executing action\",\"params\":{\"query\":\"Executing action\",\"type\":\"phrase\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index\"},\"query\":{\"match\":{\"message.keyword\":{\"query\":\"Executing action\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}}],\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Service queries per host","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Service queries per host\",\"type\":\"histogram\",\"params\":{\"type\":\"histogram\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Count\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"histogram\",\"mode\":\"stacked\",\"data\":{\"label\":\"Count\",\"id\":\"1\"},\"valueAxis\":\"ValueAxis-1\",\"drawLinesBetweenPoints\":true,\"showCircles\":true}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"hostname.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":15,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"componentname.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}]}"},"id":"7d982120-4a28-11e9-92db-b12061b7ddd0","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"},{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"kibanaSavedObjectMeta.searchSourceJSON.filter[0].meta.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-04-18T09:59:22.902Z","version":"WzI0NSwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Total messages per LogLevel","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Total messages per LogLevel\",\"type\":\"enhanced-table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"computedColumns\":[],\"computedColsPerSplitCol\":false,\"hideExportLinks\":false,\"showFilterBar\":false,\"filterCaseSensitive\":false,\"filterBarHideable\":false,\"filterAsYouType\":false,\"filterBarWidth\":\"25%\",\"showMetricsAtAllLevels\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"levelname.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":8,\"order\":\"desc\",\"orderBy\":\"1\"}}]}"},"id":"cc1e39e0-7e5e-11ea-8f8c-c10aac8009dd","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-04-18T09:59:22.902Z","version":"WzIzOSwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Table Logs per component","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Table Logs per component\",\"type\":\"enhanced-table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"computedColumns\":[],\"computedColsPerSplitCol\":false,\"hideExportLinks\":false,\"showFilterBar\":true,\"filterCaseSensitive\":false,\"filterBarHideable\":false,\"filterAsYouType\":false,\"filterBarWidth\":\"25%\",\"showMetricsAtAllLevels\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"componentname.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\"}}]}"},"id":"689bdf20-7e5f-11ea-bef2-1d663f7888cc","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-04-18T09:59:22.902Z","version":"WzI0MCwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"HeatMapLogsComponents","uiStateJSON":"{\"vis\":{\"defaultColors\":{\"0 - 75,000\":\"rgb(247,252,245)\",\"75,000 - 150,000\":\"rgb(199,233,192)\",\"150,000 - 225,000\":\"rgb(116,196,118)\",\"225,000 - 300,000\":\"rgb(35,139,69)\"},\"legendOpen\":true}}","version":1,"visState":"{\"title\":\"HeatMapLogsComponents\",\"type\":\"heatmap\",\"params\":{\"addLegend\":true,\"addTooltip\":true,\"colorSchema\":\"Greens\",\"colorsNumber\":4,\"colorsRange\":[],\"enableHover\":false,\"invertColors\":false,\"legendPosition\":\"right\",\"percentageMode\":false,\"setColorRange\":false,\"times\":[],\"type\":\"heatmap\",\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"color\":\"#555\",\"rotate\":0,\"show\":false,\"overwriteColor\":false},\"scale\":{\"defaultYExtents\":false,\"type\":\"linear\"},\"show\":false,\"type\":\"value\"}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"levelname.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":6,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"componentname.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\"}}]}"},"id":"ddc58110-7e60-11ea-bef2-1d663f7888cc","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-04-18T09:59:22.902Z","version":"WzI0MSwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"query\":\"\",\"language\":\"lucene\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"AdvancedTableLogsPerComponentAndHosts","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"AdvancedTableLogsPerComponentAndHosts\",\"type\":\"enhanced-table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"computedColumns\":[],\"computedColsPerSplitCol\":false,\"hideExportLinks\":false,\"showFilterBar\":true,\"filterCaseSensitive\":false,\"filterBarHideable\":false,\"filterAsYouType\":false,\"filterBarWidth\":\"25%\",\"showMetricsAtAllLevels\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"componentname.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Component\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"levelname.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Level\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"message.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Message\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"hostname.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}]}"},"id":"43ec8410-7e62-11ea-8f8c-c10aac8009dd","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-04-18T09:59:22.902Z","version":"WzI0NCwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"filter\":[],\"query\":{\"query\":\"levelname.keyword:\\\"WARN\\\"\",\"language\":\"lucene\"},\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Number of warning messages","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"Number of warning messages\",\"type\":\"area\",\"params\":{\"type\":\"area\",\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"}},\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"type\":\"category\",\"position\":\"bottom\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\"},\"labels\":{\"show\":true,\"truncate\":100},\"title\":{}}],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"name\":\"LeftAxis-1\",\"type\":\"value\",\"position\":\"left\",\"show\":true,\"style\":{},\"scale\":{\"type\":\"linear\",\"mode\":\"normal\"},\"labels\":{\"show\":true,\"rotate\":0,\"filter\":false,\"truncate\":100},\"title\":{\"text\":\"Number of error messages\"}}],\"seriesParams\":[{\"show\":\"true\",\"type\":\"area\",\"mode\":\"stacked\",\"data\":{\"label\":\"Number of error messages\",\"id\":\"1\"},\"drawLinesBetweenPoints\":true,\"showCircles\":true,\"interpolate\":\"linear\",\"valueAxis\":\"ValueAxis-1\"}],\"addTooltip\":true,\"addLegend\":true,\"legendPosition\":\"right\",\"times\":[],\"addTimeMarker\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Number of error messages\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"message.keyword\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}}]}"},"id":"d2f2ce90-8613-11ea-bef2-1d663f7888cc","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"},{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-04-18T09:59:22.902Z","version":"WzIzNiwxXQ=="} +{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"lucene\",\"query\":\"\"},\"highlightAll\":true,\"version\":true,\"filter\":[]}"},"optionsJSON":"{\"darkTheme\":false,\"hidePanelTitles\":false,\"useMargins\":true}","panelsJSON":"[{\"panelIndex\":\"3\",\"gridData\":{\"w\":24,\"h\":12,\"x\":0,\"y\":12,\"i\":\"3\"},\"version\":\"7.3.0\",\"panelRefName\":\"panel_0\",\"embeddableConfig\":{}},{\"panelIndex\":\"4\",\"gridData\":{\"w\":24,\"h\":12,\"x\":24,\"y\":0,\"i\":\"4\"},\"version\":\"7.3.0\",\"panelRefName\":\"panel_1\",\"embeddableConfig\":{}},{\"panelIndex\":\"5\",\"gridData\":{\"w\":24,\"h\":12,\"x\":0,\"y\":24,\"i\":\"5\"},\"version\":\"7.3.0\",\"panelRefName\":\"panel_2\",\"embeddableConfig\":{}},{\"panelIndex\":\"7\",\"gridData\":{\"w\":24,\"h\":12,\"x\":24,\"y\":36,\"i\":\"7\"},\"version\":\"7.3.0\",\"panelRefName\":\"panel_3\",\"embeddableConfig\":{}},{\"panelIndex\":\"8\",\"gridData\":{\"w\":24,\"h\":12,\"x\":24,\"y\":12,\"i\":\"8\"},\"embeddableConfig\":{\"vis\":{\"legendOpen\":true}},\"version\":\"7.3.0\",\"panelRefName\":\"panel_4\"},{\"panelIndex\":\"11\",\"gridData\":{\"w\":24,\"h\":12,\"x\":0,\"y\":0,\"i\":\"11\"},\"version\":\"7.3.0\",\"panelRefName\":\"panel_5\",\"embeddableConfig\":{}},{\"panelIndex\":\"12\",\"gridData\":{\"w\":24,\"h\":12,\"x\":0,\"y\":36,\"i\":\"12\"},\"version\":\"7.3.0\",\"panelRefName\":\"panel_6\",\"embeddableConfig\":{}},{\"panelIndex\":\"13\",\"gridData\":{\"w\":48,\"h\":20,\"x\":0,\"y\":64,\"i\":\"13\"},\"version\":\"7.3.0\",\"panelRefName\":\"panel_7\",\"embeddableConfig\":{}},{\"panelIndex\":\"14\",\"gridData\":{\"w\":48,\"h\":16,\"x\":0,\"y\":48,\"i\":\"14\"},\"version\":\"7.3.0\",\"panelRefName\":\"panel_8\",\"embeddableConfig\":{}},{\"panelIndex\":\"15\",\"gridData\":{\"w\":24,\"h\":12,\"x\":24,\"y\":24,\"i\":\"15\"},\"version\":\"7.3.0\",\"panelRefName\":\"panel_9\",\"embeddableConfig\":{}}]","timeRestore":false,"title":"DIRAC Logs dashboard","version":1},"id":"3b53b830-41b1-11e9-97b4-bda41e0e0b3d","migrationVersion":{"dashboard":"7.9.3"},"references":[{"id":"f4e760b0-41ae-11e9-92db-b12061b7ddd0","name":"panel_0","type":"visualization"},{"id":"b18a4830-41b0-11e9-92db-b12061b7ddd0","name":"panel_1","type":"visualization"},{"id":"1a1f8180-41b1-11e9-97b4-bda41e0e0b3d","name":"panel_2","type":"visualization"},{"id":"1e7a01d0-466e-11e9-92db-b12061b7ddd0","name":"panel_3","type":"visualization"},{"id":"7d982120-4a28-11e9-92db-b12061b7ddd0","name":"panel_4","type":"visualization"},{"id":"cc1e39e0-7e5e-11ea-8f8c-c10aac8009dd","name":"panel_5","type":"visualization"},{"id":"689bdf20-7e5f-11ea-bef2-1d663f7888cc","name":"panel_6","type":"visualization"},{"id":"ddc58110-7e60-11ea-bef2-1d663f7888cc","name":"panel_7","type":"visualization"},{"id":"43ec8410-7e62-11ea-8f8c-c10aac8009dd","name":"panel_8","type":"visualization"},{"id":"d2f2ce90-8613-11ea-bef2-1d663f7888cc","name":"panel_9","type":"visualization"}],"type":"dashboard","updated_at":"2023-04-18T09:59:23.532Z","version":"WzI0NiwxXQ=="} +{"exportedCount":12,"missingRefCount":0,"missingReferences":[]} diff --git a/dashboards/tornadoLogs/kibana/Dirac_logs_Viewer_ALL.ndjson b/dashboards/tornadoLogs/kibana/Dirac_logs_Viewer_ALL.ndjson new file mode 100644 index 00000000000..28c147d451d --- /dev/null +++ b/dashboards/tornadoLogs/kibana/Dirac_logs_Viewer_ALL.ndjson @@ -0,0 +1,9 @@ +{"attributes":{"fields":"[{\"count\":0,\"name\":\"@timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_score\",\"type\":\"number\",\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"name\":\"asctime\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"asctime.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"asctime\"}}},{\"count\":0,\"name\":\"componentname\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"componentname.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"componentname\"}}},{\"count\":0,\"name\":\"customname\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"customname.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"customname\"}}},{\"count\":0,\"name\":\"exc_info\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"exc_info.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"exc_info\"}}},{\"count\":0,\"name\":\"hostname\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"hostname.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"hostname\"}}},{\"count\":0,\"name\":\"levelname\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"levelname.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"levelname\"}}},{\"count\":0,\"name\":\"message\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"message.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"message\"}}},{\"count\":0,\"name\":\"raw\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"raw.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"raw\"}}},{\"count\":0,\"name\":\"tornadoComponent\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"tornadoComponent.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"tornadoComponent\"}}},{\"count\":0,\"name\":\"varmessage\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"varmessage.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"varmessage\"}}}]","timeFieldName":"@timestamp","title":"dirac-test-log-*"},"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","migrationVersion":{"index-pattern":"7.6.0"},"references":[],"type":"index-pattern","updated_at":"2023-05-04T12:33:46.172Z","version":"WzMzNSwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"title":"ParentComponentSelector","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"ParentComponentSelector\",\"type\":\"input_control_vis\",\"aggs\":[],\"params\":{\"controls\":[{\"id\":\"1681725070886\",\"fieldName\":\"componentname.keyword\",\"parent\":\"\",\"label\":\"Parent Component\",\"type\":\"list\",\"options\":{\"type\":\"terms\",\"multiselect\":true,\"dynamicOptions\":true,\"size\":5,\"order\":\"desc\"},\"indexPatternRefName\":\"control_0_index_pattern\"}],\"updateFiltersOnChange\":true,\"useTimeFilter\":true,\"pinFilters\":true}}"},"id":"8230e1d0-dd05-11ed-8455-0f18f0c32abd","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"control_0_index_pattern","type":"index-pattern"}],"type":"visualization","updated_at":"2023-04-25T09:13:34.599Z","version":"WzI5NiwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}"},"title":"TornadoComponentSelector","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"TornadoComponentSelector\",\"type\":\"input_control_vis\",\"aggs\":[],\"params\":{\"controls\":[{\"fieldName\":\"tornadoComponent.keyword\",\"id\":\"1680512232853\",\"label\":\"TornadoComponent\",\"options\":{\"dynamicOptions\":true,\"multiselect\":true,\"order\":\"desc\",\"size\":5,\"type\":\"terms\"},\"parent\":\"\",\"type\":\"list\",\"indexPatternRefName\":\"control_0_index_pattern\"}],\"pinFilters\":true,\"updateFiltersOnChange\":true,\"useTimeFilter\":true}}"},"id":"390bf740-eb2b-11ed-8455-0f18f0c32abd","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"control_0_index_pattern","type":"index-pattern"}],"type":"visualization","updated_at":"2023-05-05T11:26:13.811Z","version":"WzM0OCwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}"},"title":"LevelSelector","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"LevelSelector\",\"type\":\"input_control_vis\",\"aggs\":[],\"params\":{\"controls\":[{\"fieldName\":\"levelname.keyword\",\"id\":\"1681381529167\",\"label\":\"Level\",\"options\":{\"dynamicOptions\":true,\"multiselect\":true,\"order\":\"desc\",\"size\":5,\"type\":\"terms\"},\"parent\":\"\",\"type\":\"list\",\"indexPatternRefName\":\"control_0_index_pattern\"}],\"pinFilters\":true,\"updateFiltersOnChange\":true,\"useTimeFilter\":true}}"},"id":"bd9af260-d9e5-11ed-8455-0f18f0c32abd","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"control_0_index_pattern","type":"index-pattern"}],"type":"visualization","updated_at":"2023-04-25T09:14:33.263Z","version":"WzI5NywxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\n \"query\": {\n \"query\": \"\",\n \"language\": \"kuery\"\n },\n \"filter\": [],\n \"indexRefName\": \"kibanaSavedObjectMeta.searchSourceJSON.index\"\n}"},"title":"Logs-count","uiStateJSON":"{}","version":1,"visState":"{\n \"title\": \"Logs-count\",\n \"type\": \"metric\",\n \"aggs\": [\n {\n \"id\": \"1\",\n \"enabled\": true,\n \"type\": \"count\",\n \"params\": {\n \"customLabel\": \"Records\"\n },\n \"schema\": \"metric\"\n }\n ],\n \"params\": {\n \"addTooltip\": true,\n \"addLegend\": false,\n \"type\": \"metric\",\n \"metric\": {\n \"percentageMode\": false,\n \"useRanges\": false,\n \"colorSchema\": \"Green to Red\",\n \"metricColorMode\": \"None\",\n \"colorsRange\": [\n {\n \"from\": 0,\n \"to\": 10000\n }\n ],\n \"labels\": {\n \"show\": true\n },\n \"invertColors\": false,\n \"style\": {\n \"bgFill\": \"#000\",\n \"bgColor\": false,\n \"labelColor\": false,\n \"subText\": \"\",\n \"fontSize\": 20\n }\n }\n }\n}"},"id":"788c0830-da08-11ed-8455-0f18f0c32abd","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-04-18T10:01:53.348Z","version":"WzI0OCwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}"},"title":"ComponentSelector","uiStateJSON":"{}","version":1,"visState":"{\"aggs\":[],\"params\":{\"controls\":[{\"fieldName\":\"customname.keyword\",\"id\":\"1680512232853\",\"label\":\"Component\",\"options\":{\"dynamicOptions\":true,\"multiselect\":true,\"order\":\"desc\",\"size\":5,\"type\":\"terms\"},\"parent\":\"\",\"type\":\"list\",\"indexPatternRefName\":\"control_0_index_pattern\"}],\"pinFilters\":true,\"updateFiltersOnChange\":true,\"useTimeFilter\":true},\"title\":\"ComponentSelector\",\"type\":\"input_control_vis\"}"},"id":"9d69ef90-d1fd-11ed-8f02-6185831a5745","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"control_0_index_pattern","type":"index-pattern"}],"type":"visualization","updated_at":"2023-04-25T09:05:50.228Z","version":"WzI5MCwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Logs","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Logs\",\"type\":\"document_table\",\"aggs\":[],\"params\":{\"addRowNumberColumn\":false,\"computedColsPerSplitCol\":false,\"computedColumns\":[],\"csvEncoding\":\"utf-8\",\"csvExportWithTotal\":false,\"csvFullExport\":false,\"fieldColumns\":[{\"enabled\":true,\"field\":{\"aggregatable\":true,\"count\":0,\"esTypes\":[\"date\"],\"name\":\"@timestamp\",\"readFromDocValues\":true,\"scripted\":false,\"searchable\":true,\"type\":\"date\"},\"label\":\"Time\"},{\"enabled\":true,\"field\":{\"aggregatable\":false,\"count\":0,\"esTypes\":[\"text\"],\"name\":\"componentname\",\"readFromDocValues\":false,\"scripted\":false,\"searchable\":true,\"type\":\"string\"},\"label\":\"ParentComponent\"},{\"enabled\":true,\"field\":{\"aggregatable\":false,\"count\":0,\"esTypes\":[\"text\"],\"name\":\"tornadoComponent\",\"readFromDocValues\":false,\"scripted\":false,\"searchable\":true,\"type\":\"string\"},\"label\":\"TornadoComponent\"},{\"enabled\":true,\"field\":{\"aggregatable\":false,\"count\":0,\"esTypes\":[\"text\"],\"name\":\"customname\",\"readFromDocValues\":false,\"scripted\":false,\"searchable\":true,\"type\":\"string\"},\"label\":\"Component\"},{\"enabled\":true,\"field\":{\"aggregatable\":true,\"count\":0,\"esTypes\":[\"keyword\"],\"name\":\"levelname.keyword\",\"readFromDocValues\":true,\"scripted\":false,\"searchable\":true,\"subType\":{\"multi\":{\"parent\":\"levelname\"}},\"type\":\"string\"},\"label\":\"Level\"},{\"enabled\":true,\"field\":{\"aggregatable\":false,\"count\":0,\"esTypes\":[\"text\"],\"name\":\"message\",\"readFromDocValues\":false,\"scripted\":false,\"searchable\":true,\"type\":\"string\"},\"label\":\"Message\"},{\"enabled\":true,\"field\":{\"aggregatable\":false,\"count\":0,\"esTypes\":[\"text\"],\"name\":\"varmessage\",\"readFromDocValues\":false,\"scripted\":false,\"searchable\":true,\"type\":\"string\"},\"label\":\"VarMessage\"},{\"enabled\":true,\"field\":{\"aggregatable\":false,\"count\":0,\"esTypes\":[\"text\"],\"name\":\"exc_info\",\"readFromDocValues\":false,\"scripted\":false,\"searchable\":true,\"type\":\"string\"},\"label\":\"Exc_info\"},{\"enabled\":true,\"field\":{\"aggregatable\":true,\"count\":0,\"esTypes\":[\"keyword\"],\"name\":\"hostname.keyword\",\"readFromDocValues\":true,\"scripted\":false,\"searchable\":true,\"subType\":{\"multi\":{\"parent\":\"hostname\"}},\"type\":\"string\"},\"label\":\"Hostname\"},{\"enabled\":false,\"field\":{\"aggregatable\":true,\"count\":0,\"esTypes\":[\"keyword\"],\"name\":\"asctime.keyword\",\"readFromDocValues\":true,\"scripted\":false,\"searchable\":true,\"subType\":{\"multi\":{\"parent\":\"asctime\"}},\"type\":\"string\"},\"label\":\"AscTime\"}],\"filterAsYouType\":true,\"filterBarHideable\":true,\"filterBarWidth\":\"25%\",\"filterCaseSensitive\":false,\"filterHighlightResults\":true,\"filterTermsSeparately\":true,\"hideExportLinks\":false,\"hitsSize\":1000,\"perPage\":20,\"showFilterBar\":true,\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"sortField\":{\"aggregatable\":true,\"count\":0,\"esTypes\":[\"date\"],\"name\":\"@timestamp\",\"readFromDocValues\":true,\"scripted\":false,\"searchable\":true,\"type\":\"date\"},\"sortOrder\":\"desc\",\"sortSplitCols\":false,\"stripedRows\":true,\"totalFunc\":\"count\"}}"},"id":"94638c40-d9ea-11ed-8455-0f18f0c32abd","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-05-05T09:56:08.450Z","version":"WzM0NCwxXQ=="} +{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"hidePanelTitles\":false,\"useMargins\":false}","panelsJSON":"[{\"version\":\"7.10.2\",\"gridData\":{\"x\":0,\"y\":0,\"w\":11,\"h\":4,\"i\":\"51e25a94-b1a2-4d77-89de-58c196fb7254\"},\"panelIndex\":\"51e25a94-b1a2-4d77-89de-58c196fb7254\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_0\"},{\"version\":\"7.10.2\",\"gridData\":{\"x\":11,\"y\":0,\"w\":9,\"h\":4,\"i\":\"62bcab2d-ca76-4dda-88b2-67e67d9c7db0\"},\"panelIndex\":\"62bcab2d-ca76-4dda-88b2-67e67d9c7db0\",\"embeddableConfig\":{\"title\":\"Component Selector\",\"hidePanelTitles\":true},\"title\":\"Component Selector\",\"panelRefName\":\"panel_1\"},{\"version\":\"7.10.2\",\"gridData\":{\"x\":30,\"y\":0,\"w\":9,\"h\":4,\"i\":\"cda68935-326d-438a-ba00-aa7d0696c7c9\"},\"panelIndex\":\"cda68935-326d-438a-ba00-aa7d0696c7c9\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_2\"},{\"version\":\"7.10.2\",\"gridData\":{\"x\":39,\"y\":0,\"w\":4,\"h\":4,\"i\":\"5a3e234a-c1bd-4155-b891-0b01b35fe972\"},\"panelIndex\":\"5a3e234a-c1bd-4155-b891-0b01b35fe972\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_3\"},{\"version\":\"7.10.2\",\"gridData\":{\"x\":20,\"y\":0,\"w\":10,\"h\":4,\"i\":\"f7f5ec2f-888a-46f0-9e41-c8a87defe28b\"},\"panelIndex\":\"f7f5ec2f-888a-46f0-9e41-c8a87defe28b\",\"embeddableConfig\":{\"title\":\"Component Selector\",\"hidePanelTitles\":true},\"title\":\"Component Selector\",\"panelRefName\":\"panel_4\"},{\"version\":\"7.10.2\",\"gridData\":{\"x\":0,\"y\":4,\"w\":48,\"h\":37,\"i\":\"e419c9d8-6b23-4683-94ff-32d6dac46f82\"},\"panelIndex\":\"e419c9d8-6b23-4683-94ff-32d6dac46f82\",\"embeddableConfig\":{\"hidePanelTitles\":true,\"table\":null,\"vis\":{\"params\":{\"sort\":{\"columnIndex\":0,\"direction\":null}}}},\"panelRefName\":\"panel_5\"}]","refreshInterval":{"pause":true,"value":0},"timeFrom":"now-15m","timeRestore":true,"timeTo":"now","title":"DIRAC Logs Viewer ALL","version":1},"id":"b3664af0-d1fd-11ed-8f02-6185831a5745","migrationVersion":{"dashboard":"7.9.3"},"references":[{"id":"8230e1d0-dd05-11ed-8455-0f18f0c32abd","name":"panel_0","type":"visualization"},{"id":"390bf740-eb2b-11ed-8455-0f18f0c32abd","name":"panel_1","type":"visualization"},{"id":"bd9af260-d9e5-11ed-8455-0f18f0c32abd","name":"panel_2","type":"visualization"},{"id":"788c0830-da08-11ed-8455-0f18f0c32abd","name":"panel_3","type":"visualization"},{"id":"9d69ef90-d1fd-11ed-8f02-6185831a5745","name":"panel_4","type":"visualization"},{"id":"94638c40-d9ea-11ed-8455-0f18f0c32abd","name":"panel_5","type":"visualization"}],"type":"dashboard","updated_at":"2023-05-05T11:33:21.568Z","version":"WzM1MiwxXQ=="} +{"exportedCount":8,"missingRefCount":0,"missingReferences":[]} diff --git a/dashboards/tornadoLogs/kibana/Dirac_logs_Viewer_ALL_Singleline.ndjson b/dashboards/tornadoLogs/kibana/Dirac_logs_Viewer_ALL_Singleline.ndjson new file mode 100644 index 00000000000..31f2f8788d6 --- /dev/null +++ b/dashboards/tornadoLogs/kibana/Dirac_logs_Viewer_ALL_Singleline.ndjson @@ -0,0 +1,9 @@ +{"attributes":{"fields":"[{\"count\":0,\"name\":\"@timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_score\",\"type\":\"number\",\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"name\":\"asctime\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"asctime.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"asctime\"}}},{\"count\":0,\"name\":\"componentname\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"componentname.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"componentname\"}}},{\"count\":0,\"name\":\"customname\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"customname.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"customname\"}}},{\"count\":0,\"name\":\"exc_info\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"exc_info.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"exc_info\"}}},{\"count\":0,\"name\":\"hostname\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"hostname.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"hostname\"}}},{\"count\":0,\"name\":\"levelname\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"levelname.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"levelname\"}}},{\"count\":0,\"name\":\"message\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"message.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"message\"}}},{\"count\":0,\"name\":\"raw\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"raw.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"raw\"}}},{\"count\":0,\"name\":\"tornadoComponent\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"tornadoComponent.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"tornadoComponent\"}}},{\"count\":0,\"name\":\"varmessage\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"varmessage.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"varmessage\"}}}]","timeFieldName":"@timestamp","title":"dirac-test-log-*"},"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","migrationVersion":{"index-pattern":"7.6.0"},"references":[],"type":"index-pattern","updated_at":"2023-05-04T12:33:46.172Z","version":"WzMzNSwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"title":"ParentComponentSelector","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"ParentComponentSelector\",\"type\":\"input_control_vis\",\"aggs\":[],\"params\":{\"controls\":[{\"id\":\"1681725070886\",\"fieldName\":\"componentname.keyword\",\"parent\":\"\",\"label\":\"Parent Component\",\"type\":\"list\",\"options\":{\"type\":\"terms\",\"multiselect\":true,\"dynamicOptions\":true,\"size\":5,\"order\":\"desc\"},\"indexPatternRefName\":\"control_0_index_pattern\"}],\"updateFiltersOnChange\":true,\"useTimeFilter\":true,\"pinFilters\":true}}"},"id":"8230e1d0-dd05-11ed-8455-0f18f0c32abd","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"control_0_index_pattern","type":"index-pattern"}],"type":"visualization","updated_at":"2023-04-25T09:13:34.599Z","version":"WzI5NiwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}"},"title":"LevelSelector","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"LevelSelector\",\"type\":\"input_control_vis\",\"aggs\":[],\"params\":{\"controls\":[{\"fieldName\":\"levelname.keyword\",\"id\":\"1681381529167\",\"label\":\"Level\",\"options\":{\"dynamicOptions\":true,\"multiselect\":true,\"order\":\"desc\",\"size\":5,\"type\":\"terms\"},\"parent\":\"\",\"type\":\"list\",\"indexPatternRefName\":\"control_0_index_pattern\"}],\"pinFilters\":true,\"updateFiltersOnChange\":true,\"useTimeFilter\":true}}"},"id":"bd9af260-d9e5-11ed-8455-0f18f0c32abd","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"control_0_index_pattern","type":"index-pattern"}],"type":"visualization","updated_at":"2023-04-25T09:14:33.263Z","version":"WzI5NywxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\n \"query\": {\n \"query\": \"\",\n \"language\": \"kuery\"\n },\n \"filter\": [],\n \"indexRefName\": \"kibanaSavedObjectMeta.searchSourceJSON.index\"\n}"},"title":"Logs-count","uiStateJSON":"{}","version":1,"visState":"{\n \"title\": \"Logs-count\",\n \"type\": \"metric\",\n \"aggs\": [\n {\n \"id\": \"1\",\n \"enabled\": true,\n \"type\": \"count\",\n \"params\": {\n \"customLabel\": \"Records\"\n },\n \"schema\": \"metric\"\n }\n ],\n \"params\": {\n \"addTooltip\": true,\n \"addLegend\": false,\n \"type\": \"metric\",\n \"metric\": {\n \"percentageMode\": false,\n \"useRanges\": false,\n \"colorSchema\": \"Green to Red\",\n \"metricColorMode\": \"None\",\n \"colorsRange\": [\n {\n \"from\": 0,\n \"to\": 10000\n }\n ],\n \"labels\": {\n \"show\": true\n },\n \"invertColors\": false,\n \"style\": {\n \"bgFill\": \"#000\",\n \"bgColor\": false,\n \"labelColor\": false,\n \"subText\": \"\",\n \"fontSize\": 20\n }\n }\n }\n}"},"id":"788c0830-da08-11ed-8455-0f18f0c32abd","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-04-18T10:01:53.348Z","version":"WzI0OCwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Logs Singleline","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"aggs\":[],\"params\":{\"addRowNumberColumn\":false,\"computedColsPerSplitCol\":false,\"computedColumns\":[],\"csvEncoding\":\"utf-8\",\"csvExportWithTotal\":false,\"csvFullExport\":false,\"fieldColumns\":[{\"enabled\":false,\"field\":{\"aggregatable\":true,\"count\":0,\"esTypes\":[\"date\"],\"name\":\"@timestamp\",\"readFromDocValues\":true,\"scripted\":false,\"searchable\":true,\"type\":\"date\"},\"label\":\"Time\"},{\"enabled\":true,\"field\":{\"aggregatable\":false,\"count\":0,\"esTypes\":[\"text\"],\"name\":\"raw\",\"readFromDocValues\":false,\"scripted\":false,\"searchable\":true,\"type\":\"string\"},\"label\":\"Log\"}],\"filterAsYouType\":true,\"filterBarHideable\":true,\"filterBarWidth\":\"25%\",\"filterCaseSensitive\":false,\"filterHighlightResults\":true,\"filterTermsSeparately\":true,\"hideExportLinks\":false,\"hitsSize\":1000,\"perPage\":20,\"showFilterBar\":true,\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"sortField\":{\"aggregatable\":true,\"count\":0,\"esTypes\":[\"date\"],\"name\":\"@timestamp\",\"readFromDocValues\":true,\"scripted\":false,\"searchable\":true,\"type\":\"date\"},\"sortOrder\":\"desc\",\"sortSplitCols\":false,\"stripedRows\":true,\"totalFunc\":\"count\"},\"title\":\"Logs Singleline\",\"type\":\"document_table\"}"},"id":"d3141870-ea56-11ed-8455-0f18f0c32abd","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-05-04T13:51:54.395Z","version":"WzM0MSwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}"},"title":"ComponentSelector (copy)","uiStateJSON":"{}","version":1,"visState":"{\"aggs\":[],\"params\":{\"controls\":[{\"fieldName\":\"customname.keyword\",\"id\":\"1680512232853\",\"label\":\"Component\",\"options\":{\"dynamicOptions\":true,\"multiselect\":true,\"order\":\"desc\",\"size\":5,\"type\":\"terms\"},\"parent\":\"\",\"type\":\"list\",\"indexPatternRefName\":\"control_0_index_pattern\"}],\"pinFilters\":true,\"updateFiltersOnChange\":true,\"useTimeFilter\":true},\"title\":\"ComponentSelector\",\"type\":\"input_control_vis\"}"},"id":"0bae4660-eb38-11ed-8455-0f18f0c32abd","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"control_0_index_pattern","type":"index-pattern"}],"type":"visualization","updated_at":"2023-05-05T11:29:06.502Z","version":"WzM0OSwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}"},"title":"TornadoComponentSelector","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"TornadoComponentSelector\",\"type\":\"input_control_vis\",\"aggs\":[],\"params\":{\"controls\":[{\"fieldName\":\"tornadoComponent.keyword\",\"id\":\"1680512232853\",\"label\":\"TornadoComponent\",\"options\":{\"dynamicOptions\":true,\"multiselect\":true,\"order\":\"desc\",\"size\":5,\"type\":\"terms\"},\"parent\":\"\",\"type\":\"list\",\"indexPatternRefName\":\"control_0_index_pattern\"}],\"pinFilters\":true,\"updateFiltersOnChange\":true,\"useTimeFilter\":true}}"},"id":"390bf740-eb2b-11ed-8455-0f18f0c32abd","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"control_0_index_pattern","type":"index-pattern"}],"type":"visualization","updated_at":"2023-05-05T11:26:13.811Z","version":"WzM0OCwxXQ=="} +{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"hidePanelTitles\":false,\"useMargins\":false}","panelsJSON":"[{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":4,\"i\":\"51e25a94-b1a2-4d77-89de-58c196fb7254\",\"w\":10,\"x\":0,\"y\":0},\"panelIndex\":\"51e25a94-b1a2-4d77-89de-58c196fb7254\",\"version\":\"7.10.2\",\"panelRefName\":\"panel_0\"},{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":4,\"i\":\"cda68935-326d-438a-ba00-aa7d0696c7c9\",\"w\":9,\"x\":30,\"y\":0},\"panelIndex\":\"cda68935-326d-438a-ba00-aa7d0696c7c9\",\"version\":\"7.10.2\",\"panelRefName\":\"panel_1\"},{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":4,\"i\":\"5a3e234a-c1bd-4155-b891-0b01b35fe972\",\"w\":4,\"x\":43,\"y\":0},\"panelIndex\":\"5a3e234a-c1bd-4155-b891-0b01b35fe972\",\"version\":\"7.10.2\",\"panelRefName\":\"panel_2\"},{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":37,\"i\":\"30df3005-6a8d-49a2-bf19-1c0d6b9afb5e\",\"w\":48,\"x\":0,\"y\":4},\"panelIndex\":\"30df3005-6a8d-49a2-bf19-1c0d6b9afb5e\",\"version\":\"7.10.2\",\"panelRefName\":\"panel_3\"},{\"embeddableConfig\":{\"hidePanelTitles\":true,\"title\":\"Component Selector\"},\"gridData\":{\"h\":4,\"i\":\"3247d8e1-333c-4f42-a44e-ed9287cf4a61\",\"w\":10,\"x\":20,\"y\":0},\"panelIndex\":\"3247d8e1-333c-4f42-a44e-ed9287cf4a61\",\"title\":\"Component Selector\",\"version\":\"7.10.2\",\"panelRefName\":\"panel_4\"},{\"embeddableConfig\":{\"hidePanelTitles\":true},\"gridData\":{\"h\":4,\"i\":\"d84c552c-4785-4973-bf0e-6e246f319991\",\"w\":10,\"x\":10,\"y\":0},\"panelIndex\":\"d84c552c-4785-4973-bf0e-6e246f319991\",\"version\":\"7.10.2\",\"panelRefName\":\"panel_5\"}]","refreshInterval":{"pause":true,"value":0},"timeFrom":"now-15m","timeRestore":true,"timeTo":"now","title":"DIRAC Logs Viewer ALL Singleline","version":1},"id":"a22d7ad0-ea56-11ed-8455-0f18f0c32abd","migrationVersion":{"dashboard":"7.9.3"},"references":[{"id":"8230e1d0-dd05-11ed-8455-0f18f0c32abd","name":"panel_0","type":"visualization"},{"id":"bd9af260-d9e5-11ed-8455-0f18f0c32abd","name":"panel_1","type":"visualization"},{"id":"788c0830-da08-11ed-8455-0f18f0c32abd","name":"panel_2","type":"visualization"},{"id":"d3141870-ea56-11ed-8455-0f18f0c32abd","name":"panel_3","type":"visualization"},{"id":"0bae4660-eb38-11ed-8455-0f18f0c32abd","name":"panel_4","type":"visualization"},{"id":"390bf740-eb2b-11ed-8455-0f18f0c32abd","name":"panel_5","type":"visualization"}],"type":"dashboard","updated_at":"2023-05-05T11:32:51.688Z","version":"WzM1MSwxXQ=="} +{"exportedCount":8,"missingRefCount":0,"missingReferences":[]} diff --git a/dashboards/tornadoLogs/kibana/Dirac_logs_Viewer_DEV.ndjson b/dashboards/tornadoLogs/kibana/Dirac_logs_Viewer_DEV.ndjson new file mode 100644 index 00000000000..d3d85dbf95a --- /dev/null +++ b/dashboards/tornadoLogs/kibana/Dirac_logs_Viewer_DEV.ndjson @@ -0,0 +1,5 @@ +{"attributes":{"fields":"[{\"count\":0,\"name\":\"@timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_score\",\"type\":\"number\",\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"name\":\"asctime\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"asctime.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"asctime\"}}},{\"count\":0,\"name\":\"componentname\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"componentname.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"componentname\"}}},{\"count\":0,\"name\":\"customname\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"customname.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"customname\"}}},{\"count\":0,\"name\":\"exc_info\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"exc_info.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"exc_info\"}}},{\"count\":0,\"name\":\"hostname\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"hostname.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"hostname\"}}},{\"count\":0,\"name\":\"levelname\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"levelname.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"levelname\"}}},{\"count\":0,\"name\":\"message\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"message.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"message\"}}},{\"count\":0,\"name\":\"raw\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"raw.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"raw\"}}},{\"count\":0,\"name\":\"tornadoComponent\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"tornadoComponent.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"tornadoComponent\"}}},{\"count\":0,\"name\":\"varmessage\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"varmessage.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"varmessage\"}}}]","timeFieldName":"@timestamp","title":"dirac-test-log-*"},"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","migrationVersion":{"index-pattern":"7.6.0"},"references":[],"type":"index-pattern","updated_at":"2023-05-04T12:33:46.172Z","version":"WzMzNSwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"levelname:\\\"DEV\\\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Dev","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"title\":\"Dev\",\"type\":\"document_table\",\"aggs\":[],\"params\":{\"addRowNumberColumn\":false,\"computedColsPerSplitCol\":false,\"computedColumns\":[],\"csvEncoding\":\"utf-8\",\"csvExportWithTotal\":false,\"csvFullExport\":false,\"fieldColumns\":[{\"enabled\":true,\"field\":{\"aggregatable\":true,\"count\":0,\"esTypes\":[\"date\"],\"name\":\"@timestamp\",\"readFromDocValues\":true,\"scripted\":false,\"searchable\":true,\"type\":\"date\"},\"label\":\"Time\"},{\"enabled\":true,\"field\":{\"aggregatable\":true,\"count\":0,\"esTypes\":[\"keyword\"],\"name\":\"message.keyword\",\"readFromDocValues\":true,\"scripted\":false,\"searchable\":true,\"subType\":{\"multi\":{\"parent\":\"message\"}},\"type\":\"string\"},\"label\":\"Message\"},{\"enabled\":true,\"field\":{\"aggregatable\":true,\"count\":0,\"esTypes\":[\"keyword\"],\"name\":\"hostname.keyword\",\"readFromDocValues\":true,\"scripted\":false,\"searchable\":true,\"subType\":{\"multi\":{\"parent\":\"hostname\"}},\"type\":\"string\"},\"label\":\"Hostname\"}],\"filterAsYouType\":true,\"filterBarHideable\":true,\"filterBarWidth\":\"25%\",\"filterCaseSensitive\":false,\"filterHighlightResults\":true,\"filterTermsSeparately\":true,\"hideExportLinks\":false,\"hitsSize\":50000,\"perPage\":20,\"showFilterBar\":true,\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"sortField\":{\"aggregatable\":true,\"count\":0,\"esTypes\":[\"date\"],\"name\":\"@timestamp\",\"readFromDocValues\":true,\"scripted\":false,\"searchable\":true,\"type\":\"date\"},\"sortOrder\":\"desc\",\"sortSplitCols\":false,\"stripedRows\":true,\"totalFunc\":\"count\"}}"},"id":"d93b2550-e334-11ed-8455-0f18f0c32abd","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-04-25T06:49:14.241Z","version":"WzI3NCwxXQ=="} +{"attributes":{"columns":["_source"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[],"title":"Raw Logs","version":1},"id":"fabc5670-d481-11ed-8455-0f18f0c32abd","migrationVersion":{"search":"7.9.3"},"references":[{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2023-04-18T09:45:40.064Z","version":"WzIzMiwxXQ=="} +{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}"},"optionsJSON":"{\"hidePanelTitles\":false,\"useMargins\":false}","panelsJSON":"[{\"version\":\"7.10.2\",\"gridData\":{\"x\":0,\"y\":0,\"w\":48,\"h\":34,\"i\":\"271a92ea-1144-428a-b972-aa426731e8c1\"},\"panelIndex\":\"271a92ea-1144-428a-b972-aa426731e8c1\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_0\"},{\"version\":\"7.10.2\",\"gridData\":{\"x\":0,\"y\":34,\"w\":48,\"h\":31,\"i\":\"96f7059a-a473-4ad7-b90c-c06fb15ec396\"},\"panelIndex\":\"96f7059a-a473-4ad7-b90c-c06fb15ec396\",\"embeddableConfig\":{},\"panelRefName\":\"panel_1\"}]","refreshInterval":{"pause":true,"value":0},"timeFrom":"now-15m","timeRestore":true,"timeTo":"now","title":"DIRAC Logs Viewer DEV","version":1},"id":"24197700-e2b5-11ed-8455-0f18f0c32abd","migrationVersion":{"dashboard":"7.9.3"},"references":[{"id":"d93b2550-e334-11ed-8455-0f18f0c32abd","name":"panel_0","type":"visualization"},{"id":"fabc5670-d481-11ed-8455-0f18f0c32abd","name":"panel_1","type":"search"}],"type":"dashboard","updated_at":"2023-04-25T13:27:52.380Z","version":"WzMwNSwxXQ=="} +{"exportedCount":4,"missingRefCount":0,"missingReferences":[]} diff --git a/dashboards/tornadoLogs/kibana/Dirac_logs_Viewer_SECURITY.ndjson b/dashboards/tornadoLogs/kibana/Dirac_logs_Viewer_SECURITY.ndjson new file mode 100644 index 00000000000..163241631a8 --- /dev/null +++ b/dashboards/tornadoLogs/kibana/Dirac_logs_Viewer_SECURITY.ndjson @@ -0,0 +1,8 @@ +{"attributes":{"fields":"[{\"count\":0,\"name\":\"@timestamp\",\"type\":\"date\",\"esTypes\":[\"date\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true},{\"count\":0,\"name\":\"_id\",\"type\":\"string\",\"esTypes\":[\"_id\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_index\",\"type\":\"string\",\"esTypes\":[\"_index\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_score\",\"type\":\"number\",\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_source\",\"type\":\"_source\",\"esTypes\":[\"_source\"],\"scripted\":false,\"searchable\":false,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"_type\",\"type\":\"string\",\"esTypes\":[\"_type\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":false},{\"count\":0,\"name\":\"asctime\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"asctime.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"asctime\"}}},{\"count\":0,\"name\":\"componentname\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"componentname.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"componentname\"}}},{\"count\":0,\"name\":\"customname\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"customname.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"customname\"}}},{\"count\":0,\"name\":\"exc_info\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"exc_info.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"exc_info\"}}},{\"count\":0,\"name\":\"hostname\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"hostname.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"hostname\"}}},{\"count\":0,\"name\":\"levelname\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"levelname.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"levelname\"}}},{\"count\":0,\"name\":\"message\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"message.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"message\"}}},{\"count\":0,\"name\":\"raw\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"raw.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"raw\"}}},{\"count\":0,\"name\":\"tornadoComponent\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"tornadoComponent.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"tornadoComponent\"}}},{\"count\":0,\"name\":\"varmessage\",\"type\":\"string\",\"esTypes\":[\"text\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":false,\"readFromDocValues\":false},{\"count\":0,\"name\":\"varmessage.keyword\",\"type\":\"string\",\"esTypes\":[\"keyword\"],\"scripted\":false,\"searchable\":true,\"aggregatable\":true,\"readFromDocValues\":true,\"subType\":{\"multi\":{\"parent\":\"varmessage\"}}}]","timeFieldName":"@timestamp","title":"dirac-test-log-*"},"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","migrationVersion":{"index-pattern":"7.6.0"},"references":[],"type":"index-pattern","updated_at":"2023-05-04T12:33:46.172Z","version":"WzMzNSwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"title":"ParentComponentSelector","uiStateJSON":"{}","version":1,"visState":"{\"title\":\"ParentComponentSelector\",\"type\":\"input_control_vis\",\"aggs\":[],\"params\":{\"controls\":[{\"id\":\"1681725070886\",\"fieldName\":\"componentname.keyword\",\"parent\":\"\",\"label\":\"Parent Component\",\"type\":\"list\",\"options\":{\"type\":\"terms\",\"multiselect\":true,\"dynamicOptions\":true,\"size\":5,\"order\":\"desc\"},\"indexPatternRefName\":\"control_0_index_pattern\"}],\"updateFiltersOnChange\":true,\"useTimeFilter\":true,\"pinFilters\":true}}"},"id":"8230e1d0-dd05-11ed-8455-0f18f0c32abd","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"control_0_index_pattern","type":"index-pattern"}],"type":"visualization","updated_at":"2023-04-25T09:13:34.599Z","version":"WzI5NiwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"\"},\"filter\":[]}"},"title":"ComponentSelector","uiStateJSON":"{}","version":1,"visState":"{\"aggs\":[],\"params\":{\"controls\":[{\"fieldName\":\"customname.keyword\",\"id\":\"1680512232853\",\"label\":\"Component\",\"options\":{\"dynamicOptions\":true,\"multiselect\":true,\"order\":\"desc\",\"size\":5,\"type\":\"terms\"},\"parent\":\"\",\"type\":\"list\",\"indexPatternRefName\":\"control_0_index_pattern\"}],\"pinFilters\":true,\"updateFiltersOnChange\":true,\"useTimeFilter\":true},\"title\":\"ComponentSelector\",\"type\":\"input_control_vis\"}"},"id":"9d69ef90-d1fd-11ed-8f02-6185831a5745","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"control_0_index_pattern","type":"index-pattern"}],"type":"visualization","updated_at":"2023-04-25T09:05:50.228Z","version":"WzI5MCwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\n \"query\": {\n \"query\": \"\",\n \"language\": \"kuery\"\n },\n \"filter\": [],\n \"indexRefName\": \"kibanaSavedObjectMeta.searchSourceJSON.index\"\n}"},"title":"Logs-count","uiStateJSON":"{}","version":1,"visState":"{\n \"title\": \"Logs-count\",\n \"type\": \"metric\",\n \"aggs\": [\n {\n \"id\": \"1\",\n \"enabled\": true,\n \"type\": \"count\",\n \"params\": {\n \"customLabel\": \"Records\"\n },\n \"schema\": \"metric\"\n }\n ],\n \"params\": {\n \"addTooltip\": true,\n \"addLegend\": false,\n \"type\": \"metric\",\n \"metric\": {\n \"percentageMode\": false,\n \"useRanges\": false,\n \"colorSchema\": \"Green to Red\",\n \"metricColorMode\": \"None\",\n \"colorsRange\": [\n {\n \"from\": 0,\n \"to\": 10000\n }\n ],\n \"labels\": {\n \"show\": true\n },\n \"invertColors\": false,\n \"style\": {\n \"bgFill\": \"#000\",\n \"bgColor\": false,\n \"labelColor\": false,\n \"subText\": \"\",\n \"fontSize\": 20\n }\n }\n }\n}"},"id":"788c0830-da08-11ed-8455-0f18f0c32abd","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-04-18T10:01:53.348Z","version":"WzI0OCwxXQ=="} +{"attributes":{"description":"","kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"language\":\"kuery\",\"query\":\"message:\\\"Incoming request\\\"\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"title":"Securiy","uiStateJSON":"{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}","version":1,"visState":"{\"aggs\":[],\"params\":{\"addRowNumberColumn\":false,\"computedColsPerSplitCol\":false,\"computedColumns\":[],\"csvEncoding\":\"utf-8\",\"csvExportWithTotal\":false,\"csvFullExport\":false,\"fieldColumns\":[{\"enabled\":true,\"field\":{\"aggregatable\":true,\"count\":0,\"esTypes\":[\"date\"],\"name\":\"@timestamp\",\"readFromDocValues\":true,\"scripted\":false,\"searchable\":true,\"type\":\"date\"},\"label\":\"Time\"},{\"enabled\":true,\"field\":{\"aggregatable\":true,\"count\":0,\"esTypes\":[\"keyword\"],\"name\":\"componentname.keyword\",\"readFromDocValues\":true,\"scripted\":false,\"searchable\":true,\"subType\":{\"multi\":{\"parent\":\"componentname\"}},\"type\":\"string\"},\"label\":\"ParentComponent\"},{\"enabled\":true,\"field\":{\"aggregatable\":false,\"count\":0,\"esTypes\":[\"text\"],\"name\":\"customname\",\"readFromDocValues\":false,\"scripted\":false,\"searchable\":true,\"type\":\"string\"},\"label\":\"Component\"},{\"enabled\":true,\"field\":{\"aggregatable\":true,\"count\":0,\"esTypes\":[\"keyword\"],\"name\":\"levelname.keyword\",\"readFromDocValues\":true,\"scripted\":false,\"searchable\":true,\"subType\":{\"multi\":{\"parent\":\"levelname\"}},\"type\":\"string\"},\"label\":\"Level\"},{\"enabled\":true,\"field\":{\"aggregatable\":true,\"count\":0,\"esTypes\":[\"keyword\"],\"name\":\"message.keyword\",\"readFromDocValues\":true,\"scripted\":false,\"searchable\":true,\"subType\":{\"multi\":{\"parent\":\"message\"}},\"type\":\"string\"},\"label\":\"Message\"},{\"enabled\":true,\"field\":{\"aggregatable\":true,\"count\":0,\"esTypes\":[\"keyword\"],\"name\":\"varmessage.keyword\",\"readFromDocValues\":true,\"scripted\":false,\"searchable\":true,\"subType\":{\"multi\":{\"parent\":\"varmessage\"}},\"type\":\"string\"},\"label\":\"VarMessage\"},{\"enabled\":true,\"field\":{\"aggregatable\":true,\"count\":0,\"esTypes\":[\"keyword\"],\"name\":\"hostname.keyword\",\"readFromDocValues\":true,\"scripted\":false,\"searchable\":true,\"subType\":{\"multi\":{\"parent\":\"hostname\"}},\"type\":\"string\"},\"label\":\"Hostname\"}],\"filterAsYouType\":true,\"filterBarHideable\":true,\"filterBarWidth\":\"25%\",\"filterCaseSensitive\":false,\"filterHighlightResults\":true,\"filterTermsSeparately\":true,\"hideExportLinks\":false,\"hitsSize\":50000,\"perPage\":20,\"showFilterBar\":true,\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"sortField\":{\"aggregatable\":true,\"count\":0,\"esTypes\":[\"date\"],\"name\":\"@timestamp\",\"readFromDocValues\":true,\"scripted\":false,\"searchable\":true,\"type\":\"date\"},\"sortOrder\":\"desc\",\"sortSplitCols\":false,\"stripedRows\":true,\"totalFunc\":\"count\"},\"title\":\"Securiy\",\"type\":\"document_table\"}"},"id":"ff8a8720-e2b5-11ed-8455-0f18f0c32abd","migrationVersion":{"visualization":"7.10.0"},"references":[{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"visualization","updated_at":"2023-04-25T06:40:09.982Z","version":"WzI2NiwxXQ=="} +{"attributes":{"columns":["_source"],"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"highlightAll\":true,\"version\":true,\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[],\"indexRefName\":\"kibanaSavedObjectMeta.searchSourceJSON.index\"}"},"sort":[],"title":"Raw Logs","version":1},"id":"fabc5670-d481-11ed-8455-0f18f0c32abd","migrationVersion":{"search":"7.9.3"},"references":[{"id":"eee5ca90-ddcb-11ed-8455-0f18f0c32abd","name":"kibanaSavedObjectMeta.searchSourceJSON.index","type":"index-pattern"}],"type":"search","updated_at":"2023-04-18T09:45:40.064Z","version":"WzIzMiwxXQ=="} +{"attributes":{"description":"","hits":0,"kibanaSavedObjectMeta":{"searchSourceJSON":"{\"query\":{\"query\":\"\",\"language\":\"kuery\"},\"filter\":[]}"},"optionsJSON":"{\"hidePanelTitles\":false,\"useMargins\":false}","panelsJSON":"[{\"version\":\"7.10.2\",\"gridData\":{\"x\":0,\"y\":0,\"w\":12,\"h\":4,\"i\":\"51e25a94-b1a2-4d77-89de-58c196fb7254\"},\"panelIndex\":\"51e25a94-b1a2-4d77-89de-58c196fb7254\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_0\"},{\"version\":\"7.10.2\",\"gridData\":{\"x\":12,\"y\":0,\"w\":12,\"h\":4,\"i\":\"f7f5ec2f-888a-46f0-9e41-c8a87defe28b\"},\"panelIndex\":\"f7f5ec2f-888a-46f0-9e41-c8a87defe28b\",\"embeddableConfig\":{\"title\":\"Component Selector\",\"hidePanelTitles\":true},\"title\":\"Component Selector\",\"panelRefName\":\"panel_1\"},{\"version\":\"7.10.2\",\"gridData\":{\"x\":24,\"y\":0,\"w\":5,\"h\":4,\"i\":\"5a3e234a-c1bd-4155-b891-0b01b35fe972\"},\"panelIndex\":\"5a3e234a-c1bd-4155-b891-0b01b35fe972\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_2\"},{\"version\":\"7.10.2\",\"gridData\":{\"x\":0,\"y\":4,\"w\":48,\"h\":29,\"i\":\"70c22c34-7f6c-4979-8c58-466b9c02f330\"},\"panelIndex\":\"70c22c34-7f6c-4979-8c58-466b9c02f330\",\"embeddableConfig\":{\"hidePanelTitles\":true},\"panelRefName\":\"panel_3\"},{\"version\":\"7.10.2\",\"gridData\":{\"x\":0,\"y\":33,\"w\":48,\"h\":27,\"i\":\"103b1d61-ff1e-43b8-8057-88ee9fcaf2ae\"},\"panelIndex\":\"103b1d61-ff1e-43b8-8057-88ee9fcaf2ae\",\"embeddableConfig\":{},\"panelRefName\":\"panel_4\"}]","refreshInterval":{"pause":true,"value":0},"timeFrom":"now-1h","timeRestore":true,"timeTo":"now","title":"DIRAC Logs Viewer SECURITY","version":1},"id":"512f5d90-e2b5-11ed-8455-0f18f0c32abd","migrationVersion":{"dashboard":"7.9.3"},"references":[{"id":"8230e1d0-dd05-11ed-8455-0f18f0c32abd","name":"panel_0","type":"visualization"},{"id":"9d69ef90-d1fd-11ed-8f02-6185831a5745","name":"panel_1","type":"visualization"},{"id":"788c0830-da08-11ed-8455-0f18f0c32abd","name":"panel_2","type":"visualization"},{"id":"ff8a8720-e2b5-11ed-8455-0f18f0c32abd","name":"panel_3","type":"visualization"},{"id":"fabc5670-d481-11ed-8455-0f18f0c32abd","name":"panel_4","type":"search"}],"type":"dashboard","updated_at":"2023-04-25T13:28:43.400Z","version":"WzMwNiwxXQ=="} +{"exportedCount":7,"missingRefCount":0,"missingReferences":[]} diff --git a/dirac.cfg b/dirac.cfg index 46475fb12a3..e05b50bc6ae 100644 --- a/dirac.cfg +++ b/dirac.cfg @@ -5,10 +5,6 @@ DIRAC # The option is defined in a single VO installation. #VirtualOrganization = myVO - # The name of the DIRAC installation Setup. This option is defined in the client - # installations to define which subset of DIRAC Systems the client will work with. - Setup = mySetup - # The list of extensions to the Core DIRAC software used by the given installation #Extensions = WebApp @@ -101,31 +97,18 @@ DIRAC # grant_types = refresh_token, #} } - - } - - } - - # The subsection defines the names of different DIRAC Setups. - Setups - { - - # For each Setup known to the installation, there must be a subsection with the appropriate name. - # In each subsection of the Setup section the names of corresponding system instances are defined. - # In the example below "Production" instances of the Configuration - # and Framework systems are defined as part of the "Dirac-Production" setup. - Dirac-Production - { - # Each option represents a DIRAC System available in the Setup - # and the Value is the instance of System that is used in that setup. - # For instance, since the Configuration is unique for the whole installation, - # all setups should have the same instance for the Configuration systems. - Configuration = Production - Framework = Production } - } - +} +# This part contains anything related to DiracX +DiracX +{ + # The URL of the DIRAC Server + URL = https://diracx.invalid:8000 + # A key used to have priviledged interactions with diracx. see + LegacyExchangeApiKey = diracx:legacy:InsecureChangeMe + # List of VOs which should not use DiracX via the legacy compatibility mechanism + DisabledVOs = dteam,cta } ### Registry section: # Sections to register VOs, groups, users and hosts @@ -187,7 +170,7 @@ Registry VOMSName = lhcb # Registered identity provider associated with VO - IdP = CheckIn + IdProvider = CheckIn # Section to describe all the VOMS servers that can be used with the given VOMS VO VOMSServers @@ -262,8 +245,7 @@ Registry # Admin group lhcb_admin { - Properties = AlarmsManagement # Allow to set notifications and manage alarms - Properties += ServiceAdministrator # DIRAC Service Administrator + Properties = ServiceAdministrator # DIRAC Service Administrator Properties += CSAdministrator # possibility to edit the Configuration Service Properties += JobAdministrator # Job Administrator can manipulate everybody's jobs Properties += FullDelegation # Allow getting full delegated proxies @@ -381,6 +363,39 @@ Systems } } } + + Framework + { + Services + { + BundleDelivery + { + Protocol = https + Authorization + { + Default = authenticated + } + } + ComponentMonitoring + { + Port = 9190 + # This enables ES monitoring only for this particular service. + EnableActivityMonitoring = no + Authorization + { + Default = ServiceAdministrator + componentExists = authenticated + getComponents = authenticated + hostExists = authenticated + getHosts = authenticated + installationExists = authenticated + getInstallations = authenticated + updateLog = Operator + } + } + } + } + RequestManagementSystem { Agents @@ -452,6 +467,7 @@ Systems { Location = DIRAC/DataManagementSystem/Agent/RequestOperations/ReplicateAndRegister FTSMode = True # If True, will use FTS to transfer files + DMMode = True # If false, DataManager will not be used as a failover of FTS transfers FTSBannedGroups = lhcb_user # list of groups for which not to use FTS } SetFileStatus @@ -527,28 +543,24 @@ Systems ##END } } - Framework + WorkloadManagementSystem { - Services + Databases { - ComponentMonitoring + JobParametersDB { - Port = 9190 - # This enables ES monitoring only for this particular service. - EnableActivityMonitoring = no - Authorization - { - Default = ServiceAdministrator - componentExists = authenticated - getComponents = authenticated - hostExists = authenticated - getHosts = authenticated - installationExists = authenticated - getInstallations = authenticated - updateLog = Operator - } + # Host of OpenSearch instance + Host=host.some.where + + # index name (default is "job_parameters") + index_name=a_different_name } } + JobWrapper + { + # Minimum output buffer requested for running jobs + MinOutputDataBufferGB = 5 + } } } Resources @@ -596,19 +608,26 @@ Resources # # Values are overwritten by the most specialized option. + # Default local CE to use on all CEs (Pool, Singularity, InProcess, etc) + # There is no default value + DefaultLocalCEType = Singularity + # The options below can be valid for all computing element types CEDefaults { - # Default environment file sourced before calling rid commands, without extension '.sh' - GridEnv = /opt/dirac/gridenv - # Will be added to the pilot configuration as /LocalSite/SharedArea SharedArea = /cvmfs/lhcb.cern.ch/lib - # or adding some generic pilot options (only for pilots submitted by SiteDirectors) - # the example below will add the environment variable DIRACSYSCONFIG (see :ref:`bashrc_variables`) - ExtraPilotOptions = --userEnvVariables DIRACSYSCONFIG:::pilot.cfg + # For adding Extra environments (only for pilots submitted by SiteDirectors) + UserEnvVariables = DIRACSYSCONFIG:::pilot.cfg,RUCIO_HOME:::/home/dirac/rucio + + # for adding some extra pilot options (only for pilots submitted by SiteDirectors) + ExtraPilotOptions = --pilotLogging True + + # for adding some generic pilot options (only for pilots submitted by SiteDirectors) + # which will be transleted as "-o" options of the Pilot + GenericOptions = diracInstallOnly, someThing # for adding the --modules=value option to dirac-pilot Modules = @@ -628,10 +647,6 @@ Resources # Default: /cvmfs/cernvm-prod.cern.ch/cvm4 ContainerRoot = /cvmfs/cernvm-prod.cern.ch/cvm4 - # The binary to start the container - # default: singularity - ContainerBin = /opt/extras/bin/singularity - # List of directories to bind ContainerBind = /etc/grid-security,someDir:::BoundHere @@ -717,17 +732,6 @@ Resources } ## - ## PUSP type: - MY_PUSP - { - - ProviderType = DIRACCA - - # PUSP service URL - ServiceURL = https://mypuspserver.com/ - } - ## - ## OAuth2 type: MY_OAuth2 { @@ -765,7 +769,7 @@ Resources } } # FTS endpoint definition http://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/DataManagement/fts.htmlfts-servers-definition - # Passed to the constructor of the pluginFTSEndpoints + FTSEndpoints { FTS3 { @@ -791,6 +795,7 @@ Resources SpaceReservation = LHCb-EOS # Space reservation name if any. Concept like SpaceToken ArchiveTimeout = 84600 # Timeout for the FTS archiving BringOnlineTimeout = 84600 # Timeout for the bring online operation used by FTS + WLCGTokenBasePath = /eos/lhcb # EXPERIMENTAL Path from which the token should be relative to # Protocol section, see http://dirac.readthedocs.io/en/latest/AdministratorGuide/Resources/Storages/index.html#available-protocol-plugins GFAL2_SRM2 { @@ -852,6 +857,173 @@ Resources # This option is used in the Job Wrapper and, if set, requires the RequestManagementSystem to be installed Tier1-Failover = CERN-FAILOVER,CNAF-FAILOVER } + # Definition of the sites + # See http://dirac.readthedocs.io/en/latest/AdministratorGuide/Resources/site.html + Sites + { + LCG + { + ##BEGIN SiteConfiguration + LCG.CERN.ch + { + # Local Storages + SE = CERN-RAW, CERN-RDST, CERN-USER # (Optional) SEs Local to the site + + # Overwrite definities of StorageElement (discouraged) + # or StorageElementGroups for that Site + AssociatedSEs + { + # Tier1-Failover is now only CERN-FAILOVER when running a Job at CERN + Tier1-Failover = CERN-FAILOVER + } + Name = CERN-PROD # (Optional) Name of the site from the admin, e.g in GOCDB + Coordinates = 06.0458:46.2325 # (Optional) Geographical coordinates + Mail = grid-cern-prod-admins@cern.ch # (Optional) Site Admin email + MoUTierLevel = 0 # (Optional) Tier level + Description = CERN European Organization for Nuclear Research # (Optional) ... + # Subsection to describe each CE available + CEs + { + # Subsection named as the CE fully qualified name + ce503.cern.ch + { + + # (Optional) CE architecture + architecture = x86_64 + + # (Optional) CE operating system in a DIRAC format (purely for description) + OS = ScientificCERNSLC_Carbon_6.4 + + # (Optional) Boolean attributes that indicates if the site accept pilots (default: True) + Pilot = False + + # Type of CE, can take any CE type DIRAC recognizes (:ref: `CE`) + CEType = HTCondorCE + + # (Optional) Type of 'Inner' CE, normally empty. Default = "InProcess". + # Possibilities: potentially all CE types, but in practice + # the most valid would be: InProcess, Sudo, Singularity, Pool. + # Pool CE in turn uses InProcess (Default) + # or Sudo or Singularity. To specify, use Pool/ce_type. + # This option can also go at the Queue level. + LocalCEType = Pool + + # (Optional) max number of processors that DIRAC pilots are allowed to exploit. Implicit default = 1 + NumberOfProcessors = 12 + + # (Optional) Number of available worker nodes per allocation. + # Values can be a number (e.g. 2 nodes) or a range of values + # (e.g. from 2 to 4 nodes) which leaves the choice to the batch + # system. + NumberOfNodes = 2 + # NumberOfNodes = 2-4 + + # (Optional) CE allows *whole node* jobs + WholeNode = True + + # (Optional) List of tags specific for the CE + Tag = GPU, 96RAM + + # (Optional) List of required tags that a job to be eligible must have + RequiredTag = GPU,96RAM + + # Queues available for this VO in the CE + Queues + { + # Name of the queue + ce503.cern.ch-condor + { + # Name of the queue in the corresponding CE if not the same + # as the name of the queue section + # (should be avoided) + CEQueueName = pbs-grid + VO = lhcb + + # CE CPU Scaling Reference + SI00 = 3100 + + # Maximum number of jobs in all statuses + MaxTotalJobs = 5000 + + # Maximum number of jobs in waiting status + MaxWaitingJobs = 200 + + # Maximum time allowed to jobs to run in the queue + maxCPUTime = 7776 + + # The URL where to find the outputs + OutputURL = gsiftp://locahost + + # Overwrites NumberOfProcessors at the CE level + NumberOfProcessors = 12 + + # Overwrites NumberOfNodes at the CE level + NumberOfNodes = 12 + + # Overwrites WholeNode at the CE level + WholeNode = True + + # Overwrites LocalCEType at the CE level + LocalCEType = Pool/Singularity + + # List of tags specific for the Queue + Tag = MultiProcessor + + # List of required tags that a job to be eligible must have + RequiredTag = GPU,96RAM + } + } + VO = lhcb + MaxRAM = 0 + UseLocalSchedd = False + DaysToKeepLogs = 1 + } + } + } + ##END + } + } + + ##BEGIN CountriesConfiguration + Countries + { + # Configuration for ``pl`` sites + pl + { + # Redirect to ``de`` configuration + AssignedTo = de + } + de + { + AssociatedSEs + { + # Overwrite the Tier1-Failover StorageElementGroup + # For all German site which do not have a specific + # configuration (see # See https://dirac.readthedocs.io/en/latest/AdministratorGuide/Resources/storage.html#mapping-storages-to-sites-and-countries) + Tier1-Failover = GRIDKA-FAILOVER + } + } + } + ##END + + # Configuration for logging backends + # https://dirac.readthedocs.io/en/latest/DeveloperGuide/AddingNewComponents/Utilities/gLogger/gLogger/Basics/index.html#backend-resources + LogBackends + { + # Configure the stdout backend + stdout + { + LogLevel = INFO + } + # Example for a log backend sending to message queue + # see https://dirac.readthedocs.io/en/latest/AdministratorGuide/ServerInstallations/centralizedLogging.html + mqLogs + { + MsgQueue = lhcb-mb.cern.ch::Queues::lhcb.dirac.logging + # Name of the plugin if not the section name + Plugin = messageQueue + } + } } Operations { @@ -875,6 +1047,8 @@ Operations Defaults { # Flag for globally disabling the use of the SecurityLogging service + # This is False by default, as should be migrated to use centralized logging + # (see https://dirac.readthedocs.io/en/latest/AdministratorGuide/ServerInstallations/centralizedLogging.html#logstash-and-elk-configurations) EnableSecurityLogging = False DataManagement { @@ -890,7 +1064,6 @@ Operations WriteProtocols = srm WriteProtocols += dips # FTS related options. See http://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/DataManagement/fts.html - FTSVersion = FTS3 # should only be that... FTSPlacement { FTS3 @@ -930,6 +1103,51 @@ Operations } } } + # Specify how job access their data + # None of these fields is mandatory + # See https://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/WorkloadManagement/InputDataResolution.html + InputDataPolicy + { + # Default policy + Default = DIRAC.WorkloadManagementSystem.Client.InputDataByProtocol + # A job running at CERN would stream the data + LCG.CERN.cern = DIRAC.WorkloadManagementSystem.Client.InputDataByProtocol + # A job running at GRIDKA would download the files on the WN + LCG.GRIDKA.de = DIRAC.WorkloadManagementSystem.Client.DownloadInputData + # Shortcut for the JobAPI: job.setInputDataPolicy('Download') + Download = DIRAC.WorkloadManagementSystem.Client.DownloadInputData + # Shortcut for the JobAPI: job.setInputDataPolicy('Protocol') + Protocol = DIRAC.WorkloadManagementSystem.Client.InputDataByProtocol + # Used to limit or not the replicas considered by a Job in case of streaming + # See src/DIRAC/WorkloadManagementSystem/Client/InputDataByProtocol.py + AllReplicas = True + # List of protocols to use for streaming + Protocols + { + # This list is used if the we are getting a file from a + # StorageElement local to the site we are running on + Local = file, xroot, root + # This list is used if the SE is not local + Remote = xroot, root + } + # Module used for InputData resolution if not specified in the JDL + InputDataModule = DIRAC.Core.Utilities.InputDataResolution + } + Logging + { + # Default log backends and level applied to Services if + # it is not defined in the service specific section + DefaultServicesBackends = stdout + DefaultServicesLogLevel = INFO + + # Similar options for agents + DefaultAgentsBackends = stdout, mqLogs + DefaultAgentsLogLevel = VERBOSE + + # Default log level that is applied in last resort + DefaultLogLevel = DEBUG + + } # Options for the pilot3 # See https://dirac.readthedocs.io/en/latest/AdministratorGuide/Systems/WorkloadManagement/Pilots/Pilots3.html Pilot @@ -940,7 +1158,6 @@ Operations pilotVORepo = https://github.com/MyDIRAC/VOPilot.git # git repository of the pilot extension pilotVOScriptsPath = VOPilot # Path to the code, inside the Git repository pilotVORepoBranch = master # Branch to use - uploadToWebApp = True # Try to upload the files from the CS to the list of servers workDir = /tmp/pilot3Files # Local work directory on the masterCS for synchronisation } Services diff --git a/docs/README b/docs/README index 82d26615fc8..57547330219 100644 --- a/docs/README +++ b/docs/README @@ -3,10 +3,19 @@ How to build DIRAC documentation 1. Create DIRAC client environment by an appropriate source bashrc + or + source diracos/diracosrc + + Appropriate means according to the version of DIRAC you want to build the documentation for. See also the [developer + instructions](https://dirac.readthedocs.io/en/latest/DeveloperGuide/DevelopmentEnvironment/DeveloperInstallation/editingCode.html#installing-the-local-version) + + +1.1 Depending on how you installed DIRAC you might need to install + pip install sphinx_rtd_theme sphinx_design 2. Go to the Documentation directory of the DIRAC source code repository cd DIRAC/docs/ - export PYTHONPATH=$PWD/diracdoctools:$PYTHONPATH + export PYTHONPATH=$PWD:$PYTHONPATH 3. Run the documentation building script @@ -14,6 +23,8 @@ How to build DIRAC documentation make htmlall + Note: You must avoid having the string "test" in your folder structure or no code documentation will be created. + 3.1 to run incremental builds after rst files have changed, only run make html diff --git a/docs/diracdoctools/CustomizedDocs/CustomClient.py b/docs/diracdoctools/CustomizedDocs/CustomClient.py index 355e1c96d5b..ee032ba756c 100644 --- a/docs/diracdoctools/CustomizedDocs/CustomClient.py +++ b/docs/diracdoctools/CustomizedDocs/CustomClient.py @@ -4,7 +4,7 @@ MODULE = "DIRAC.Core.Base.Client" -class CustomClient(object): # pylint: disable=too-few-public-methods +class CustomClient: # pylint: disable=too-few-public-methods """Add the initialize function to Core.Base.Client docs""" def __init__(self): diff --git a/docs/diracdoctools/Utilities.py b/docs/diracdoctools/Utilities.py index fb1c31d096f..f99ea767a52 100644 --- a/docs/diracdoctools/Utilities.py +++ b/docs/diracdoctools/Utilities.py @@ -57,7 +57,7 @@ def writeLinesToFile(filename, lines): except AttributeError: # ignore decode if newContent is python3 str pass - except (UnicodeDecodeError) as e: + except UnicodeDecodeError as e: LOG.error('Failed to decode newContent with "utf-8": %r', e) raise rst.write(newContent) @@ -149,7 +149,7 @@ def check(): ] ret = subprocess.run(cmd, check=False) if ret.returncode != 1: - print("Return code from {} was {}".format(" ".join(cmd), ret.returncode)) + print(f"Return code from {' '.join(cmd)} was {ret.returncode}") print("This means :param or :return in the html and points to faulty " "syntax, missing empty lines, etc.") # https://bugs.python.org/issue27035 os._exit(1) # pylint: disable=protected-access diff --git a/docs/diracdoctools/__init__.py b/docs/diracdoctools/__init__.py index 5bc09f91ae7..de9db961bd7 100644 --- a/docs/diracdoctools/__init__.py +++ b/docs/diracdoctools/__init__.py @@ -9,7 +9,7 @@ "_arc", "arc", "cmreslogging", - "fts3", + "diracx", "gfal2", "git", "lcg_util", diff --git a/docs/diracdoctools/cmd/codeReference.py b/docs/diracdoctools/cmd/codeReference.py index d719fa624f8..65a1deacb8d 100644 --- a/docs/diracdoctools/cmd/codeReference.py +++ b/docs/diracdoctools/cmd/codeReference.py @@ -87,9 +87,9 @@ def mkPackageRst(self, filename, modulename, fullmodulename, subpackages=None, m modulefinal = modulename lines = [] - lines.append("%s" % modulefinal) + lines.append(f"{modulefinal}") lines.append("=" * len(modulefinal)) - lines.append(".. automodule:: %s " % fullmodulename) + lines.append(f".. automodule:: {fullmodulename} ") lines.append(" :members:") lines.append("") @@ -108,7 +108,7 @@ def mkPackageRst(self, filename, modulename, fullmodulename, subpackages=None, m lines.append(" :maxdepth: 1") lines.append("") for package in sorted(subpackages): - lines.append(" {}/{}_Module.rst".format(package, package.split("/")[-1])) + lines.append(f" {package}/{package.split('/')[-1]}_Module.rst") lines.append("") # remove CLI etc. because we drop them earlier @@ -121,7 +121,7 @@ def mkPackageRst(self, filename, modulename, fullmodulename, subpackages=None, m lines.append(" :maxdepth: 1") lines.append("") for module in sorted(modules): - lines.append(" {}.rst".format(module.split("/")[-1])) + lines.append(f" {module.split('/')[-1]}.rst") lines.append("") writeLinesToFile(filename, lines) @@ -131,10 +131,10 @@ def mkDummyRest(self, classname, fullclassname): filename = classname + ".rst" lines = [] - lines.append("%s" % classname) + lines.append(f"{classname}") lines.append("=" * len(classname)) lines.append("") - lines.append(".. py:module:: %s" % fullclassname) + lines.append(f".. py:module:: {fullclassname}") lines.append("") lines.append("This is an empty file, because we cannot parse this file correctly or it causes problems.") lines.append("Please look at the source code directly") @@ -146,10 +146,10 @@ def mkModuleRst(self, classname, fullclassname, buildtype="full"): filename = classname + ".rst" lines = [] - lines.append("%s" % classname) + lines.append(f"{classname}") lines.append("=" * len(classname)) - lines.append(".. automodule:: %s" % fullclassname) + lines.append(f".. automodule:: {fullclassname}") if buildtype == "full": lines.append(" :members:") if not any(x in self.config.code_noInherited for x in [classname, fullclassname]): @@ -337,7 +337,7 @@ def createCodeDocIndex(self, subpackages, modules, buildtype="full"): lines.append(" :maxdepth: 1") lines.append("") for package in systemPackages: - lines.append(" {}/{}_Module.rst".format(package, package.split("/")[-1])) + lines.append(f" {package}/{package.split('/')[-1]}_Module.rst") lines.append("") lines.append("=====") @@ -348,11 +348,11 @@ def createCodeDocIndex(self, subpackages, modules, buildtype="full"): lines.append(" :maxdepth: 1") lines.append("") for package in otherPackages: - lines.append(" {}/{}_Module.rst".format(package, package.split("/")[-1])) + lines.append(f" {package}/{package.split('/')[-1]}_Module.rst") if modules: for module in sorted(modules): - lines.append(" {}.rst".format(module.split("/")[-1])) + lines.append(f" {module.split('/')[-1]}.rst") if self.config.code_add_commands_section: lines.append("") diff --git a/docs/diracdoctools/cmd/commandReference.py b/docs/diracdoctools/cmd/commandReference.py index 7cd44cd2954..f7507eefa65 100644 --- a/docs/diracdoctools/cmd/commandReference.py +++ b/docs/diracdoctools/cmd/commandReference.py @@ -52,7 +52,7 @@ def __init__(self, configFile="docs.conf", debug=False): self.scriptDocs = {} # Scripts docs collection if not os.path.exists(self.config.sourcePath): - LOG.error("%s does not exist" % self.config.sourcePath) + LOG.error(f"{self.config.sourcePath} does not exist") raise RuntimeError("Package not found") def createSectionAndIndex(self, sectionDict: dict): @@ -116,7 +116,9 @@ def createAllScriptsDocsAndWriteToRST(self): .. _cmd: + ================= Command Reference + ================= In this subsection all commands are collected: @@ -144,9 +146,9 @@ def createAllScriptsDocsAndWriteToRST(self): f""" .. _{system}_cmd: - {"=" * len(system)} + {"-" * len(system)} {system} - {"=" * len(system)} + {"-" * len(system)} """ ) @@ -179,6 +181,7 @@ def createScriptDoc(self, script: str): if not helpMessage: LOG.warning("NO DOC for %s", scriptName) helpMessage = "Oops, we couldn't generate a description for this command." + self.exitcode = 1 # Script reference fileContent = textwrap.dedent( @@ -186,9 +189,8 @@ def createScriptDoc(self, script: str): .. _{scriptName}: - {'-' * len(scriptName)} {scriptName} - {'-' * len(scriptName)} + {'`' * len(scriptName)} """ ) diff --git a/docs/diracdoctools/cmd/concatcfg.py b/docs/diracdoctools/cmd/concatcfg.py index 1c367828c49..ac0fc5134d1 100644 --- a/docs/diracdoctools/cmd/concatcfg.py +++ b/docs/diracdoctools/cmd/concatcfg.py @@ -116,7 +116,7 @@ def parseConfigTemplate(self, templatePath, cfg): templatePath = os.path.join(templatePath, "ConfigTemplate.cfg") if not os.path.exists(templatePath): - return S_ERROR("File not found: %s" % templatePath) + return S_ERROR(f"File not found: {templatePath}") loadCfg = CFG() try: @@ -125,7 +125,7 @@ def parseConfigTemplate(self, templatePath, cfg): LOG.error("Failed loading file %r: %r", templatePath, err) self.retVal = 1 return S_ERROR() - cfg.createNewSection("/Systems/%s" % system, contents=loadCfg) + cfg.createNewSection(f"/Systems/{system}", contents=loadCfg) return S_OK(cfg) diff --git a/docs/diracdoctools/scripts/dirac-docs-get-release-notes.py b/docs/diracdoctools/scripts/dirac-docs-get-release-notes.py index 9b3fc28f99e..bfe360b8050 100755 --- a/docs/diracdoctools/scripts/dirac-docs-get-release-notes.py +++ b/docs/diracdoctools/scripts/dirac-docs-get-release-notes.py @@ -77,7 +77,7 @@ def githubSetup(GITHUBTOKEN=""): except ImportError: raise ImportError(G_ERROR) if GITHUBTOKEN: - SESSION.headers.update({"Accept": "application/vnd.github.v3+json", "Authorization": "token %s " % GITHUBTOKEN}) + SESSION.headers.update({"Accept": "application/vnd.github.v3+json", "Authorization": f"token {GITHUBTOKEN} "}) def gitlabSetup(GITLABTOKEN=""): @@ -92,11 +92,11 @@ def gitlabSetup(GITLABTOKEN=""): SESSION.headers.update({"PRIVATE-TOKEN": GITLABTOKEN}) -def req2Json(url, parameterDict=None, requestType="GET"): +def req2Json(url, parameterDict=None, requestType="GET", queryParameters=None): """Call to github API using requests package.""" log = LOGGER.getChild("Requests") log.debug("Running %s with %s ", requestType, parameterDict) - req = getattr(SESSION, requestType.lower())(url, json=parameterDict) + req = getattr(SESSION, requestType.lower())(url, json=parameterDict, params=queryParameters) if req.status_code not in (200, 201): log.error("Unable to access API: %s", req.text) raise RuntimeError("Failed to access API") @@ -428,12 +428,12 @@ def parseOptions(self): self.owner = repos[0] self.repo = repos[1] else: - raise RuntimeError("Cannot parse repo option: %s" % repo) + raise RuntimeError(f"Cannot parse repo option: {repo}") for var, val in sorted(vars(parsed).items()): log.info("Using options: %s = %s", var, pformat(val)) - def _github(self, action): + def _github(self, action, per_page=None): """Return the url to perform actions on github. :param str action: command to use in the gitlab API, see documentation there @@ -442,7 +442,8 @@ def _github(self, action): log = LOGGER.getChild("GitHub") options = dict(self._options) options["action"] = action - ghURL = "https://api.github.com/repos/%(owner)s/%(repo)s/%(action)s" % options + + ghURL = f"https://api.github.com/repos/{options['owner']}/{options['repo']}/{options['action']}" log.debug("Calling: %s", ghURL) return ghURL @@ -456,7 +457,7 @@ def _gitlab(self, action): def getGitlabPRs(self, state="opened"): """Get PRs in the gitlab repository.""" - glURL = self._gitlab("merge_requests?state=%s" % state) + glURL = self._gitlab(f"merge_requests?state={state}") return req2Json(glURL) def getGithubPRs(self, state="open", mergedOnly=False, perPage=100): @@ -504,9 +505,9 @@ def getGithubLatestTagDate(self, sinceTag): log = LOGGER.getChild("getGithubLatestTagDate") # Get all tags - tags = req2Json(url=self._github("tags")) + tags = req2Json(url=self._github("tags"), queryParameters={"per_page": 100}) if isinstance(tags, dict) and "Not Found" in tags.get("message"): - raise RuntimeError("Package not found: %s" % str(self)) + raise RuntimeError(f"Package not found: {str(self)}") if sinceTag: for tag in tags: @@ -514,7 +515,7 @@ def getGithubLatestTagDate(self, sinceTag): latestTag = tag break else: - raise ValueError("Tag %s not found" % sinceTag) + raise ValueError(f"Tag {sinceTag} not found") else: sortedTags = sorted(tags, key=lambda tag: LooseVersion(tag["name"]), reverse=True) latestTag = sortedTags[0] @@ -523,7 +524,7 @@ def getGithubLatestTagDate(self, sinceTag): # Use the sha of the commit to finally retrieve the date latestTagCommitSha = latestTag["commit"]["sha"] - commitInfo = req2Json(url=self._github("git/commits/%s" % latestTagCommitSha)) + commitInfo = req2Json(url=self._github(f"git/commits/{latestTagCommitSha}")) startDate = dateutil.parser.isoparse(commitInfo["committer"]["date"]) @@ -615,16 +616,16 @@ def collateReleaseNotes(self, prs): # CAUTION: no [tag] will be written if not prs: if headerMessage: - releaseNotes += "%s\n\n" % headerMessage + releaseNotes += f"{headerMessage}\n\n" if footerMessage: - releaseNotes += "%s\n" % footerMessage + releaseNotes += f"{footerMessage}\n" return releaseNotes prMarker = "#" if self.useGithub else "!" for baseBranch, pr in prs.items(): - releaseNotes += "[%s]\n\n" % baseBranch + releaseNotes += f"[{baseBranch}]\n\n" if headerMessage: - releaseNotes += "%s\n\n" % headerMessage + releaseNotes += f"{headerMessage}\n\n" systemChangesDict = defaultdict(list) for prid, content in pr.items(): notes = content["comment"] @@ -641,13 +642,13 @@ def collateReleaseNotes(self, prs): for system, changes in systemChangesDict.items(): if system: - releaseNotes += "*%s\n\n" % system + releaseNotes += f"*{system}\n\n" releaseNotes += "\n".join(changes) releaseNotes += "\n\n" releaseNotes += "\n" if footerMessage: - releaseNotes += "\n%s\n" % footerMessage + releaseNotes += f"\n{footerMessage}\n" return releaseNotes @@ -720,7 +721,6 @@ def createRelease(self): if __name__ == "__main__": - RUNNER = GithubInterface() try: RUNNER.parseOptions() diff --git a/docs/docs.conf b/docs/docs.conf index 7f1482963cb..870e5d28875 100644 --- a/docs/docs.conf +++ b/docs/docs.conf @@ -29,11 +29,11 @@ document_private_members = FCConditionsParser # whose docstrings are not reST formatted no_inherited_members = DIRAC.Core.Utilities.Graphs.GraphUtilities, - DIRAC.DataManagementSystem.private.HttpStorageAccessHandler + DIRAC.DataManagementSystem.private.HttpStorageAccessHandler, + DIRAC.FrameworkSystem.private.standardLogging.LogLevels, # only creating dummy files, because they cannot be safely imported due to sideEffects -create_dummy_files = lfc_dfc_copy, lfc_dfc_db_copy, JobWrapperTemplate, PlotCache, - PlottingHandler +create_dummy_files = lfc_dfc_copy, lfc_dfc_db_copy, JobWrapperTemplate, JobWrapperOfflineTemplate # do not include these files in the documentation tree ignore_folders = diracdoctools, /test, /scripts @@ -95,7 +95,7 @@ sectionPath = source/AdministratorGuide/CommandReference title = General information # [mandatory] pattern to match in the full path of the command names. -pattern = dirac-admin-service-ports, dirac-platform +pattern = dirac-platform # this list of patterns will reject scripts that are matched by the patterns above # exclude = user @@ -137,7 +137,7 @@ prefix = admin_software [commands.admin.g7] title = User convenience -pattern = utils, myproxy, cert, accounting-d +pattern = utils, cert, accounting-d prefix = admin_user [commands.admin.g8] @@ -162,6 +162,13 @@ title = Workload Management sectionPath = source/UserGuide/CommandReference/%(title)s [commands.z_section3] -pattern = dirac-proxy, dirac-info, myproxy, -resource- +pattern = dirac-proxy, dirac-info, -resource- title = Others sectionPath = source/UserGuide/CommandReference/%(title)s + +[commands.comdirac] +prefix = shorthand +pattern = d +exclude = dirac +title = Shorthand +sectionPath = source/UserGuide/CommandReference/%(title)s diff --git a/docs/setup.py b/docs/setup.py index 7d1708bca76..aaaa18d7d33 100755 --- a/docs/setup.py +++ b/docs/setup.py @@ -11,15 +11,15 @@ # Take all the packages but the scripts and tests ALL_PACKAGES = find_packages(where=BASE_DIR, exclude=["*test*"]) -PACKAGE_DIR = {"%s" % p: os.path.join(BASE_DIR, p.replace(".", "/")) for p in ALL_PACKAGES} +PACKAGE_DIR = {f"{p}": os.path.join(BASE_DIR, p.replace(".", "/")) for p in ALL_PACKAGES} # We rename the packages so that they contain diracdoctools -ALL_PACKAGES = ["diracdoctools.%s" % p for p in ALL_PACKAGES] +ALL_PACKAGES = [f"diracdoctools.{p}" for p in ALL_PACKAGES] ALL_PACKAGES.insert(0, "diracdoctools") PACKAGE_DIR["diracdoctools"] = BASE_DIR # The scripts to be distributed -SCRIPTS = glob.glob("%s/scripts/*.py" % BASE_DIR) +SCRIPTS = glob.glob(f"{BASE_DIR}/scripts/*.py") setup( name="diracdoctools", @@ -29,5 +29,5 @@ package_dir=PACKAGE_DIR, packages=ALL_PACKAGES, scripts=SCRIPTS, - install_requires=["sphinx_rtd_theme", "sphinx_panels"], + install_requires=["sphinx_rtd_theme", "sphinx_design"], ) diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Operations/DataManagement/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Operations/DataManagement/index.rst index a2edc57c0e4..0ff5dd6d359 100644 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Operations/DataManagement/index.rst +++ b/docs/source/AdministratorGuide/Configuration/ConfReference/Operations/DataManagement/index.rst @@ -17,7 +17,6 @@ Operations / DataManagement * WriteProtocols (['srm', 'dips']): list of the possible protocols to be used to perform the write and remove operations. Overwritten at the level of a StorageElement configuration. * AllowUserReplicaManagement (False): if set to True, clients without the FileCatalogManagement property can use the dirac-dms-remove-catalog-* commands to manipulate the file catalog. * ForceIndexedMetadata (False): if True disables implicit creation of non-indexed Metadata. -* FTSVersion (FTS2): version of FTS to use. Possibilities: FTS3 or FTS2 (deprecated) * FTSPlacement section: - FTS3 section: diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Operations/JobScheduling/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Operations/JobScheduling/index.rst index 13a8d6098b1..101f119460f 100644 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Operations/JobScheduling/index.rst +++ b/docs/source/AdministratorGuide/Configuration/ConfReference/Operations/JobScheduling/index.rst @@ -4,7 +4,7 @@ Job Scheduling ========================================= -The */Operations///JobScheduling* section contains all parameters that define DIRAC's behaviour when deciding what job has to be +The */Operations//JobScheduling* section contains all parameters that define DIRAC's behaviour when deciding what job has to be executed. Here's a list of parameters that can be defined: ========================= ======================================================== =============================================================================================== @@ -46,7 +46,7 @@ Example ======== An example with all the options under *JobScheduling* follows. Remember that JobScheduling is defined under -*/Operations///JobScheduling* for multi-VO installations, and */Operations//JobScheduling* for single-VO ones:: +*/Operations//JobScheduling* for multi-VO installations, and */Operations/JobScheduling* for single-VO ones:: JobScheduling { diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Operations/Pilots/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Operations/Pilots/index.rst index 419bbc2de0e..6e4902c43f3 100644 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Operations/Pilots/index.rst +++ b/docs/source/AdministratorGuide/Configuration/ConfReference/Operations/Pilots/index.rst @@ -2,7 +2,7 @@ Pilot version ========================================= -The */Operations///Pilot* section define What version of DIRAC will be used to submit pilot jobs to the resources. +The */Operations//Pilot* section define What version of DIRAC will be used to submit pilot jobs to the resources. ================== ======================================================== =============================================================================================== Parameter Description Default value diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Operations/VOs/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Operations/VOs/index.rst index a788351cc2d..9f118fdfc1e 100644 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Operations/VOs/index.rst +++ b/docs/source/AdministratorGuide/Configuration/ConfReference/Operations/VOs/index.rst @@ -1,18 +1,16 @@ Operations / VOs - Subsections ============================== - subsections allows to define pilot jobs versions for each setup defined for each VO supported by the server. + subsections allows to define pilot jobs DIRAC versions for each VO supported by the server. +-----------------------------------------------+----------------------------------------------+---------------------------+ | **Name** | **Description** | **Example** | +-----------------------------------------------+----------------------------------------------+---------------------------+ | ** | Subsection: Virtual organization name | vo.formation.idgrilles.fr | +-----------------------------------------------+----------------------------------------------+---------------------------+ -| *//* | Subsection: VO Setup name | Dirac-Production | -+-----------------------------------------------+----------------------------------------------+---------------------------+ This section will progressively incorporate most of the other sections under /Operations in such a way -that different values can be defined for each [VO] (in multi-VO installations) and [Setup]. A helper +that different values can be defined for each [VO] (in multi-VO installations). A helper class is provided to access to these new structure. :: diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Operations/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Operations/index.rst index 3666fae9135..b9b738cf5a2 100644 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Operations/index.rst +++ b/docs/source/AdministratorGuide/Configuration/ConfReference/Operations/index.rst @@ -13,8 +13,6 @@ This section allows to configure options concerning to: - Virtual Organization special parameters - Component Monitoring -In the short term, most of this schema will be moved into [vo]/[setup] dependent sections in order to allow better support for multi-VO installations. - .. toctree:: :maxdepth: 2 diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Resources/FileCatalogs/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Resources/FileCatalogs/index.rst deleted file mode 100644 index defefecd85a..00000000000 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Resources/FileCatalogs/index.rst +++ /dev/null @@ -1,16 +0,0 @@ -Resources / FileCatalogs - Subsections -====================================== - -This subsection include the definition of the File Catalogs to be used in the installation. In case there is more than one File Catalog defined in this section, the first one in the section will be used as default by the ReplicaManager client. - -+---------------------------+-------------------------------------------------+----------------------------+ -| **Name** | **Description** | **Example** | -+---------------------------+-------------------------------------------------+----------------------------+ -| *FileCatalog* | Subsection used to configure DIRAC File catalog | FileCatalog | -+---------------------------+-------------------------------------------------+----------------------------+ -| *FileCatalog/AccessType* | Access type allowed to the particular catalog | AccessType = Read-Write | -+---------------------------+-------------------------------------------------+----------------------------+ -| *FileCatalog/Status* | To define the catalog as active or inactive | Status = Active | -+---------------------------+-------------------------------------------------+----------------------------+ -| *FileCatalog/MetaCatalog* | If the Catalog is a MetaDataCatalog | MetaCatalog = True | -+---------------------------+-------------------------------------------------+----------------------------+ diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Resources/Sites/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Resources/Sites/index.rst deleted file mode 100644 index de00b4588fb..00000000000 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Resources/Sites/index.rst +++ /dev/null @@ -1,265 +0,0 @@ -.. _cs-site: - -Resources / Sites - Subsections -=============================== - -In this section each DIRAC site available for the users is described. The convention to name the sites consist of 3 strings: - -- Grid site name, expressed in uppercase, for example: LCG, EELA -- Institution acronym in uppercase, for example: CPPM -- Country: country where the site is located, expressed in lowercase, for example fr - -The three strings are concatenated with "." to produce the name of the sites. - -+------------------------------------+-----------------------------------------------+-----------------------------------+ -| **Name** | **Description** | **Example** | -+------------------------------------+-----------------------------------------------+-----------------------------------+ -| ** | Subsection named with the site name | LCG.CPPM.fr | -+------------------------------------+-----------------------------------------------+-----------------------------------+ -| */Name* | Site name gave by the site administrator | Name = in2p3 | -| | e.g.: the name of the site in GOCDB (optional)| | -+------------------------------------+-----------------------------------------------+-----------------------------------+ -| */CE* | List of CEs using CE FQN | CE = ce01.in2p3.fr | -| | These CEs are updated by the BDII2CSAgent | CE += ce02.in2p3.fr | -| | in the CEs section | | -+------------------------------------+-----------------------------------------------+-----------------------------------+ -| */MoUTierLevel* | Tier Level (optional) | MoUTierLevel = 1 | -+------------------------------------+-----------------------------------------------+-----------------------------------+ -| */CEs/* | Subsection used to describe each CE available | CEs | -+------------------------------------+-----------------------------------------------+-----------------------------------+ -| */Coordinates* | Site geographical coordinates (optional) | Coordinates = -8.637979:41.152461 | -+------------------------------------+-----------------------------------------------+-----------------------------------+ -| */Mail* | Mail address site responsable (optional) | Mail = atsareg@in2p3.fr | -+------------------------------------+-----------------------------------------------+-----------------------------------+ -| */SE* | Closest SE respect to the CE (optional) | SE = se01.in2p3.fr | -+------------------------------------+-----------------------------------------------+-----------------------------------+ - - -CEs sub-subsection -------------------- - -This sub-subsection specifies the attributes of each particular CE of the site. For each DIRAC site there can be more than one CE. - -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| **Name** | **Description** | **Example** | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| ** | Subsection named as the CE fully qualified name | ce01.in2p3.fr | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */architecture* | CE architecture | architecture = x86_64 | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */CEType* | Type of CE, can take values as LCG or CREAM | CEType = ARC | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */LocalCEType* | Type of 'Inner' CE, normally empty. Default = "InProcess". | LocalCEType = Pool | -| | Possibilities: potentially all CE types, but in practice | | -| | the most valid would be: InProcess, Sudo, Singularity, Pool.| | -| | Pool CE in turn uses InProcess (Default) | | -| | or Sudo or Singularity. To specify, use Pool/ce_type. | LocalCEType = Pool/Singularity | -| | This option can also go at the Queue level. | | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */OS* | CE operating system in a DIRAC format | OS = ScientificLinux_Boron_5.3 | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */Pilot* | Boolean attributes than indicates if the site accept pilots | Pilot = True | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */SubmissionMode* | If the CE is a cream CE the mode of submission | SubmissionMode = Direct | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */wnTmpDir* | Worker node temporal directory | wnTmpDir = /tmp | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */NumberOfProcessors* | Number of available processors on worker nodes | NumberOfProcessors = 12 | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */NumberOfNodes* | Number of available worker nodes per allocation. | NumberOfNodes = 2 | -| | Values can be a number (e.g. 2 nodes) or a range of values | | -| | (e.g. from 2 to 4 nodes) which lets the choice to the batch | NumberOfNodes = 2 or 2-4 | -| | system. | | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */WholeNode* | CE allows *whole node* jobs | WholeNode = True | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */Tag* | List of tags specific for the CE | Tag = GPU,96RAM | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */RequiredTag* | List of required tags that a job to be eligible must have | RequiredTag = GPU,96RAM | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */Queues* | Subsection. Queues available for this VO in the CE | Queues | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */Queues/* | Name of the queue exactly how is published | jobmanager-pbs-formation | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */Queues//CEQueueName* | Name of the queue in the corresponding CE if not the same | | -| | as the name of the queue section | CEQueueName = pbs-grid | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */Queues//maxCPUTime* | Maximum time allowed to jobs to run in the queue | maxCPUTime = 1440 | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */Queues//MaxTotalJobs* | If the CE is a CREAM CE the maximum number of jobs in all | MaxTotalJobs =200 | -| | the status | | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */Queues//MaxWaitingJobs* | If the CE is a CREAM CE the maximum number of jobs in | MaxWaitingJobs = 70 | -| | waiting status | | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */Queues//OutputURL* | If the CE is a CREAM CE the URL where to find the outputs | OutputURL = gsiftp://localhost | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */Queues//SI00* | CE CPU Scaling Reference | SI00 = 2130 | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */Queues//NumberOfProcessors* | overrides */NumberOfProcessors* at queue level | NumberOfProcessors = 12 | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */Queues//NumberOfNodes* | overrides */NumberOfNodes* | NumberOfNodes = 2 | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */Queues//WholeNode* | overrides */WholeNode* at queue level | WholeNode = True | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */Queues//Tag* | List of tags specific for the Queue | Tag = GPU,96RAM | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */Queues//RequiredTag* | List of required tags that a job to be eligible must have | RequiredTag = GPU,96RAM | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ -| */Queues//LocalCEType* | Same as */LocalCEType* (see above) but per queue. | LocalCEType = Pool/Singularity | -+----------------------------------------------------+-------------------------------------------------------------+--------------------------------+ - - -An example for this session follows:: - - Sites - { - LCG - { - LCG.CERN.cern - { - SE = CERN-RAW - SE += CERN-RDST - SE += CERN-USER - CE = ce503.cern.ch - CE += ce504.cern.ch - Name = CERN-PROD - Coordinates = 06.0458:46.2325 - Mail = grid-cern-prod-admins@cern.ch - MoUTierLevel = 0 - Description = CERN European Organization for Nuclear Research - CEs - { - ce503.cern.ch - { - wnTmpDir = . - architecture = x86_64 - OS = ScientificCERNSLC_Carbon_6.4 - SI00 = 0 - Pilot = False - CEType = HTCondorCE - SubmissionMode = Direct - Queues - { - ce503.cern.ch-condor - { - VO = lhcb - VO += LHCb - SI00 = 3100 - MaxTotalJobs = 5000 - MaxWaitingJobs = 200 - maxCPUTime = 7776 - LocalCEType = Pool/Singularity - Tag = MultiProcessor - } - } - VO = lhcb - MaxRAM = 0 - UseLocalSchedd = False - DaysToKeepLogs = 1 - } - ce504.cern.ch - { - wnTmpDir = . - architecture = x86_64 - OS = ScientificCERNSLC_Carbon_6.4 - SI00 = 0 - Pilot = False - CEType = HTCondorCE - LocalCEType = Pool - SubmissionMode = Direct - Queues - { - ce504.cern.ch-condor - { - VO = lhcb - VO += LHCb - SI00 = 3100 - MaxTotalJobs = 5000 - MaxWaitingJobs = 200 - maxCPUTime = 7776 - } - } - } - } - } - } - DIRAC - { - DIRAC.HLTFarm.lhcb - { - Name = LHCb-HLTFARM - CE = OnlineCE.lhcb - CEs - { - OnlineCE.lhcb - { - CEType = CREAM - Queues - { - OnlineQueue - { - maxCPUTime = 2880 - } - } - } - } - AssociatedSEs - { - Tier1-RDST = CERN-RDST - Tier1_MC-DST = CERN_MC-DST-EOS - Tier1-Buffer = CERN-BUFFER - Tier1-Failover = CERN-EOS-FAILOVER - Tier1-BUFFER = CERN-BUFFER - Tier1-USER = CERN-USER - SE-USER = CERN-USER - } - } - } - VAC - { - VAC.Manchester.uk - { - Name = UKI-NORTHGRID-MAN-HEP - CE = vac01.blackett.manchester.ac.uk - CE += vac02.blackett.manchester.ac.uk - Coordinates = -2.2302:53.4669 - Mail = ops@NOSPAMtier2.hep.manchester.ac.uk - CEs - { - vac01.blackett.manchester.ac.uk - { - CEType = Vac - architecture = x86_64 - OS = ScientificSL_Carbon_6.4 - wnTmpDir = /scratch - SI00 = 2200 - MaxCPUTime = 1000 - Queues - { - default - { - maxCPUTime = 1000 - } - } - } - vac02.blackett.manchester.ac.uk - { - CEType = Vac - architecture = x86_64 - OS = ScientificSL_Carbon_6.4 - wnTmpDir = /scratch - SI00 = 2200 - MaxCPUTime = 1000 - Queues - { - default - { - maxCPUTime = 1000 - } - } - } - } - } - } - } diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Resources/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Resources/index.rst deleted file mode 100644 index d18a2b3a176..00000000000 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Resources/index.rst +++ /dev/null @@ -1,13 +0,0 @@ -.. _dirac-resources-cs: - -Resources - Section -=================== - -In this section all the physical resources than can be used by DIRAC users are described. - - -.. toctree:: - :maxdepth: 2 - - FileCatalogs/index - Sites/index diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Accounting/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Accounting/index.rst index c34d6fc5c2f..48327d881dc 100644 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Accounting/index.rst +++ b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Accounting/index.rst @@ -1,7 +1,7 @@ Accounting System configuration ================================== -In this subsection are described the databases, services and URLs related with Accounting framework for each setup. +In this subsection are described the databases, services and URLs related with Accounting framework. .. toctree:: :maxdepth: 2 diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Configuration/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Configuration/index.rst index 1ac5dd906ef..73759abd54c 100644 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Configuration/index.rst +++ b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Configuration/index.rst @@ -1,7 +1,7 @@ Configuration System configuration ================================== -In this subsection are described the databases, services and URLs related with Accounting framework for each setup. +In this subsection are described the databases, services and URLs related with Accounting framework. .. toctree:: :maxdepth: 2 diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/DataManagement/Services/StorageElementProxy/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/DataManagement/Services/StorageElementProxy/index.rst deleted file mode 100644 index c084b20e240..00000000000 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/DataManagement/Services/StorageElementProxy/index.rst +++ /dev/null @@ -1,12 +0,0 @@ -Systems / DataManagement / / Service / StorageElementProxy - Sub-subsection -====================================================================================== - -This is a service which represents a DISET proxy to the Storage Element component. -This is used to get and put files from a remote storage. - -+------------+---------------------+---------------------------+ -| **Name** | **Description** | **Example** | -+------------+---------------------+---------------------------+ -| *BasePath* | Temporary directory | BasePath = storageElement | -| | use for transfers | | -+------------+---------------------+---------------------------+ diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/DataManagement/Services/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/DataManagement/Services/index.rst index 2d5abbe5265..6dca087d44e 100644 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/DataManagement/Services/index.rst +++ b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/DataManagement/Services/index.rst @@ -32,4 +32,3 @@ DataManagement services are: :maxdepth: 2 FileCatalog/index - StorageElementProxy/index diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/DataManagement/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/DataManagement/index.rst index e32dd9535f5..4d744f64763 100644 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/DataManagement/index.rst +++ b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/DataManagement/index.rst @@ -1,7 +1,7 @@ DataManagement System configuration ====================================== -In this subsection are described the databases, services and URLs related with the DataManagement system for each setup. +In this subsection are described the databases, services and URLs related with the DataManagement system. .. toctree:: :maxdepth: 2 diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Agents/CAUpdateAgent/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Agents/CAUpdateAgent/index.rst deleted file mode 100644 index f948472311d..00000000000 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Agents/CAUpdateAgent/index.rst +++ /dev/null @@ -1,6 +0,0 @@ -Systems / Framework / / Agents / CAUpdateAgent - Sub-subsection -========================================================================== - -CA Update agent uses the Framework/BundleDelivery service to get up-to-date CAs and CRLs for all agent and servers using the same dirac installation. - -This agent has no options. diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Agents/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Agents/index.rst deleted file mode 100644 index 0f5bd50695e..00000000000 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Agents/index.rst +++ /dev/null @@ -1,41 +0,0 @@ -Systems / Framework / / Agents - Sub-subsection -============================================================== - - In this subsection each agent is described. - -+----------+----------------------------------+---------------+ -| **Name** | **Description** | **Example** | -+----------+----------------------------------+---------------+ -| *Agent* | Subsection named as the agent is | CAUpdateAgent | -| | called. | | -+----------+----------------------------------+---------------+ - -Common options for all the agents: - -+---------------------+---------------------------------------+------------------------------+ -| **Name** | **Description** | **Example** | -+---------------------+---------------------------------------+------------------------------+ -| *LogLevel* | Log Level associated to the agent | LogLevel = DEBUG | -+---------------------+---------------------------------------+------------------------------+ -| *LogBackends* | | LogBackends = stdout, ... | -+---------------------+---------------------------------------+------------------------------+ -| *MaxCycles* | Maximum number of cycles made for | MaxCycles = 500 | -| | Agent | | -+---------------------+---------------------------------------+------------------------------+ -| *MonitoringEnabled* | Indicates if the monitoring of agent | MonitoringEnabled = True | -| | is enabled. Boolean values | | -+---------------------+---------------------------------------+------------------------------+ -| *PollingTime* | Each many time a new cycle must start | PollingTime = 2600 | -| | expresed in seconds | | -+---------------------+---------------------------------------+------------------------------+ -| *Status* | Agent Status, possible values Active | Status = Active | -| | or Inactive | | -+---------------------+---------------------------------------+------------------------------+ - - -Agents associated with Framework System: - -.. toctree:: - :maxdepth: 2 - - CAUpdateAgent/index diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Databases/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Databases/index.rst index ccfa89a1dd7..69f71e8b41d 100644 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Databases/index.rst +++ b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Databases/index.rst @@ -17,6 +17,5 @@ Databases used by Framework System. Note that each database is a separate subsec +--------------------------------+----------------------------------------------+----------------------+ The databases associated to Framework System are: -- NotificationDB - ProxyDB - UserProfileDB diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Services/BundleDelivery/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Services/BundleDelivery/index.rst deleted file mode 100644 index 913ef6e14f0..00000000000 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Services/BundleDelivery/index.rst +++ /dev/null @@ -1,15 +0,0 @@ -Systems / Framework / / Service / BundleDelivery - Sub-subsection -============================================================================ - -Bundle delivery services is used to transfer Directories to clients by making tarballs. - -+---------------------+---------------------------------------+---------------------------------------+ -| **Name** | **Description** | **Example** | -+---------------------+---------------------------------------+---------------------------------------+ -| *CAs* | Boolean, bundle CAs | CAs = True | -+---------------------+---------------------------------------+---------------------------------------+ -| *CRLs* | Boolean, bundle CRLs | CRLs = True | -+---------------------+---------------------------------------+---------------------------------------+ -| *DirsToBundle* | Section with Additional directories | DirsToBundle/NameA = /opt/dirac/NameA | -| | to serve | | -+---------------------+---------------------------------------+---------------------------------------+ diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Services/Monitoring/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Services/Monitoring/index.rst deleted file mode 100644 index e8845814243..00000000000 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Services/Monitoring/index.rst +++ /dev/null @@ -1,12 +0,0 @@ -Systems / Framework / / Service / Monitoring - Sub-subsection -======================================================================== - -Monitoring service is in charge of recollect the information necessary to create the plots. - -Extra options required to configure the monitoring system are: - -+----------------+------------------------------------------+--------------------------------+ -| **Name** | **Description** | **Example** | -+----------------+------------------------------------------+--------------------------------+ -| *DataLocation* | Path where data for monitoring is stored | DataLocation = data/Monitoring | -+----------------+------------------------------------------+--------------------------------+ diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Services/Notification/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Services/Notification/index.rst deleted file mode 100644 index 723abee499d..00000000000 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Services/Notification/index.rst +++ /dev/null @@ -1,25 +0,0 @@ -Systems / Framework / / Service / Notification - Sub-subsection -========================================================================== - -The Notification service provides a toolkit to contact people via email -(eventually SMS etc.) to trigger some actions. - -The original motivation for this is due to some sites restricting the -sending of email but it is useful for e.g. crash reports to get to their -destination. - -Another use-case is for users to request an email notification for the -completion of their jobs. When output data files are uploaded to the -Grid, an email could be sent by default with the metadata of the file. - -It can also be used to set alarms to be promptly forwarded to those -subscribing to them. - - -Extra options required to configure the Notification system are: - -+-------------+----------------------------------+---------------------------+ -| **Name** | **Description** | **Example** | -+-------------+----------------------------------+---------------------------+ -| *SMSSwitch* | SMS switch used to send messages | SMSSwithc = sms.switch.ch | -+-------------+----------------------------------+---------------------------+ diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Services/Plotting/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Services/Plotting/index.rst deleted file mode 100644 index 77c7e131965..00000000000 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Services/Plotting/index.rst +++ /dev/null @@ -1,12 +0,0 @@ -Systems / Framework / / Service / Plotting - Sub-subsection -======================================================================= - -Plotting Service generates graphs according to the client specifications and data. - -Extra options required to configure plotting system are: - -+-----------------+------------------------------------------+----------------------------+ -| **Name** | **Description** | **Example** | -+-----------------+------------------------------------------+----------------------------+ -| *PlotsLocation* | Path where data for monitoring is stored | PlotsLocation = data/plots | -+-----------------+------------------------------------------+----------------------------+ diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Services/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Services/index.rst index 823e349892d..f6ed7f07d0e 100644 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Services/index.rst +++ b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/Services/index.rst @@ -34,9 +34,5 @@ Services associated with Framework system are: .. toctree:: :maxdepth: 2 - BundleDelivery/index - Monitoring/index - Notification/index - Plotting/index SystemAdministrator/index UserProfileManager/index diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/URLs/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/URLs/index.rst deleted file mode 100644 index 3189ae157b6..00000000000 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/URLs/index.rst +++ /dev/null @@ -1,33 +0,0 @@ -Systems / Framework / / URLs - Sub-subsection -========================================================== - -Framework Services URLs. - -+------------------+--------------------------------------+-----------------------------------------------------------------+ -| **Name** | **Description** | **Example** | -+------------------+--------------------------------------+-----------------------------------------------------------------+ -| ** | URL associated with the service, the | Plotting = dips://dirac.eela.if.ufrj.br:9157/Framework/Plotting | -| | value is a URL using dips protocol | | -+------------------+--------------------------------------+-----------------------------------------------------------------+ - -Services associated with Framework System: - -+-----------------------+----------+ -| **Service** | **Port** | -+-----------------------+----------+ -| *BundleDelivery* | 9158 | -+-----------------------+----------+ -| *Monitoring* | 9142 | -+-----------------------+----------+ -| *Notification* | 9154 | -+-----------------------+----------+ -| *Plotting* | 9157 | -+-----------------------+----------+ -| *ProxyManagement* | 9152 | -+-----------------------+----------+ -| *SecurityLogging* | 9153 | -+-----------------------+----------+ -| *SystemAdministrator* | 9162 | -+-----------------------+----------+ -| *UserProfileManager* | 9155 | -+-----------------------+----------+ diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/index.rst index d605b114731..d45f74be544 100644 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/index.rst +++ b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Framework/index.rst @@ -1,12 +1,10 @@ Framework System configuration ================================== -In this subsection are described the databases, services and URLs related with Framework System for each setup. +In this subsection are described the databases, services and URLs related with Framework System. .. toctree:: :maxdepth: 2 Databases/index Services/index - Agents/index - URLs/index diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/StorageManagement/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/StorageManagement/index.rst index 88210e6ee07..458dea4f60b 100644 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/StorageManagement/index.rst +++ b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/StorageManagement/index.rst @@ -1,7 +1,7 @@ StorageManagement System configuration ========================================= -In this subsection are described the databases, services and URLs related with RequestManagement System for each setup. +In this subsection are described the databases, services and URLs related with RequestManagement System. .. toctree:: :maxdepth: 2 diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Transformation/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Transformation/index.rst index 8d009df3bee..2087f75fa48 100644 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Transformation/index.rst +++ b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/Transformation/index.rst @@ -2,7 +2,7 @@ Transformation System configuration ========================================= In this subsection are described the databases, services, agents, and URLs related to -Transformation System for each setup. +Transformation System. .. toctree:: :maxdepth: 2 diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/WorkloadManagement/Agents/PilotStatusAgent/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/WorkloadManagement/Agents/PilotStatusAgent/index.rst index 760010814dc..e3145a4d249 100644 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/WorkloadManagement/Agents/PilotStatusAgent/index.rst +++ b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/WorkloadManagement/Agents/PilotStatusAgent/index.rst @@ -8,9 +8,6 @@ Special attributes for this agent are: +--------------------------+--------------------------------------------+-----------------------------------+ | **Name** | **Description** | **Example** | +--------------------------+--------------------------------------------+-----------------------------------+ -| *GridEnv* | Path where is located the file to | GridEnv = /usr/profile.d/grid-env | -| | load Grid Environment Variables | | -+--------------------------+--------------------------------------------+-----------------------------------+ | *PilotAccountingEnabled* | Boolean type attribute than allows to | PilotAccountingEnabled = Yes | | | specify if accounting is enabled | | +--------------------------+--------------------------------------------+-----------------------------------+ diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/WorkloadManagement/Services/SandboxStore/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/WorkloadManagement/Services/SandboxStore/index.rst index 6df8c417ebb..e02976c93f5 100644 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/WorkloadManagement/Services/SandboxStore/index.rst +++ b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/WorkloadManagement/Services/SandboxStore/index.rst @@ -8,15 +8,8 @@ Some extra options are required to configure this service: +---------------------------+----------------------------------------------+-----------------------------------------+ | **Name** | **Description** | **Example** | +---------------------------+----------------------------------------------+-----------------------------------------+ -| *Backend* | | Backend = local | -+---------------------------+----------------------------------------------+-----------------------------------------+ | *BasePath* | Base path where the files are stored | BasePath = /opt/dirac/storage/sandboxes | | | task queues in the system | | +---------------------------+----------------------------------------------+-----------------------------------------+ -| *DelayedExternalDeletion* | Boolean used to define if the external | DelayedExternalDeletion = True | -| | deletion must be done | | -+---------------------------+----------------------------------------------+-----------------------------------------+ | *MaxSandboxSize* | Maximum size of sanbox files expressed in MB | MaxSandboxSize = 10 | +---------------------------+----------------------------------------------+-----------------------------------------+ -| *SandboxPrefix* | Path prefix where sandbox are stored | SandboxPrefix = Sandbox | -+---------------------------+----------------------------------------------+-----------------------------------------+ diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/WorkloadManagement/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/WorkloadManagement/index.rst index dc269a3e01f..a794d981887 100644 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/WorkloadManagement/index.rst +++ b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/WorkloadManagement/index.rst @@ -1,7 +1,7 @@ WorkloadManagement System configuration ======================================== -In this subsection are described the databases, services and URLs related with WorkloadManagement System for each setup. +In this subsection are described the databases, services and URLs related with WorkloadManagement System. .. toctree:: :maxdepth: 2 diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/index.rst index 64001a2b0e2..17237cbd36f 100644 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/index.rst +++ b/docs/source/AdministratorGuide/Configuration/ConfReference/Systems/index.rst @@ -25,7 +25,7 @@ Each DIRAC system has its corresponding section in the Configuration namespace. Default structure ----------------- -In each system, per setup, you normally find the following sections: +In each system you normally find the following sections: * Agents: definition of each agent * Services: definition of each service @@ -43,7 +43,7 @@ For this reason, there is the possibility to define a entry in the Operation sec .. code-block:: guess - Operations//MainServers = server1, server2 + Operations/MainServers = server1, server2 There should be no port, no protocol. In the system configuration, one can then write: diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Tips/Authorization/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Tips/Authorization/index.rst index 59a3e414e34..fb89af3bf25 100644 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Tips/Authorization/index.rst +++ b/docs/source/AdministratorGuide/Configuration/ConfReference/Tips/Authorization/index.rst @@ -11,8 +11,6 @@ are showed in the next table: +----------------------------+------------------------------------------------------------------+-------------+ | *AccountingMonitor* | Allow access to accounting data for all groups | | +----------------------------+------------------------------------------------------------------+-------------+ -| *AlarmsManagement* | Allow to set notifications and manage alarms | | -+----------------------------+------------------------------------------------------------------+-------------+ | *CSAdministrator* | CS Administrator - possibility to edit the Configuration Service | | +----------------------------+------------------------------------------------------------------+-------------+ | *FileCatalogManagement* | Allow FC Management | | @@ -31,11 +29,13 @@ are showed in the next table: +----------------------------+------------------------------------------------------------------+-------------+ | *Operator* | Operator | | +----------------------------+------------------------------------------------------------------+-------------+ -| *Pilot* | Private pilot | | -+----------------------------+------------------------------------------------------------------+-------------+ | *PrivateLimitedDelegation* | Allow getting only limited proxies for one self | | +----------------------------+------------------------------------------------------------------+-------------+ -| *ProductionManagement* | Allow managing production | | +| *ProductionManagement* | Allow managing all productions | | ++----------------------------+------------------------------------------------------------------+-------------+ +| *ProductionSharing* | Allow managing productions owned by the same group | | ++----------------------------+------------------------------------------------------------------+-------------+ +| *ProductionUser* | Allow managing productions owned by the same user | | +----------------------------+------------------------------------------------------------------+-------------+ | *ProxyManagement* | Allow managing proxies | | +----------------------------+------------------------------------------------------------------+-------------+ diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/Tips/ServicesPorts/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/Tips/ServicesPorts/index.rst index 5a391150baa..6d3507b0a4b 100644 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/Tips/ServicesPorts/index.rst +++ b/docs/source/AdministratorGuide/Configuration/ConfReference/Tips/ServicesPorts/index.rst @@ -23,14 +23,10 @@ Ordered by System / Services +------+----------------------+---------------------+ | 9148 | *DataManagement* | StorageElement | +------+----------------------+---------------------+ -| 9149 | *DataManagement* | StorageElementProxy | -+------+----------------------+---------------------+ | 9158 | *Framework* | BundleDelivery | +------+----------------------+---------------------+ | 9154 | *Framework* | Notification | +------+----------------------+---------------------+ -| 9157 | *Framework* | Plotting | -+------+----------------------+---------------------+ | 9152 | *Framework* | ProxyManager | +------+----------------------+---------------------+ | 9153 | *Framework* | SecurityLogging | @@ -78,8 +74,6 @@ Ordered by port number +------+----------------------+---------------------+ | 9148 | *DataManagement* | StorageElement | +------+----------------------+---------------------+ -| 9149 | *DataManagement* | StorageElementProxy | -+------+----------------------+---------------------+ | 9152 | *Framework* | ProxyManager | +------+----------------------+---------------------+ | 9153 | *Framework* | SecurityLogging | @@ -88,8 +82,6 @@ Ordered by port number +------+----------------------+---------------------+ | 9155 | *Framework* | UserProfileManager | +------+----------------------+---------------------+ -| 9157 | *Framework* | Plotting | -+------+----------------------+---------------------+ | 9158 | *Framework* | BundleDelivery | +------+----------------------+---------------------+ | 9162 | *Framework* | SystemAdministrator | diff --git a/docs/source/AdministratorGuide/Configuration/ConfReference/index.rst b/docs/source/AdministratorGuide/Configuration/ConfReference/index.rst index 080d132f3c9..15512da58a8 100644 --- a/docs/source/AdministratorGuide/Configuration/ConfReference/index.rst +++ b/docs/source/AdministratorGuide/Configuration/ConfReference/index.rst @@ -17,7 +17,6 @@ The detailed configuration options for agents, services, and executors are in th :maxdepth: 1 Operations/index - Resources/index Systems/index WebSite/index Tips/index diff --git a/docs/source/AdministratorGuide/Configuration/ConfigurationStructure/index.rst b/docs/source/AdministratorGuide/Configuration/ConfigurationStructure/index.rst index e2415a1f3be..8162acd6fbe 100644 --- a/docs/source/AdministratorGuide/Configuration/ConfigurationStructure/index.rst +++ b/docs/source/AdministratorGuide/Configuration/ConfigurationStructure/index.rst @@ -31,7 +31,7 @@ DIRAC Registry The *Registry* contains information about DIRAC users, groups and communities (VOs). -:ref:`Resources ` +Resources The *Resources* section provides description of all the DIRAC computing resources. This includes computing and storage elements as well as descriptions of several DIRAC and third party services. @@ -52,7 +52,7 @@ in the order of preference of the option resolution: For all the DIRAC commands there is option '-o' defined which takes one configuration option setting. For example:: - dirac-wms-job-submit job.jdl -o /DIRAC/Setup=Dirac-Production + dirac-wms-job-submit job.jdl *Command line argument specifying a CFG file* A config file can be passed to any dirac command with the ``--cfg`` flag:: diff --git a/docs/source/AdministratorGuide/Configuration/index.rst b/docs/source/AdministratorGuide/Configuration/index.rst index 607406f47a6..1e85209401e 100644 --- a/docs/source/AdministratorGuide/Configuration/index.rst +++ b/docs/source/AdministratorGuide/Configuration/index.rst @@ -5,7 +5,7 @@ DIRAC Configuration =================== The Configuration Service is providing the necessary information for the operations -of a whole DIRAC Installation (which might include several *Setups*). In this section, +of a whole DIRAC Installation. In this section, the structure of the DIRAC Configuration and its contents are described. The procedure to add new configuration data and to update the existing settings is explained. diff --git a/docs/source/AdministratorGuide/ExternalsSupport/index.rst b/docs/source/AdministratorGuide/ExternalsSupport/index.rst index efbbcef48f0..2cdf39fb09d 100644 --- a/docs/source/AdministratorGuide/ExternalsSupport/index.rst +++ b/docs/source/AdministratorGuide/ExternalsSupport/index.rst @@ -9,7 +9,7 @@ OS DIRAC needs `DIRACOS `_ for its dependencies. DIRACOS includes all dependencies (except for glibc), including Python and Grid middleware. -DIRAC *client* installation is supported for all x86_64 Linux installations using `DIRACOS `_. This includes installations made by the pilots. +DIRAC *client* installation is supported for all x86_64, aarch64, ppc64le Linux and Darwin installations using `DIRACOS `_. This includes installations made by the pilots. DIRAC *server* installation is supported for *most* x86_64 Linux installations using `DIRACOS `_. Other architectures and platforms may work as a server installation, however this is on a best effort basis and is not regularly tested. @@ -18,7 +18,6 @@ MySQL versions MySQL is a hard dependency for all DIRAC servers installations. Supported versions: -- MySQL 5.7 - MySQL 8.0 - MariaDB versions "compatible" with the above MySQL versions. @@ -39,17 +38,13 @@ generic connection details can be applied in CS location below (the shown values } -ElasticSearch versions ----------------------- +OpenSearch versions +------------------- -ElasticSearch is an optional dependency for DIRAC servers installations. Supported versions: +OpenSearch is a non-optional dependency for DIRAC servers installations. +OpenSearch server is not shipped with DIRAC. You are responsible of its administration. -- 7.x -- OpenDistro and OpenSearch releases "compatible" with the above ElasticSearch versions. - -ElasticSearch server is not shipped with DIRAC. You are responsible of its administration. - -You can run your ES cluster without authentication, or using User name and password, or using certificates. You may add the following parameters: +You can run your OpenSearch cluster without authentication, or using User name and password, or using certificates. You may add the following parameters: - ``User`` (default:``''``) - ``Password`` (default:``''``) diff --git a/docs/source/AdministratorGuide/HowTo/SystemAdministratorInterface.rst b/docs/source/AdministratorGuide/HowTo/SystemAdministratorInterface.rst index f0283c10bc0..67cb9a99d73 100644 --- a/docs/source/AdministratorGuide/HowTo/SystemAdministratorInterface.rst +++ b/docs/source/AdministratorGuide/HowTo/SystemAdministratorInterface.rst @@ -82,7 +82,6 @@ Show setup command allows administrators to know which components, Services and mardirac1.in2p3.fr >show setup {'Agents': {'Configuration': ['CE2CSAgent'], - 'Framework': ['CAUpdateAgent'], 'WorkloadManagement': ['JobHistoryAgent', 'InputDataAgent', 'StalledJobAgent', diff --git a/docs/source/AdministratorGuide/HowTo/authentication.rst b/docs/source/AdministratorGuide/HowTo/authentication.rst index bbcf764dddf..6f5d96efa7d 100644 --- a/docs/source/AdministratorGuide/HowTo/authentication.rst +++ b/docs/source/AdministratorGuide/HowTo/authentication.rst @@ -39,7 +39,7 @@ All procedure have a list of required :mod:`~DIRAC.Core.Security.Properties` and There are two main ways to define required properties: - "Hardcoded" way: Directly in the code, in your request handler you can write ```auth_yourMethodName = listOfProperties```. It can be useful for development or to provide default values. -- Via the configuration system at ```/DIRAC/Systems/(SystemName)/(InstanceName)/Services/(ServiceName)/Authorization/(methodName)```, if you have also define hardcoded properties, hardcoded properties will be ignored. +- Via the configuration system at ```/DIRAC/Systems/(SystemName)/Services/(ServiceName)/Authorization/(methodName)```, if you have also define hardcoded properties, hardcoded properties will be ignored. A complete list of properties is available in :ref:`systemAuthorization`. If you don't want to define specific properties you can use "authenticated", "any" and "all". diff --git a/docs/source/AdministratorGuide/HowTo/dedicateddfc.rst b/docs/source/AdministratorGuide/HowTo/dedicateddfc.rst index 5f43bc2dc5a..6ad460f09f5 100644 --- a/docs/source/AdministratorGuide/HowTo/dedicateddfc.rst +++ b/docs/source/AdministratorGuide/HowTo/dedicateddfc.rst @@ -14,7 +14,7 @@ Prepare the CS for the new Database ------------------------------------ First the information for the new database is added to the Configuration System. All the parameters in the -``Systems/DataManagement//Databases/NFCDB`` section can be copied from the ``FileCatalogDB`` section, except that +``Systems/DataManagement/Databases/NFCDB`` section can be copied from the ``FileCatalogDB`` section, except that the ``DBName`` has to be pointing to the soon to be created database:: DBName = NFCDB diff --git a/docs/source/AdministratorGuide/HowTo/index.rst b/docs/source/AdministratorGuide/HowTo/index.rst index 493e7590a9b..0181737f59d 100644 --- a/docs/source/AdministratorGuide/HowTo/index.rst +++ b/docs/source/AdministratorGuide/HowTo/index.rst @@ -15,3 +15,4 @@ FIXME: These sections describes things multiVO pitExport dedicateddfc + pilotsWithTokens diff --git a/docs/source/AdministratorGuide/HowTo/multiVO.rst b/docs/source/AdministratorGuide/HowTo/multiVO.rst index 20e2990ea56..fde730c42e8 100644 --- a/docs/source/AdministratorGuide/HowTo/multiVO.rst +++ b/docs/source/AdministratorGuide/HowTo/multiVO.rst @@ -206,15 +206,6 @@ Resources/StorageElements/ProductionSandboxSE } } -WorkloadManagement - PilotStatusAgent -------------------------------------- - -Option value could be different, it depends on UI -installed on server -:: - - Systems/WorkloadManagement//Agents/PilotStatusAgent/GridEnv = /etc/profile.d/grid-env - DONE diff --git a/docs/source/AdministratorGuide/HowTo/pilotsWithTokens.rst b/docs/source/AdministratorGuide/HowTo/pilotsWithTokens.rst new file mode 100644 index 00000000000..9fdaf1dbbf0 --- /dev/null +++ b/docs/source/AdministratorGuide/HowTo/pilotsWithTokens.rst @@ -0,0 +1,87 @@ +.. _pilots-with-tokens: + +===================================== +Submitting pilots to CEs using tokens +===================================== + + +This guide outlines the process of setting up DIRAC to submit pilots using access tokens obtained via a ``client_credentials`` flow from a token provider. + +Setting up an ``IdProvider`` +---------------------------- + +- Set up an OAuth2 client in the token provider and obtain a ``client_id`` and a ``client_secret``. + + .. warning:: The client credentials obtained are confidential, store them in a secure place. + Any malicious user able to get access to them would be able to generate access tokens on your behalf. + To avoid any major issue, we recommend you to only grant essential privileges to the client (``compute`` scopes). + +- Add the client credentials in the ``dirac.cfg`` of the relevant server configuration such as: + + .. code-block:: guess + + Resources + { + IdProviders + { + + { + client_id = + client_secret = + } + } + } + +- Then in your global configuration, add the following section to set up an ``IdProvider`` interface: + + .. code-block:: guess + + Resources + { + IdProviders + { + + { + issuer = + } + } + } + +- Finally, connect the OIDC provider to a specific VO by adding the following option: + + .. code-block:: guess + + Registry + { + VO + { + + { + IdProvider = + } + } + } + +.. note:: Get more details about the DIRAC configuration from the :ref:`Configuration ` section. + +Launching the ``TokenManagerHandler`` +------------------------------------- + +Run the following commands from a DIRAC client to install the ``Framework/TokenManager`` Tornado service: + +.. code-block:: console + + $ dirac-proxy-init -g dirac_admin + + $ dirac-admin-sysadmin-cli --host + + > install service Framework TokenManager + +.. note:: ``Tornado`` and then ``TokenManager`` might need to be restarted. +.. note:: Get more details about the system administrator interface from the :ref:`System Administrator Interface ` section. + +Marking computing resources and VOs as token-ready +-------------------------------------------------- + +To specify that a given VO is ready to use tokens on a given CE, add the ``Tag = Token:`` option within the CE section, and then restart the ``Site Directors``. +Once all your VOs are ready to use tokens, just specify ``Tag = Token``. diff --git a/docs/source/AdministratorGuide/Introduction/configurationbasics.rst b/docs/source/AdministratorGuide/Introduction/configurationbasics.rst index eee88d19b27..b4a74613fe0 100644 --- a/docs/source/AdministratorGuide/Introduction/configurationbasics.rst +++ b/docs/source/AdministratorGuide/Introduction/configurationbasics.rst @@ -8,17 +8,14 @@ these components are organized in *systems*, and these components can be install using the :ref:`system-admin-console`. The components don't need to be all resident on the same host, in fact it's common practice to have several hosts -for large installations. +for large installations. Puppet and Kubernetes are pupular choices for deploying DIRAC components. -Normally, services are always exposed on the same port, which is defined in the configuration for each of them. - -.. note:: in the near future the services are planned to run on one port, see :ref:`httpsTornado` and some services already run on one port, by default on 8443 +Most of DIRAC services can be exposed using either the DIPs or the HTTPs protocol. As a general rule, services can be duplicated, meaning you can have the same service running on multiple hosts, thus reducing the load. -There are only 2 cases of DIRAC services that have a "master/slave" concept, and these are the Configuration Service +There are only 2 cases of DIRAC services that have a "controller/worker" concept, and these are the Configuration Service and the Accounting/DataStore service. -The WorkloadManagement/Matcher service should also not be duplicated. Same can be said for executors: you can have many residing on different hosts. diff --git a/docs/source/AdministratorGuide/Introduction/diraccomponents.rst b/docs/source/AdministratorGuide/Introduction/diraccomponents.rst index 4583aa849bc..b20abec1b66 100644 --- a/docs/source/AdministratorGuide/Introduction/diraccomponents.rst +++ b/docs/source/AdministratorGuide/Introduction/diraccomponents.rst @@ -32,22 +32,8 @@ And then there are databases, which keep the persistent state of a *System*. They are accessed by Services and Agents as a kind of shared memory. To achieve a functional DIRAC installation, cooperation of different *Systems* is required. -A set of *Systems* providing a complete functionality to the end user form a DIRAC *Setup*. -All DIRAC client installations will point to a particular DIRAC *Setup*. *Setups* can span -multiple server installations. Each server installation belongs to a DIRAC *Instance* that can -be shared by multiple *Setups*. - -A *Setup* is the highest level of the DIRAC component hierarchy. *Setups* are combining -together instances of *Systems*. Within a given installation there may be several *Setups*. -For example, there can be "Production" *Setup* together with "Test" or "Certification" -*Setups* used for development and testing of the new functionality. An instance of a *System* -can belong to one or more *Setups*, in other words, different *Setups* can share some *System* -instances. Multiple *Setups* for the given community share the same Configuration information -which allows them to access the same computing resources. - -Each *System* and *Setup* instance has a distinct name. The mapping of *Systems* to -*Setups* is described in the Configuration of the DIRAC installation in the "/DIRAC/Setups" -section. +A set of *Systems* provide a complete functionality to the end user. +Each *System* instance has a distinct name. .. image:: ../../_static/setup_structure.png :alt: DIRAC setup structure illustration (source https://github.com/TaykYoku/DIRACIMGS/raw/main/DIRAC_Setup_structure.ai) diff --git a/docs/source/AdministratorGuide/Resources/computingelements.rst b/docs/source/AdministratorGuide/Resources/computingelements.rst index 07e7635c7cd..27850cb7cc0 100644 --- a/docs/source/AdministratorGuide/Resources/computingelements.rst +++ b/docs/source/AdministratorGuide/Resources/computingelements.rst @@ -47,7 +47,7 @@ Each computing resource is accessed using an appropriate :mod:`~DIRAC.Resources. base class. The *ComputingElements* should be properly described to be useful. The configuration -of the *ComputingElement* is located in the inside the corresponding site section in the +of the *ComputingElement* is located inside the corresponding site section in the /Resources section. An example of a site description is given below:: Resources @@ -73,12 +73,7 @@ of the *ComputingElement* is located in the inside the corresponding site sectio ce01.infn.it { # Type of the CE - CEType = CREAM - - # Submission mode should be "direct" in order to work with SiteDirector - # Otherwise the CE will be eligible for the use with third party broker, e.g. - # gLite WMS - SubmissionMode = direct + CEType = HTCondorCE # Section to describe various queue in the CE Queues @@ -100,6 +95,7 @@ This is the general structure in which specific CE descriptions are inserted. The CE configuration is part of the general DIRAC configuration It can be placed in the general Configuration Service or in the local configuration of the DIRAC installation. Examples of the configuration can be found in the :ref:`full_configuration_example`, in the *Resources/Computing* section. +You can find the options of a specific CE in the code documentation: :mod:`DIRAC.Resources.Computing`. Some CE parameters are confidential, e.g. password of the account used for the SSH tunnel access to a site. The confidential parameters @@ -116,149 +112,29 @@ processor jobs to multiprocessor queues, add the ``RequiredTag=MultiProcessor`` automatically create the equivalent single core queues, see the :mod:`~DIRAC.ConfigurationSystem.Agent.Bdii2CSAgent` configuration. +Interacting with Grid Sites +@@@@@@@@@@@@@@@@@@@@@@@@@@@ +The :mod:`~DIRAC.Resources.Computing.HTCondorCEComputingElement` and the :mod:`~DIRAC.Resources.Computing.AREXComputingElement` eases +the interactions with grid sites, by managing pilots using the underlying batch systems. +Instances of such CEs are generally setup by the site administrators. + + +Leveraging Opportunistic computing clusters +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +Sites that do not manage CEs can generally still be accessed via SSH. +The :mod:`~DIRAC.Resources.Computing.SSHComputingElement` and :mod:`~DIRAC.Resources.Computing.SSHBatchComputingElement` +can be used to submit pilots through an SSH tunnel to computing clusters with various batch systems: :mod:`~DIRAC.Resources.Computing.BatchSystems`. + + +Dealing with the Cloud resources +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +The :mod:`~DIRAC.Resources.Computing.CloudComputingElement` allows submission to cloud sites using libcloud +(via the standard SiteDirector agent). The instances are contextualised using cloud-init. + -CREAM Computing Element -@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ - -A commented example follows:: - - # Section placed in the */Resources/Sites///CEs* directory - ce01.infn.it - { - CEType = CREAM - SubmissionMode = direct - - - Queues - { - # The queue section name should be the same as in the BDII description - long - { - # Max CPU time in HEP'06 unit secs - maxCPUTime = 10000 - # Max total number of jobs in the queue - MaxTotalJobs = 5 - # Max number of waiting jobs in the queue - MaxWaitingJobs = 2 - } - } - } - -SSH Computing Element -@@@@@@@@@@@@@@@@@@@@@ - -The SSHComputingElement is used to submit pilots through an SSH tunnel to -computing clusters with various batch systems. A commented example of its -configuration follows :: - - # Section placed in the */Resources/Sites///CEs* directory - pc.farm.ch - { - CEType = SSH - # Type of the local batch system. Available batch system implementations are: - # Torque, Condor, GE, LSF, OAR, SLURM - BatchSystem = Torque - SubmissionMode = direct - SSHHost = pc.domain.ch - # SSH connection details to be defined in the local configuration - # of the corresponding SiteDirector - SSHUser = dirac_ssh - SSHPassword = XXXXXXX - # Alternatively, the private key location can be specified instead - # of the SSHPassword - SSHKey = /path/to/the/key - # SSH port if not standard one - SSHPort = 222 - # Sometimes we need an extra tunnel where the batch system is on accessible - # directly from the site gateway host - SSHTunnel = ssh pcbatch.domain.ch - # SSH type: ssh (default) or gsissh - SSHType = ssh - # Options to SSH command - SSHOptions = -o option1=something -o option2=somethingelse - # Queues section contining queue definitions - Queues - { - # The queue section name should be the same as the name of the actual batch queue - long - { - # Max CPU time in HEP'06 unit secs - maxCPUTime = 10000 - # Max total number of jobs in the queue - MaxTotalJobs = 5 - # Max number of waitin jobs in the queue - MaxWaitingJobs = 2 - # Flag to include pilot proxy in the payload sent to the batch system - BundleProxy = True - # Directory on the CE site where the pilot standard output stream will be stored - BatchOutput = /home/dirac_ssh/localsite/output - # Directory on the CE site where the pilot standard output stream will be stored - BatchError = /home/dirac_ssh/localsite/error - # Directory where the payload executable will be stored temporarily before - # submission to the batch system - ExecutableArea = /home/dirac_ssh/localsite/submission - # Extra options to be passed to the qsub job submission command - SubmitOptions = - # Flag to remove the pilot output after it was retrieved - RemoveOutput = True - } - } - } - - - -SSHBatch Computing Element -@@@@@@@@@@@@@@@@@@@@@@@@@@ - -This is an extension of the SSHComputingElement capable of submitting several jobs on one host. - -Like all SSH Computing Elements, it's defined like the following:: - - # Section placed in the */Resources/Sites///CEs* directory - pc.farm.ch - { - CEType = SSHBatch - SubmissionMode = direct - - # Parameters of the SSH conection to the site. The /2 indicates how many cores can be used on that host. - # It's equivalent to the number of jobs that can run in parallel. - SSHHost = pc.domain.ch/2 - SSHUser = dirac_ssh - # if SSH password is not given, the public key connection is assumed. - # Do not put this in the CS, put it in the local dirac.cfg of the host. - # You don't want external people to see the password. - SSHPassword = XXXXXXXXX - # If no password, specify the key path - SSHKey = /path/to/key.pub - # In case your SSH connection requires specific attributes (see below) available in late v6r10 versions (TBD). - SSHOptions = -o option1=something -o option2=somethingelse - - Queues - { - # Similar to the corresponding SSHComputingElement section - } - } - - - -.. versionadded:: > v6r10 - The SSHOptions option. - -The ``SSHOptions`` is needed when for example the user used to run the agent isn't local and requires access to afs. As the way the agents are started isn't a login, they does not -have access to afs (as they have no token), so no access to the HOME directory. Even if the HOME environment variable is replaced, ssh still looks up the original home directory. -If the ssh key and/or the known_hosts file is hosted on afs, the ssh connection is likely to fail. The solution is to pass explicitely the options to ssh with the SSHOptions option. -For example:: - - SSHOptions = -o UserKnownHostsFile=/local/path/to/known_hosts - -allows to have a local copy of the ``known_hosts`` file, independent of the HOME directory. - - - -InProcessComputingElement -@@@@@@@@@@@@@@@@@@@@@@@@@ - -The InProcessComputingElement is usually invoked by a JobAgent to execute user +Computing Elements within allocated computing resources +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +The :mod:`~DIRAC.Resources.Computing.InProcessComputingElement` is usually invoked by a Pilot-Job (JobAgent agent) to execute user jobs in the same process as the one of the JobAgent. Its configuration options are usually defined in the local configuration /Resources/Computing/CEDefaults section :: @@ -276,10 +152,8 @@ section :: } } -PoolComputingElement -@@@@@@@@@@@@@@@@@@@@ -The Pool Computing Element is used on multi-processor nodes, e.g. cloud VMs +The :mod:`~DIRAC.Resources.Computing.PoolComputingElement` is used on multi-processor nodes, e.g. cloud VMs and can execute several user payloads in parallel using an internal ProcessPool. Its configuration is also defined by pilots locally in the /Resources/Computing/CEDefaults section :: @@ -301,3 +175,37 @@ section :: } } } + +Applying cgroup2 limits to computing resources +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + +Both the :mod:`~DIRAC.Resources.Computing.InProcessComputingElement` and +:mod:`~DIRAC.Resources.Computing.SingularityComputingElement` CEs support applying Linux cgroup2 CPU and memory limits to +the slot. These will be applied if the site allows cgroup2 delegation, if this is not available execution will continue +without the limits. The limit values can be specified using the following CE parameters (all settings are optional and can +be left undefined if not needed): + +- CPULimit (float) - The number of cores that the job may use. Usage beyond this will be throttled. +- MemoryLimitMB (int) - The memory limit for the job in MB. Usage beyond this will trigger the out-of-memory killer + considering processes within the slot. +- MemoryNoSwap (bool) - If yes or true, the job will not be allowed to use swap memory. Swap memory is not included + in the main memory limit. + +Note that the memory limit should be lower than the amount requested with the submission CE in order to allow the main +pilot processes to be protected. For example if you request 4096M (e.g. via XRSL) at submission, around 150M is needed +for the pilot, so a limit of 3950M would be recommended. + +These can be specified in the CEDefaults section to apply a standardised slot size limit:: + + Resources + { + Computing + { + CEDefaults + { + CPULimit = 1.0 + MemoryLimitMB = 3950 + MemoryNoSwap = True + } + } + } diff --git a/docs/source/AdministratorGuide/Resources/country.rst b/docs/source/AdministratorGuide/Resources/country.rst new file mode 100644 index 00000000000..3de42fd5156 --- /dev/null +++ b/docs/source/AdministratorGuide/Resources/country.rst @@ -0,0 +1,14 @@ +Countries +========= + +The country definition is mostly used for overwriting some ``StorageElementGroups`` for job output upload at the country level. See :ref:`storageMapping` + + +Configuration +------------- + +.. literalinclude:: /dirac.cfg + :start-after: ##BEGIN CountriesConfiguration + :end-before: ## + :dedent: 2 + :caption: Countries configuration diff --git a/docs/source/AdministratorGuide/Resources/index.rst b/docs/source/AdministratorGuide/Resources/index.rst index a0cd6930da5..ecbf596e041 100644 --- a/docs/source/AdministratorGuide/Resources/index.rst +++ b/docs/source/AdministratorGuide/Resources/index.rst @@ -16,6 +16,8 @@ contributing with their computing and storage capacity, available as conventiona .. toctree:: :maxdepth: 2 + site + country catalog computingelements messagequeues diff --git a/docs/source/AdministratorGuide/Resources/messagequeues.rst b/docs/source/AdministratorGuide/Resources/messagequeues.rst index a97f8ae7f67..28d43181750 100644 --- a/docs/source/AdministratorGuide/Resources/messagequeues.rst +++ b/docs/source/AdministratorGuide/Resources/messagequeues.rst @@ -95,9 +95,6 @@ like described in :ref:`development_use_mq`, for example:: message = result['Value'] -In order not to spam the logs, the log output of Stomp is always silence, unless the environment variable `DIRAC_DEBUG_STOMP` is set to any value. - - Message Queue nomenclature in DIRAC ----------------------------------- diff --git a/docs/source/AdministratorGuide/Resources/proxyprovider.rst b/docs/source/AdministratorGuide/Resources/proxyprovider.rst index 1d77b53de0b..41c42957012 100644 --- a/docs/source/AdministratorGuide/Resources/proxyprovider.rst +++ b/docs/source/AdministratorGuide/Resources/proxyprovider.rst @@ -4,7 +4,7 @@ ProxyProvider ============== -This resource type provides an interface to obtain proxy certificates using a user identifier. The following proxy providers are presented here: ``DIRACCA``, ``PUSP``. When all users upload their proxies to proxy manager manually, you do not need to deploy these resources. The **/Registry/Users** section describes how to specify a proxy provifer for a user's DN. +This resource type provides an interface to obtain proxy certificates using a user identifier. The following proxy providers are presented here: ``DIRACCA``. When all users upload their proxies to proxy manager manually, you do not need to deploy these resources. The **/Registry/Users** section describes how to specify a proxy provifer for a user's DN. ---------------------- DIRACCA proxy provider @@ -32,19 +32,6 @@ The Proxy provider supports the following distinguished names, `more details her * SP,ST(stateOrProvinceName) * SERIALNUMBER(serialNumber) -------------------- -PUSP proxy provider -------------------- - -ProxyProvider implementation for a Per-User Sub-Proxy(PUSP) proxy generation using PUSP proxy server. `More details about PUSP here `_. Required parameters in the DIRAC configuration for its implementation: - -.. literalinclude:: /dirac.cfg - :start-after: ## PUSP type: - :end-before: ## - :dedent: 2 - :caption: /Resources/ProxyProviders section - - Usage ^^^^^ diff --git a/docs/source/AdministratorGuide/Resources/site.rst b/docs/source/AdministratorGuide/Resources/site.rst new file mode 100644 index 00000000000..5011a2c712d --- /dev/null +++ b/docs/source/AdministratorGuide/Resources/site.rst @@ -0,0 +1,45 @@ +.. _cs-site: + +Sites +===== + + +Site Names +---------- + +Sites have names resulting from the concatenation of: + +- Domain: Grid site name, expressed in uppercase, for example: LCG, EELA +- Site: Institution, for example: CPPM +- Country: country where the site is located, expressed in lowercase, for example fr + + +The full DIRAC Site Name becomes of the form: [Domain].[Site].[co]. +The full site names are used everywhere when the site resources are assigned to the context of a particular Domain: +in the accounting, monitoring, configuration of the Operations parameters, etc. + +Examples of valid site names are: + +* LCG.CERN.ch +* CLOUD.IN2P3.fr +* VAC.Manchester.uk +* DIRAC.farm.cern + +The [Domain] may imply a (set of) technologies used for exploiting the resources, even though this is not necessarily true. +The use of these Domains is mostly for reporting purposes, +and it is the responsibility of the administrator of the DIRAC installation to chose them +in such a way that they are meaningful for the communities and for the computing resources served by the installation. +In any case, DIRAC will always be a default Domain if nothing else is specified for a given resource. + +The Domain, Site and the country must be unique alphanumeric strings, irrespective of case, with a possible use of the following characters: "_" "-". + + + +Configuration +------------- + +.. literalinclude:: /dirac.cfg + :start-after: ##BEGIN SiteConfiguration + :end-before: ## + :dedent: 4 + :caption: Site configuration diff --git a/docs/source/AdministratorGuide/Resources/storage.rst b/docs/source/AdministratorGuide/Resources/storage.rst index 8136102adb5..5e352402a8d 100644 --- a/docs/source/AdministratorGuide/Resources/storage.rst +++ b/docs/source/AdministratorGuide/Resources/storage.rst @@ -59,6 +59,7 @@ Configuration options are: * ``SpaceReservation``: just a name of a zone of the physical storage which can have some space reserved. Extends the SRM ``SpaceToken`` concept. * ``ArchiveTimeout``: for tape SE only. If set to a value in seconds, enables the `FTS Archive Monitoring feature `_ * ``BringOnlineTimeout``: for tape SE only. If set to a value in seconds, specify the BringOnline parameter for FTS transfers. Otherwise, the default is whatever is in the ``FTS3Job`` class. +* ``WLCGTokenBasePath``: EXPERIMENTAL Path from which the token should be relative to VO specific paths ----------------- @@ -206,7 +207,6 @@ These are the plugins that you should define in the ``PluginName`` option of you - DIP: used for dips, the DIRAC custom protocol (useful for example for DIRAC SEs). - File: offers an abstraction of the local access as an SE. - RFIO (deprecated): for the rfio protocol. - - Proxy: to be used with the StorageElementProxy. - S3: for S3 (e.g. AWS, CEPH) support (see :ref:`s3_support`) @@ -268,6 +268,48 @@ The ``OccupancyPlugin`` allows to change the way space occupancy is measured. Se * WLCGAccountingHTTPJson: :py:mod:`~DIRAC.Resources.Storage.OccupancyPlugins.WLCGAccountingHTTPJson` (likely to become the default in the future) +Locally mounted filesystems +--------------------------- + +Some sites mount their storage directly on the worker nodes. In order to access these files, you can rely on the ``GFAL2_SRM2`` plugin or on the ``File`` plugin. + +With ``File`` +^^^^^^^^^^^^^ + +If the local path follows the DIRAC convention (i.e. finishes with the LFN), then you can use the ``File`` plugin. This is simply defined like that:: + + File + { + Protocol = file + Path = /mnt/lustre_3/storm_3/lhcbdisk/ + Host = localhost + Access = local + } + + +With ``SRM2`` +^^^^^^^^^^^^^ + +In case there is some mangling done somewhere, and the file path does not follow the DIRAC convention, you may need to ask the local path to SRM. +You need to define a protocol section with SRM, specifying that a ``file`` URL can be generated and that it is valid only in local:: + + + GFAL2_SRM2_LOCAL + { + PluginName = GFAL2_SRM2 + Host = storm-fe-lhcb.cr.cnaf.infn.it + Port = 8444 + Protocol = srm + Path = /disk + # This is different from the ``standard`` definition + Access = local + SpaceToken = LHCb-Disk + WSUrl = /srm/managerv2?SFN= + # This is different from the ``standard`` definition + OutputProtocols = file, https, gsiftp, root, srm + } + + .. _multiProtocol: @@ -312,9 +354,7 @@ Multi Protocol with FTS External services like FTS requires pair of URLs to perform third party copy. This is implemented using the same logic as described above. There is however an extra step: once the common protocols between 2 SEs have been filtered, an extra loop filter is done to make sure that the selected protocol can be used as read from the source and as write to the destination. Finally, the URLs which are returned are not necessarily the url of the common protocol, but are the native urls of the plugin that can accept/generate the common protocol. For example, if the common protocol is gsiftp but one of the SE has only an SRM plugin, then you will get an srm URL (which is compatible with gsiftp). - -.. versionadded:: v7r1p37 - The FTS3Agent can now use plugins to influence the list of TPC protocols used. See :ref:`fts3` +The FTS3Agent can use plugins to influence the list of TPC protocols used. See :ref:`fts3` MultiHop support @@ -336,8 +376,17 @@ Up to recently, any protocol that was defined as ``AccessProtocols`` was also us This is not true for `CTA `_ . Because ``CTA`` can stage with xroot only, but we may need to use another protocol to transfer to a another site, we need to distinguish between staging and accessing. To the best of my knowledge, only ``CTA`` is like this, and thus, it is the only place where you may need to define ``StageProtocols``. In case of FTS transfer from CTA where the stage and transfer protocols are different, we rely on the multihop mechanism of FTS to do the protocol translations. More technical details are available in :py:mod:`DIRAC.DataManagementSystem.Client.FTS3Job` --------------------- + StorageElementGroups -------------------- StorageElements can be grouped together in a ``StorageElementGroup``. This allows the systems or the users to refer to ``any storage within this group``. + + + +.. _storageMapping: + +Mapping Storages to Sites and Countries +--------------------------------------- + +Both ``Sites`` and ``Countries`` can have ``StorageElement`` (discouraged) or ``StorageElementGroup`` associated. This shows particularly useful if we want to restrict the job output upload to specific locations, due to network constraints for example. This is done using the ``AssociatedSEs`` parameter of the ``Site`` or ``Country``. The resolution order and logic is explained in :py:func:`~DIRAC.DataManagementSystem.Utilities.ResolveSE.getDestinationSEList` and well illustrated with examples in the `associated tests `_ diff --git a/docs/source/AdministratorGuide/Resources/supercomputers.rst b/docs/source/AdministratorGuide/Resources/supercomputers.rst index f52913e990b..a3edb237275 100644 --- a/docs/source/AdministratorGuide/Resources/supercomputers.rst +++ b/docs/source/AdministratorGuide/Resources/supercomputers.rst @@ -108,11 +108,14 @@ This case has not been addressed yet. No outbound connectivity ------------------------ +Submission management +--------------------- + Solutions seen in the previous section cannot work in an environment without external connectivity. The well-known Pilot-Job paradigm on which the DIRAC WMS is based does not apply in these circumstances: the Pilot-Jobs cannot fetch jobs from DIRAC. Thus, such supercomputers require slightly changes in the WMS: we reintroduced the push model. -To leverage the Push model, one has to add the :mod:`~DIRAC.WorkloadManagementSystem.Agent.PushJobAgent` to the ``Systems/WorkloadManagement//Agents`` CS section, such as:: +To leverage the Push model, one has to add the :mod:`~DIRAC.WorkloadManagementSystem.Agent.PushJobAgent` to the ``Systems/WorkloadManagement/Agents`` CS section, such as:: Systems PushJobAgent_ @@ -126,6 +129,12 @@ To leverage the Push model, one has to add the :mod:`~DIRAC.WorkloadManagementSy # Control the number of jobs handled on the machine MaxJobsToSubmit = 100 Module = PushJobAgent + # SubmissionPolicy can be "Application" or "JobWrapper" + # - Application (soon deprecated): the agent will submit a workflow to a PoolCE, the workflow is responsible for interacting with the remote site + # - JobWrapper (default): the agent will submit a JobWrapper directly to the remote site, it is responsible of the remote execution + SubmissionPolicy = + # The CVMFS location to be used for the job execution on the remote site + CVMFSLocation = "/cvmfs/dirac.egi.eu/dirac/pro" } One has also to authorize the machine hosting the :mod:`~DIRAC.WorkloadManagementSystem.Agent.PushJobAgent` to process jobs via the ``Registry/Hosts/`` CS section:: @@ -133,7 +142,7 @@ One has also to authorize the machine hosting the :mod:`~DIRAC.WorkloadManagemen Properties += GenericPilot Properties += FileCatalogManagement -One has to specify the concerned VO in the targeted CEs, such as:: +One has to specify the concerned VO, the platform and the CPU Power in the targeted CEs, such as:: { @@ -141,29 +150,71 @@ One has to specify the concerned VO in the targeted CEs, such as:: VO = # Required because we are on a host (not on a worker node) VirtualOrganization = + # To match compatible jobs + Platform = + Queues + { + + { + CPUNormalizationFactor = + } + } + } Finally, one has to make sure that job scheduling parameters are correctly fine-tuned. Further details in the :ref:`JobScheduling section `. -:mod:`~DIRAC.WorkloadManagementSystem.Agent.PushJobAgent` inherits from :mod:`~DIRAC.WorkloadManagementSystem.Agent.JobAgent` and proposes a similar structure: it fetches a job from the :mod:`~DIRAC.WorkloadManagementSystem.Service.MatcherHandler` service and submit it to a :mod:`~DIRAC.Resources.Computing.PoolComputingElement`. - - It provides an additional parameter in ``/LocalSite`` named ``RemoteExecution`` that can be used later in the process to identify computing resources with no external connectivity. - - There is no ``timeLeft`` attribute: it runs on the DIRAC side as an ``Agent``. - - ``MaxJobsToSubmit`` corresponds to the maximum number of jobs the agent can handle at the same time. - - To fetch a job, :mod:`~DIRAC.WorkloadManagementSystem.Agent.PushJobAgent` sends the dictionary of the target CE to the :mod:`~DIRAC.WorkloadManagementSystem.Service.MatcherHandler` service. +The :mod:`~DIRAC.WorkloadManagementSystem.Agent.PushJobAgent` class extends the functionality of the :mod:`~DIRAC.WorkloadManagementSystem.Agent.JobAgent` and operates specifically on a VO box. It follows a similar architecture by retrieving jobs from the :mod:`~DIRAC.WorkloadManagementSystem.Service.MatcherHandler` service and submitting them to a :mod:`~DIRAC.Resources.Computing.ComputingElement`. Although `PushJobAgent` does not inherit directly from the :mod:`~DIRAC.WorkloadManagementSystem.Agent.SiteDirector`, it incorporates several comparable features, including: + +- Supervising specific Sites, Computing Elements (CEs), or Queues. +- Placing problematic queues on hold and retrying after a pre-defined number of cycles. + +The :mod:`~DIRAC.WorkloadManagementSystem.Agent.PushJobAgent` supports two distinct submission modes: **JobWrapper** and **Application**. + +JobWrapper Mode (Default and Recommended) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The **JobWrapper** mode is the default and recommended submission method, due to its reliability and efficiency. The workflow for this mode includes: -:mod:`~DIRAC.WorkloadManagementSystem.Agent.PushJobAgent` does not inherit from :mod:`~DIRAC.WorkloadManagementSystem.Agent.SiteDirector` but embeds similar features: - - It supervises specific Sites/CEs/Queues. - - If there is an error with a queue, it puts it on hold and waits for a certain number of cycles before trying again. +1. **Job Retrieval**: Fetch a job from the :mod:`~DIRAC.WorkloadManagementSystem.Service.MatcherHandler` service. +2. **Pre-processing**: Pre-process the job by fetching the input sandbox and any necessary data. +3. **Template Generation**: Create a :mod:`~DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapperOfflineTemplate` designed to execute the job’s payload. +4. **Submission**: Submit the generated `JobWrapperOfflineTemplate` along with the inputs to the target Computing Element (CE). +5. **Monitoring**: Continuously monitor the status of the submitted jobs until they are completed. +6. **Output Retrieval**: Retrieve the outputs of the finished jobs from the target CE. +7. **Post-processing**: Conduct any necessary post-processing of the outputs. -Internally, the workflow modules originally in charge of executing the script/application (:mod:`~DIRAC.Workflow.Modules.Script`) check whether the workload should be -sent to a remote location before acting. -:mod:`~DIRAC.WorkloadManagementSystem.Utilities.RemoteRunner` attempts to extract the value from the environment variable initialized by the :mod:`~DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapper`. -If the variable is not set, then the application is run locally via ``systemCall()``, else the application is submitted to a remote Computing Element such as ARC. -:mod:`~DIRAC.WorkloadManagementSystem.Utilities.RemoteRunner` wraps the script/application command in an executable, gets all the files of the working directory that correspond to input files and submits the executable along with the input files. It gets the status of the application submitted every 2 minutes until it is finished and finally gets the outputs. +Certainly! Here's an enhanced version of the reStructuredText (reST) content: -What if the :mod:`~DIRAC.WorkloadManagementSystem.Agent.PushJobAgent` is suddenly stopped while processing jobs? Jobs would be declared as ``Stalled``. Administrators would have to manually clean up input directories (by default, they should be located in ``/opt/dirac/runit/WorkloadManagement/PushJobAgent/``). -Administrators may also have to kill processes related to the execution of the jobs: ``dirac-jobexec``. +.. warning:: The `JobWrapper` mode assumes that the job can execute without external connectivity. As an administrator, if any step of your job workflow requires external connectivity, it is crucial to review and adjust your logic accordingly. The :mod:`~DIRAC.WorkloadManagementSystem.JobWrapper.JobExecutionCoordinator` can assist in this process. It enables you to define custom pre-processing and post-processing logic based on specific job and CE attributes. For more detailed information, refer to the :mod:`~DIRAC.WorkloadManagementSystem.JobWrapper.JobExecutionCoordinator` documentation. +.. image:: ../../_static/pja_jobwrapper_submission.png + :alt: PushJobAgent JobWrapper submission + :align: center + +Application Mode (Deprecated) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The **Application** mode is deprecated and slated for removal in future versions. It is considered less reliable due to higher memory consumption and sensitivity to CE-related issues, which restrict the number of jobs that can be processed concurrently. The workflow for this mode is as follows: + +1. **Job Retrieval**: Fetch a job from the :mod:`~DIRAC.WorkloadManagementSystem.Service.MatcherHandler` service using the target CE attributes. +2. **Template Generation**: Generate a :mod:`~DIRAC.WorkloadManagementSystem.JobWrapper.JobWrapperTemplate`. +3. **Submission**: Submit the generated `JobWrapperTemplate` to a :mod:`~DIRAC.Resources.Computing.PoolComputingElement`. + - The submission includes an additional parameter in ``/LocalSite`` named ``RemoteExecution``, used to identify computing resources lacking external connectivity. + - The ``MaxJobsToSubmit`` setting defines the maximum number of jobs the agent can handle simultaneously. +4. **Execution**: The :mod:`~DIRAC.Resources.Computing.PoolComputingElement` executes the `JobWrapperTemplate` in a new process. +5. **Script Execution**: Within this context, the `JobWrapperTemplate` can only execute the :mod:`~DIRAC.WorkloadManagementSystem.scripts.dirac_jobexec` script in a new process. +6. **Workflow Module Processing**: Workflow modules responsible for script or application execution (:mod:`~DIRAC.Workflow.Modules.Script`) determine whether the payload needs to be offloaded to a remote location. +7. **Remote Execution**: :mod:`~DIRAC.WorkloadManagementSystem.Utilities.RemoteRunner` checks for the environment variable initialized by the `JobWrapper`. + - If the variable is unset, the application runs locally via ``systemCall()``; otherwise, it is submitted to a remote CE such as ARC. + - `RemoteRunner` wraps the script or application command in an executable, gathers input files from the working directory, and submits these along with the executable to the remote CE. + - The status of the submitted application is monitored every 2 minutes until completion, after which the outputs are retrieved. + +.. warning:: If the `PushJobAgent` is interrupted while processing jobs, administrators must manually clean up input directories (usually located at ``/opt/dirac/runit/WorkloadManagement/PushJobAgent/``) and terminate any associated processes (e.g., ``dirac-jobexec``). + +.. image:: ../../_static/pja_application_submission.png + :alt: PushJobAgent Application submission + :align: center Multi-core/node allocations --------------------------- diff --git a/docs/source/AdministratorGuide/ServerInstallations/HTTPSServices.rst b/docs/source/AdministratorGuide/ServerInstallations/HTTPSServices.rst new file mode 100644 index 00000000000..8ffce0b2f1b --- /dev/null +++ b/docs/source/AdministratorGuide/ServerInstallations/HTTPSServices.rst @@ -0,0 +1,106 @@ +.. _https_services: + +=============================== +Note on HTTPs services in DIRAC +=============================== + +.. contents:: + +Background +********** + +For many years, DIRAC services have been exposed through the historical DISET (``dips://``) protocol. +Few years ago DIRAC developers started exposing DIRAC services using the HTTPs protocol. +This page explains how to migrate existing DIPs protocol to HTTPs. For a developer view, refer to :ref:`httpsTornado`. + +For a summary presentation you can check `this `_. + +NB: not all DIPs services can be exposed through HTTPs. For a comprehensve list, please refer to :ref:`scalingLimitations`. + +General principle +================= + +Contrary to the DISET protocol where each service would have its own process and port opened, HTTPs services are served by a unique process and port, based on Tornado. + +There are exceptions to that, due to the use of global variables in some parts of DIRAC. Namely: + +* Master CS + +All other services can follow the standard procedure described below. + +First, the following configuration subsections have to be added to CS:: + + + # Add Tornado to Systems section + Systems + { + ... + Tornado + { + Production + { + Port = 443 + } + } + } + + +Installation of an HTTPs based service +====================================== + +Just run ``dirac-install-component`` with the service you are interested in, for example +``dirac-install-component WorkloadManagement/JobMonitoring``. This will install an ``runit`` component running ``tornado-start-all``. + +Alternatively, use ``dirac-admin-sysadmi-cli``. + +Case: you are already running the equivalent DISET service +----------------------------------------------------------- + +You can (before or after installing the tornado-based service) fully remove the DISET version of the service with ``dirac-uninstall-component DataManagement JobMonitoring`` + +Example of configuration before/after: + +.. literalinclude:: /../../src/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg + :start-after: ##BEGIN JobMonitoring + :end-before: ##END + :dedent: 2 + :caption: JobMonitoring configuration for DISET + +.. literalinclude:: /../../src/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg + :start-after: ##BEGIN TornadoJobMonitoring + :end-before: ##END + :dedent: 2 + :caption: JobMonitoring configuration for HTTPs + +In any case, do not forget to update the URL of the service you just installed, such that other services can reach it. + + +Adding more tornado instances on a different machine +==================================================== + +Simply use ``dirac-install-component`` with no arguments on the new machine. + + +MasterCS special case +===================== + +The master Configuration Server is different and needs to be run in a separate process. In order to do so: + +* Do NOT specify ``Protocol=https`` in the service description, otherwise it will be ran with all the other Tornado services. +* If you run on the same machine as other TornadoService, specify a ``Port`` in the service description (you can keep the existing 9135, if already there). +* Modify the content of the Configuration so that URLs are updated to use HTTPs instead of DIPs. +* Add `HandlerPath = DIRAC/ConfigurationSystem/Service/TornadoConfigurationHandler.py` in the etc/dirac.cfg configuration file of the machine where the master CS is running (needed for bootstrap). + +Finally, there is no automatic installations script. So just install a CS as you normally would do, and then edit the ``run`` file like that:: + + #!/bin/bash + rcfile=/opt/dirac/bashrc + [ -e $rcfile ] && source $rcfile + # + export DIRAC_USE_TORNADO_IOLOOP=Yes + exec 2>&1 + # + [ "service" = "agent" ] && renice 20 -p $$ + # + # + exec tornado-start-CS -ddd diff --git a/docs/source/AdministratorGuide/ServerInstallations/InstallingDiracServer.rst b/docs/source/AdministratorGuide/ServerInstallations/InstallingDiracServer.rst index 2622b1e2b54..9264fea1006 100644 --- a/docs/source/AdministratorGuide/ServerInstallations/InstallingDiracServer.rst +++ b/docs/source/AdministratorGuide/ServerInstallations/InstallingDiracServer.rst @@ -74,8 +74,8 @@ Requirements - Grid host certificates in pem format; - At least one of the servers of the installation must have updated CAs and CRLs files; if you want to install - the standard Grid CAs you can follow the instructions at https://wiki.egi.eu/wiki/EGI_IGTF_Release. They - are usally installed /etc/grid-security/certificates. You may also need to install the ``fetch-crl`` package, + the standard Grid CAs you can follow the instructions at https://docs.egi.eu/providers/operations-manuals/howto01_using_igtf_ca_distribution/. + They are usally installed /etc/grid-security/certificates. You may also need to install the ``fetch-crl`` package, and run the ``fetch-crl`` command once installed. - If gLite third party services are needed (for example, for the pilot job submission via WMS or for data transfer using FTS) gLite User Interface must be installed and the environment set up @@ -251,14 +251,12 @@ be taken based on the Python version you wish to install. # i.e., to produce the initial dirac.cfg for the server # # Give a Name to your User Community, it does not need to be the same name as in EGI, - # it can be used to cover more than one VO in the grid sense + # it can be used to cover more than one VO in the grid sense. + # If you are going to setup DIRAC as a multi-VO instance, remove the VirtualOrganization parameter. VirtualOrganization = Name of your VO # Site name SiteName = DIRAC.HostName.ch - # Setup name (every installation can have multiple setups, but give a name to the first one) - Setup = MyDIRAC-Production - # Default name of system instances - InstanceName = Production + # Flag to skip download of CAs, on the first Server of your installation you need to get CAs # installed by some external means SkipCADownload = yes @@ -305,6 +303,7 @@ be taken based on the Python version you wish to install. Systems += StorageManagement Systems += Transformation Systems += WorkloadManagement + Systems += Tornado # # List of DataBases to be installed (what's here is a list for a basic installation) Databases = InstalledComponentsDB @@ -318,9 +317,9 @@ be taken based on the Python version you wish to install. # Host = dirac.cern.ch # List of Services to be installed (what's here is a list for a basic installation) Services = Configuration/Server - Services += Framework/ComponentMonitoring + Services += Framework/TornadoComponentMonitoring Services += Framework/SystemAdministrator - Services += ResourceStatus/ResourceStatus + Services += ResourceStatus/TornadoResourceStatus # Flag determining whether the Web Portal will be installed WebPortal = yes # @@ -364,11 +363,10 @@ of the status of running DIRAC services, e.g.:: Name : Runit Uptime PID Configuration_Server : Run 41 30268 Framework_SystemAdministrator : Run 21 30339 - Framework_ComponentMonitoring : Run 11 30340 - ResourceStatus_ResourceStatus : Run 9 30341 + Tornado_Tornado : Run 11 30340 -Now the basic services - Configuration, SystemAdministrator, ComponentMonitoring and ResourceStatus - are installed, +Now the basic services - Configuration, SystemAdministrator, TornadoComponentMonitoring and TornadoResourceStatus - are installed, or at least their DBs should be installed, and their services up and running. There are anyway a couple more steps that should be done to fully activate the ComponentMonitoring and the ResourceStatus. @@ -427,22 +425,18 @@ operation is the registration of the new host in the already functional Configur VirtualOrganization = Name of your VO # Site name SiteName = DIRAC.HostName2.ch - # Setup name - Setup = MyDIRAC-Production - # Default name of system instances - InstanceName = Production # Flag to use the server certificates UseServerCertificate = yes # Configuration Server URL (This should point to the URL of at least one valid Configuration # Service in your installation, for the primary server it should not used) - ConfigurationServer = dips://myprimaryserver.name:9135/Configuration/Server - ConfigurationServer += dips://localhost:9135/Configuration/Server + ConfigurationServer = https://myprimaryserver.name:9135/Configuration/Server + ConfigurationServer += https://localhost:8443/Tornado/Tornado # Configuration Name ConfigurationName = MyConfiguration # # These options define the DIRAC components being installed on "this" DIRAC server. - # The simplest option is to install a slave of the Configuration Server and a + # The simplest option is to install a worker of the Configuration Server and a # SystemAdministrator for remote management. # # The following options defined components to be installed @@ -510,7 +504,7 @@ To install the DIRAC Client, follow the procedure described in the User Guide. - Install services and agents, for example:: - $ install service WorkloadManagement JobMonitoring + $ install service WorkloadManagement TornadoJobMonitoring $ install agent Configuration Bdii2CSAgent Note that all the necessary commands above can be collected in a text file and the whole installation can be diff --git a/docs/source/AdministratorGuide/ServerInstallations/InstallingWebAppDIRAC.rst b/docs/source/AdministratorGuide/ServerInstallations/InstallingWebAppDIRAC.rst index da43cf92f6a..cd0ecbfea61 100644 --- a/docs/source/AdministratorGuide/ServerInstallations/InstallingWebAppDIRAC.rst +++ b/docs/source/AdministratorGuide/ServerInstallations/InstallingWebAppDIRAC.rst @@ -52,7 +52,6 @@ Installation configuration:: Services = Framework/SystemAdministrator UseServerCertificate = yes SkipCADownload = yes - Setup = your setup # for example: LHCb-Certification ConfigurationMaster = no ConfigurationServer = your configuration service } @@ -111,7 +110,6 @@ Make sure that the configuration /opt/dirac/pro/etc/dirac.cfg file is correct. I DIRAC { - Setup = LHCb-Certification Configuration { Servers = @@ -120,14 +118,6 @@ Make sure that the configuration /opt/dirac/pro/etc/dirac.cfg file is correct. I { } Extensions = WebApp - Setups - { - LHCb-Certification - { - Configuration = LHCb-Certification - Framework = LHCb-Certification - } - } } Update using :ref:`dirac-admin-sysadmin-cli `. @@ -164,7 +154,6 @@ Section has the following structure:: Applications { Accounting = DIRAC.Accounting - Activity Monitor = DIRAC.ActivityMonitor Component History = DIRAC.ComponentHistory Configuration Manager = DIRAC.ConfigurationManager Downtimes = DIRAC.Downtimes @@ -282,7 +271,7 @@ Make sure there is a line 'include /etc/nginx/conf.d/\*.conf;', then create a si listen [::]:80 default_server; # Your server name if you have weird network config. Otherwise leave commented #server_name your.server.domain; - return 301 https://$server_name$request_uri; + return 301 https://$host$request_uri; } server { @@ -394,7 +383,7 @@ Make sure there is a line 'include /etc/nginx/conf.d/\*.conf;', then create a si } location / { - rewrite ^ https://$server_name/DIRAC/ permanent; + rewrite ^ https://$host/DIRAC/ permanent; } } diff --git a/docs/source/AdministratorGuide/ServerInstallations/centralizedLogging.rst b/docs/source/AdministratorGuide/ServerInstallations/centralizedLogging.rst index a2233aa64d6..331fbff38e7 100644 --- a/docs/source/AdministratorGuide/ServerInstallations/centralizedLogging.rst +++ b/docs/source/AdministratorGuide/ServerInstallations/centralizedLogging.rst @@ -62,7 +62,7 @@ Logstash and ELK configurations The suggested logstash configuration (``/etc/logstash/conf.d/configname``) can be found in https://gitlab.cern.ch/ai/it-puppet-module-dirac/-/blob/qa/code/templates/logstash.conf.erb (check the `full documentation `_) -The ElasticSearch template ``lhcb-dirac-logs_default`` looks like:: +The OpenSearch template ``lhcb-dirac-logs_default`` looks like:: { "order": 1, @@ -159,4 +159,4 @@ The ElasticSearch template ``lhcb-dirac-logs_default`` looks like:: Kibana dashboard ================ -A dashboard for the logs can be found `here `_ in both json and ndjson format, as ES services are dropping support for .json imports in newer versions. +A dashboard for the logs can be found `here `_ in both json and ndjson format, as ES services are dropping support for .json imports in newer versions. diff --git a/docs/source/AdministratorGuide/ServerInstallations/environment_variable_configuration.rst b/docs/source/AdministratorGuide/ServerInstallations/environment_variable_configuration.rst index 1e17e11e874..2576f7e5216 100644 --- a/docs/source/AdministratorGuide/ServerInstallations/environment_variable_configuration.rst +++ b/docs/source/AdministratorGuide/ServerInstallations/environment_variable_configuration.rst @@ -14,13 +14,14 @@ DIRAC_DEBUG_DENCODE_CALLSTACK DIRAC_DEBUG_M2CRYPTO If ``true`` or ``yes``, print a lot of SSL debug output -DIRAC_DEBUG_STOMP - If set, the stomp library will print out debug information - DIRAC_DEPRECATED_FAIL If set, the use of functions or objects that are marked ``@deprecated`` will fail. Useful for example in continuous integration tests against future versions of DIRAC +DIRAC_DISABLE_GCONFIG_REFRESH + If set, attempting to start the ``gConfig`` refresh thread will result in an exception. + This is used by DiracX to accidental use of vanilla DIRAC in contexts where it won't work. + DIRAC_FEWER_CFG_LOCKS If ``true`` or ``yes`` or ``on`` or ``1`` or ``y`` or ``t``, DIRAC will reduce the number of locks used when accessing the CS for better performance (default, ``no``). @@ -56,11 +57,8 @@ DIRAC_MYSQL_OPTIMIZER_TRACES_PATH DIRAC_NO_CFG If set to anything, cfg files on the command line must be passed to the command using the --cfg option. -DIRAC_USE_JSON_DECODE - Controls the transition to JSON serialization. See the information in :ref:`jsonSerialization` page (default=Yes since v7r2) - DIRAC_USE_JSON_ENCODE - Controls the transition to JSON serialization. See the information in :ref:`jsonSerialization` page (default=No) + Controls the transition to JSON serialization (default=Yes since 9.0) DIRAC_ROOT_PATH If set, overwrites the value of DIRAC.rootPath. diff --git a/docs/source/AdministratorGuide/ServerInstallations/index.rst b/docs/source/AdministratorGuide/ServerInstallations/index.rst index f7a3977bf9e..54adbd51d74 100644 --- a/docs/source/AdministratorGuide/ServerInstallations/index.rst +++ b/docs/source/AdministratorGuide/ServerInstallations/index.rst @@ -4,7 +4,7 @@ Server and Infrastructure Installations ======================================= -This sections constains the documentation for installing new DIRAC Servers or services needed outside of DIRAC +This sections contains the documentation for installing new DIRAC Servers or services needed outside of DIRAC .. toctree:: @@ -12,7 +12,8 @@ This sections constains the documentation for installing new DIRAC Servers or se InstallingDiracServer InstallingWebAppDIRAC + HTTPSServices centralizedLogging - rabbitmq + tornadoComponentsLogs scalingAndLimitations environment_variable_configuration diff --git a/docs/source/AdministratorGuide/ServerInstallations/rabbitmq.rst b/docs/source/AdministratorGuide/ServerInstallations/rabbitmq.rst deleted file mode 100644 index cd50e2a0860..00000000000 --- a/docs/source/AdministratorGuide/ServerInstallations/rabbitmq.rst +++ /dev/null @@ -1,27 +0,0 @@ -======== -RabbitMQ -======== - - -RabbitMQ Administration Tools ------------------------------- - -RabbitMQ uses a two-step access-control(https://www.rabbitmq.com/access-control.html). Apart -from the standard user/password (or ssl-based) authentication, RabbitMQ has an internal database -with the list of users and permissions settings. -DIRAC provides an interface to the internal RabbitMQ user database via the RabbitMQAdmin class. -Internally it uses rabbitmqctl command (https://www.rabbitmq.com/man/rabbitmqctl.1.man.html) -Only the user with the granted permissions can execute those commands. -The interface provides methods for adding or removing users, setting the permission etc. -The interface do not provide the possibilty to e.g. create or destroy queues, because according -to the AMPQ and general RabbitMQ philosophy those operations should be done by consumers/producer -with given permissions. - - -Synchronization of RabbitMQ user database ------------------------------------------ - -The synchronization between the DIRAC Configuration System and the RabbitMQ internal -database is assured by RabbitMQSynchronizer. -It checks the current list of users and hosts which are allowed to send messages to -RabbitMQ and updates the internal RabbitMQ database accordingly. diff --git a/docs/source/AdministratorGuide/ServerInstallations/scalingAndLimitations.rst b/docs/source/AdministratorGuide/ServerInstallations/scalingAndLimitations.rst index 5820e8549b8..7916815bf9d 100644 --- a/docs/source/AdministratorGuide/ServerInstallations/scalingAndLimitations.rst +++ b/docs/source/AdministratorGuide/ServerInstallations/scalingAndLimitations.rst @@ -52,6 +52,25 @@ When you servers are heavily loaded, you may want to tune some kernel parameters net.core.somaxconn net.core.netdev_max_backlog +You can also adjust the limit of opened files descriptors in the ``Service`` section of the ``/usr/lib/systemd/system/runsvdir-start.service`` file:: + + LimitNOFILE=500000 + + +Databases +========= + +Every now and then, it is interesting to look at the fragmentation status of your database. This is done by using the ``analyze table`` statement (https://dev.mysql.com/doc/refman/8.4/en/analyze-table.html) possibly followed by the ``optimize table`` statement (https://dev.mysql.com/doc/refman/8.4/en/optimize-table.html). + +To know whether your tables are fragmented:: + + select table_schema,table_name, sys.format_bytes(data_length) table_size, sys.format_bytes(data_free) empty_space from information_schema.tables where data_length >= (1024*1024*1024) order by data_length desc; + + +The fragmented space should be very small with respect to the overall table size. + + + Duplications ============ @@ -69,53 +88,45 @@ Services ======== +--------------------+---------------------------------------------------------------------------------------------------+-------------+---------------------------------------------------------------------------+-----------+ -| **System** | **Component** |**Duplicate**| **Remarque** | **HTTPs** + +| **System** | **Component** |**Duplicate**| **Remarks** | **HTTPs** + +--------------------+---------------------------------------------------------------------------------------------------+-------------+---------------------------------------------------------------------------+-----------+ -| Accounting | :mod:`DataStore ` | PARTIAL | One master and helpers (See :ref:`datastorehelpers`) | + +| Accounting | :mod:`DataStore ` | PARTIAL | One controller and helpers (See :ref:`datastorehelpers`) | + + +---------------------------------------------------------------------------------------------------+-------------+---------------------------------------------------------------------------+-----------+ | | :mod:`ReportGenerator ` | | | + +--------------------+---------------------------------------------------------------------------------------------------+-------------+---------------------------------------------------------------------------+-----------+ -| Configuration | :mod:`Configuration ` | PARTIAL | One master (rw) and slaves (ro). It's advised to have several CS slaves | YES + +| Configuration | :mod:`Configuration ` | PARTIAL | One controller (rw) and workers (ro). Should have several CS workers | YES + +--------------------+---------------------------------------------------------------------------------------------------+-------------+---------------------------------------------------------------------------+-----------+ | DataManagement | :mod:`DataIntegrity ` | YES | | YES + + +---------------------------------------------------------------------------------------------------+-------------+---------------------------------------------------------------------------+-----------+ | | :mod:`FileCatalog ` | YES | | YES + + +---------------------------------------------------------------------------------------------------+-------------+---------------------------------------------------------------------------+-----------+ -| | :mod:`FileCatalogProxy ` | | | + -+ +---------------------------------------------------------------------------------------------------+-------------+---------------------------------------------------------------------------+-----------+ | | :mod:`FTS3Manager ` | YES | | YES + + +---------------------------------------------------------------------------------------------------+-------------+---------------------------------------------------------------------------+-----------+ -| | :mod:`S3Gateway ` | YES | | + +| | :mod:`S3Gateway ` | YES | | YES + + +---------------------------------------------------------------------------------------------------+-------------+---------------------------------------------------------------------------+-----------+ | | :mod:`StorageElement ` | | | + -+ +---------------------------------------------------------------------------------------------------+-------------+---------------------------------------------------------------------------+-----------+ -| | :mod:`StorageElementProxy ` | | | + +--------------------+---------------------------------------------------------------------------------------------------+-------------+---------------------------------------------------------------------------+-----------+ | Framework | :mod:`BundleDelivery ` | YES | | YES + + +---------------------------------------------------------------------------------------------------+-------------+---------------------------------------------------------------------------+-----------+ -| | :mod:`ComponentMonitoring ` | YES | | + -+ +---------------------------------------------------------------------------------------------------+-------------+---------------------------------------------------------------------------+-----------+ -| | :mod:`Monitoring ` | NO | | + -+ +---------------------------------------------------------------------------------------------------+-------------+---------------------------------------------------------------------------+-----------+ -| | :mod:`Notification ` | | | + -+ +---------------------------------------------------------------------------------------------------+-------------+---------------------------------------------------------------------------+-----------+ -| | :mod:`Plotting ` | | | + +| | :mod:`ComponentMonitoring ` | YES | | YES + + +---------------------------------------------------------------------------------------------------+-------------+---------------------------------------------------------------------------+-----------+ -| | :mod:`ProxyManager ` | YES | | + +| | :mod:`Notification ` | YES | | YES + + +---------------------------------------------------------------------------------------------------+-------------+---------------------------------------------------------------------------+-----------+ -| | :mod:`RabbitMQSync ` | | | + +| | :mod:`ProxyManager ` | YES | | YES + + +---------------------------------------------------------------------------------------------------+-------------+---------------------------------------------------------------------------+-----------+ -| | :mod:`SecurityLogging ` | YES | | + +| | :mod:`SecurityLogging ` | **NO** | | + + +---------------------------------------------------------------------------------------------------+-------------+---------------------------------------------------------------------------+-----------+ | | :mod:`SystemAdministrator ` | **MUST** | There should be one on each and every machine | + + +---------------------------------------------------------------------------------------------------+-------------+---------------------------------------------------------------------------+-----------+ -| | :mod:`UserProfileManager ` | | | + +| | :mod:`TokenManager ` | YES | | YES(only) + ++ +---------------------------------------------------------------------------------------------------+-------------+---------------------------------------------------------------------------+-----------+ +| | :mod:`UserProfileManager ` | YES | | YES + +--------------------+---------------------------------------------------------------------------------------------------+-------------+---------------------------------------------------------------------------+-----------+ -| Monitoring | :mod:`Monitoring ` | YES | | + +| Monitoring | :mod:`Monitoring ` | YES | | YES + +--------------------+---------------------------------------------------------------------------------------------------+-------------+---------------------------------------------------------------------------+-----------+ -| RequestManagement | :mod:`ReqManager ` | YES | | + +| RequestManagement | :mod:`ReqManager ` | YES | | YES + + +---------------------------------------------------------------------------------------------------+-------------+---------------------------------------------------------------------------+-----------+ -| | :mod:`ReqProxy ` | YES | | + +| | :mod:`ReqProxy ` | PARTIAL | Relies on local storage | + +--------------------+---------------------------------------------------------------------------------------------------+-------------+---------------------------------------------------------------------------+-----------+ | ResourcesStatus | :mod:`Publisher ` | YES | | YES + + +---------------------------------------------------------------------------------------------------+-------------+---------------------------------------------------------------------------+-----------+ @@ -135,7 +146,7 @@ Services + +---------------------------------------------------------------------------------------------------+-------------+---------------------------------------------------------------------------+-----------+ | | :mod:`Matcher ` | **NO** | | + + +---------------------------------------------------------------------------------------------------+-------------+---------------------------------------------------------------------------+-----------+ -| | :mod:`OptimizationMind ` | | | + +| | :mod:`OptimizationMind ` | **NO** | | + + +---------------------------------------------------------------------------------------------------+-------------+---------------------------------------------------------------------------+-----------+ | | :mod:`PilotManager ` | PARTIAL | In case there are HTCondor CEs to deal with, the HTCondor | + | | | | WorkingDirectory should exist and be accessible in each and every machine | + @@ -150,7 +161,7 @@ Agents ====== +--------------------+---------------------------------------------------------------------------------------------------+---------------+-----------------------------------------------------------------------------------+ -| **System** | **Component** | **Duplicate** | **Remarque** | +| **System** | **Component** | **Duplicate** | **Remarks** | +--------------------+---------------------------------------------------------------------------------------------------+---------------+-----------------------------------------------------------------------------------+ | Accounting | :mod:`~DIRAC.AccountingSystem.Agent.NetworkAgent` | | | +--------------------+---------------------------------------------------------------------------------------------------+---------------+-----------------------------------------------------------------------------------+ @@ -162,13 +173,11 @@ Agents +--------------------+---------------------------------------------------------------------------------------------------+---------------+-----------------------------------------------------------------------------------+ | DataManagement | :mod:`~DIRAC.DataManagementSystem.Agent.FTS3Agent` | YES | | +--------------------+---------------------------------------------------------------------------------------------------+---------------+-----------------------------------------------------------------------------------+ -| Framework | :mod:`~DIRAC.FrameworkSystem.Agent.CAUpdateAgent` | | | -+ +---------------------------------------------------------------------------------------------------+---------------+-----------------------------------------------------------------------------------+ | RequestManagement | :mod:`~DIRAC.RequestManagementSystem.Agent.CleanReqDBAgent` | NO | | + +---------------------------------------------------------------------------------------------------+---------------+-----------------------------------------------------------------------------------+ | | :mod:`~DIRAC.RequestManagementSystem.Agent.RequestExecutingAgent` | YES | | +--------------------+---------------------------------------------------------------------------------------------------+---------------+-----------------------------------------------------------------------------------+ -| ResourceStatus | :mod:`~DIRAC.ResourceStatusSystem.Agent.CacheFeederAgent` | | | +| ResourceStatus | :mod:`~DIRAC.ResourceStatusSystem.Agent.CacheFeederAgent` | YES | | + +---------------------------------------------------------------------------------------------------+---------------+-----------------------------------------------------------------------------------+ | | :mod:`~DIRAC.ResourceStatusSystem.Agent.ElementInspectorAgent` | | | + +---------------------------------------------------------------------------------------------------+---------------+-----------------------------------------------------------------------------------+ @@ -206,11 +215,17 @@ Agents + +---------------------------------------------------------------------------------------------------+---------------+-----------------------------------------------------------------------------------+ | | :mod:`~DIRAC.WorkloadManagementSystem.Agent.JobAgent` | | Installed by Pilots on Worker Nodes, not for server installations | + +---------------------------------------------------------------------------------------------------+---------------+-----------------------------------------------------------------------------------+ -| | :mod:`~DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent` | | | +| | :mod:`~DIRAC.WorkloadManagementSystem.Agent.JobCleaningAgent` | YES | | ++ +---------------------------------------------------------------------------------------------------+---------------+-----------------------------------------------------------------------------------+ +| | :mod:`~DIRAC.WorkloadManagementSystem.Agent.PilotSyncAgent` | YES | | + +---------------------------------------------------------------------------------------------------+---------------+-----------------------------------------------------------------------------------+ -| | :mod:`~DIRAC.WorkloadManagementSystem.Agent.PilotStatusAgent` | | | +| | :mod:`~DIRAC.WorkloadManagementSystem.Agent.PilotStatusAgent` | YES | | + +---------------------------------------------------------------------------------------------------+---------------+-----------------------------------------------------------------------------------+ -| | :mod:`~DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent` | | | +| | :mod:`~DIRAC.WorkloadManagementSystem.Agent.PushJobAgent` | YES | Split by Sites | ++ +---------------------------------------------------------------------------------------------------+---------------+-----------------------------------------------------------------------------------+ +| | :mod:`~DIRAC.WorkloadManagementSystem.Agent.StalledJobAgent` | YES | | + +---------------------------------------------------------------------------------------------------+---------------+-----------------------------------------------------------------------------------+ | | :mod:`~DIRAC.WorkloadManagementSystem.Agent.StatesAccountingAgent` | NO | | ++ +---------------------------------------------------------------------------------------------------+---------------+-----------------------------------------------------------------------------------+ +| | :mod:`~DIRAC.WorkloadManagementSystem.Agent.TaskQueuesAgent` | YES | | +--------------------+---------------------------------------------------------------------------------------------------+---------------+-----------------------------------------------------------------------------------+ diff --git a/docs/source/AdministratorGuide/ServerInstallations/tornadoComponentsLogs.rst b/docs/source/AdministratorGuide/ServerInstallations/tornadoComponentsLogs.rst new file mode 100644 index 00000000000..5e6139f0d37 --- /dev/null +++ b/docs/source/AdministratorGuide/ServerInstallations/tornadoComponentsLogs.rst @@ -0,0 +1,254 @@ +.. _tornado_components_logs: + +=============================== +Split Tornado logs by component +=============================== + +Dirac offers the ability to write logs for each component. One can find logs in /startup//log/current + +In case of Tornado, logs come from many components, and can be hard to sort. + +Using Fluent-bit will allow to collect logs from files, rearrange content, then send them elsewhere like an ELK instance or simply other files. +Thus, in case of ELK, it's now possible to monitor and display informations through Kibana and Grafana tools, using filters to sort logs, or simply read other splitted log files, one by component. + +The idea behind that is to deal with logs independantly from Dirac. It is also possible to grab servers metrics such as cpu, memory and disk usage, giving the opportunity to make correlations between logs and server usage. + +DIRAC Configuration +------------------- + +First of all, you should configure a JSON Log Backend in your ``Resources`` and ``Operations`` like:: + + Resources + { + LogBackends + { + StdoutJson + { + Plugin = StdoutJson + } + } + } + + Operations + { + Defaults + { + Logging + { + DefaultBackends = StdoutJson + } + } + } + + +Fluent-bit Installation +----------------------- + +On each Dirac server, install Fluent-bit (https://docs.fluentbit.io):: + + curl https://raw.githubusercontent.com/fluent/fluent-bit/master/install.sh | sh + +Fluent-bit Configuration +------------------------ + +Edit and add in /etc/fluent-bit/fluent-bit.conf:: + + @INCLUDE dirac-json.conf + +Create following files in /etc/fluent-bit + +dirac-json.conf (Add all needed components and choose the output you want):: + + [SERVICE] + flush 1 + log_level info + parsers_file dirac-parsers.conf + + [INPUT] + name cpu + tag metric + Interval_Sec 10 + + [INPUT] + name mem + tag metric + Interval_Sec 10 + + [INPUT] + name disk + tag metric + Interval_Sec 10 + + [INPUT] + name tail + parser dirac_parser_json + path /startup//log/current + Tag log..log + Mem_Buf_Limit 50MB + + [INPUT] + name tail + parser dirac_parser_json + path /startup//log/current + Tag log..log + Mem_Buf_Limit 50MB + + [FILTER] + Name modify + Match log.* + Rename log message + Add levelname DEV + + [FILTER] + Name modify + Match * + Add hostname ${HOSTNAME} + + [FILTER] + Name Lua + Match log.* + script dirac.lua + call add_raw + + [FILTER] + Name rewrite_tag + Match log.tornado + Rule $tornadoComponent .$ $TAG.$tornadoComponentclean.log false + Emitter_Name re_emitted + + #[OUTPUT] + # name stdout + # match * + + [OUTPUT] + Name file + Match log.* + Path /vo/dirac/logs + Mkdir true + Format template + Template {raw} + + [OUTPUT] + name es + host + port + logstash_format true + logstash_prefix + tls on + tls.verify off + tls.ca_file + tls.crt_file + tls.key_file + match log.* + + [OUTPUT] + name es + host + port + logstash_format true + logstash_prefix + tls on + tls.verify off + tls.ca_file + tls.crt_file + tls.key_file + match metric + +``dirac-json.conf`` is the main file, it defines different steps such as:: + [SERVICE] where we describe our json parser (from dirac Json log backend) + [INPUT] where we describe dirac components log file and the way it will be parsed (json) + [FILTER] where we apply modifications to parsed data, for example adding a levelname "DEV" whenever logs are not well formatted, typically "print" in code, or adding fields like hostname to know from which host logs are coming, but also more complex treatments like in dirac.lua script (described later) + [OUTPUT] where we describe formatted logs destination, here, we have stdout, files on disks and opensearch. + +dirac-parsers.conf:: + + [PARSER] + Name dirac_parser_json + Format json + Time_Key asctime + Time_Format %Y-%m-%d %H:%M:%S,%L + Time_Keep On + +``dirac-parsers.conf`` describes the source format that will be parsed, and the time that will be used (here asctime field) as reference + +dirac.lua:: + + function add_raw(tag, timestamp, record) + new_record = record + + if record["asctime"] ~= nil then + raw = record["asctime"] .. " [" .. record["levelname"] .. "] [" .. record["componentname"] .. "] " + if record["tornadoComponent"] ~= nil then + patterns = {"/"} + str = record["tornadoComponent"] + for i,v in ipairs(patterns) do + str = string.gsub(str, v, "_") + end + new_record["tornadoComponentclean"] = str + raw = raw .. "[" .. record["tornadoComponent"] .. "] " + else + raw = raw .. "[]" + end + raw = raw .. "[" .. record["customname"] .. "] " .. record["message"] .. " " .. record["varmessage"] .. " [" .. record["hostname"] .. "]" + new_record["raw"] = raw + else + new_record["raw"] = os.date("%Y-%m-%d %H:%M:%S %Z") .. " [" .. record["levelname"] .. "] " .. record["message"] .. " [" .. record["hostname"] .. "]" + end + + return 2, timestamp, new_record + end + +``dirac.lua`` is the most important transformation we perform on primarily logs, it builds new record depending on logs containing or not special field tornadocomponent, then cleans and formats it before sending to the outputs. + +Testing +------- + +Before throwing logs to OpenSearch, config can be tested in Standard output by uncommenting:: + + [OUTPUT] + name stdout + match * + +...and commenting OpenSearch outputs. + +Then by using command:: + + /opt/fluent-bit/bin/fluent-bit -c //etc/fluent-bit/fluent-bit.conf + +NOTE: When all is OK, uncomment OpenSearch outputs and comment stdout output + +Service +------- + +``sudo systemctl start/stop fluent-bit.service`` + +Dashboards +---------- + +In case of logs sent to an ELK instance, dashboards are available `here `_. + +On disk +------- + +In case of logs sent to local files, Logrotate is mandatory. + +Having a week log retention, Logrotate config file should look like +/etc/logrotate.d/diraclogs:: + + /vo/dirac/logs/* { + rotate 7 + daily + missingok + notifempty + compress + delaycompress + create 0644 diracsgm dirac + sharedscripts + postrotate + /bin/kill -HUP `cat /var/run/syslogd.pid 2>/dev/null` 2>/dev/null || true + endscript + } + +along with crontab line like + +``0 0 * * * logrotate /etc/logrotate.d/diraclogs`` diff --git a/docs/source/AdministratorGuide/Systems/Accounting/index.rst b/docs/source/AdministratorGuide/Systems/Accounting/index.rst index bf69f424648..b77ea651f38 100644 --- a/docs/source/AdministratorGuide/Systems/Accounting/index.rst +++ b/docs/source/AdministratorGuide/Systems/Accounting/index.rst @@ -18,7 +18,7 @@ in "two" different formats: The system consists of the following accounting types: - Job: for creating reports of the activity on the computing resources such as Grid, Cloud, etc. - - Pilot: for creating reports for pilot jobs running on different computing elements such as ARC CE, CREAM, VAC, etc. + - Pilot: for creating reports for pilot jobs running on different computing elements such as ARC, HTCondor, VAC, etc. - Data operation: for creating reports about data activities: transfers, replication, removal, etc. - WMS History: This it used for monitoring the DIRAC Workload Management system. This type is replaced by the WMS monitoring which is part of the Monitoring system. It is replaced, because the WMS History type is for real diff --git a/docs/source/AdministratorGuide/Systems/Configuration/index.rst b/docs/source/AdministratorGuide/Systems/Configuration/index.rst index 22a02e7c5e7..9cec6415c8a 100644 --- a/docs/source/AdministratorGuide/Systems/Configuration/index.rst +++ b/docs/source/AdministratorGuide/Systems/Configuration/index.rst @@ -5,22 +5,22 @@ Configuration System ==================== The configuration system serves the configuration to any other client (be it another server or a standard client). -The infrastructure is master/slave based. +The infrastructure is controller/worker based. -****** -Master -****** +********** +Controller +********** -The master Server holds the central configuration in a local file. This file is then served to the clients, and synchronized with the slave servers. +The controller Server holds the central configuration in a local file. This file is then served to the clients, and synchronized with the worker servers. -the master server also regularly pings the slave servers to make sure they are still alive. If not, they are removed from the list of CS. +the controller server also regularly pings the worker servers to make sure they are still alive. If not, they are removed from the list of CS. -When changes are committed to the master, a backup of the existing configuration file is made in ``etc/csbackup``. +When changes are committed to the controller, a backup of the existing configuration file is made in ``etc/csbackup``. -****** -Slaves -****** +******* +Workers +******* -Slave server registers themselves to the master when starting. +worker server registers themselves to the controller when starting. They synchronize their configuration on a regular bases (every 5 minutes by default). -Note that the slave CS do not hold the configuration in a local file, but only in memory. +Note that the worker CS do not hold the configuration in a local file, but only in memory. diff --git a/docs/source/AdministratorGuide/Systems/DataManagement/fts3.rst b/docs/source/AdministratorGuide/Systems/DataManagement/fts3.rst index 0566058e3ea..7822ce7c1a1 100644 --- a/docs/source/AdministratorGuide/Systems/DataManagement/fts3.rst +++ b/docs/source/AdministratorGuide/Systems/DataManagement/fts3.rst @@ -4,8 +4,6 @@ FTS3 support in DIRAC --------------------- -.. versionadded:: v6r20 - .. contents:: Table of contents :depth: 2 @@ -42,13 +40,11 @@ In order for the transfers to be submitted to the FTS system, the following opti * ``FTSMode`` must be True * ``FTSBannedGroups`` should contain the list of groups for which you'd rather do direct transfers. - * ``UseNewFTS3`` should be True in order to use this new FTS system (soon to be deprecated) ======================== Operations configuration ======================== - * DataManagement/FTSVersion: FTS2/FTS3. Set it to FTS3... * DataManagement/FTSPlacement/FTS3/ServerPolicy: Policy to choose the FTS server see `FTSServer policy`_. * DataManagement/FTSPlacement/FTS3/FTS3Plugin: Plugin to alter the behavior of the FTS3Agent @@ -150,10 +146,6 @@ The FTS3Operation goes to ``Processed`` when all the files are in a final state, FTS3 Plugins ------------ -.. versionadded:: v7r1p37 - The ``FTS3Plugin`` option - - The ``FTS3Plugin`` option allows one to specify a plugin to alter some default choices made by the FTS3 system. These choices concern: * the list of third party protocols used @@ -167,8 +159,6 @@ This can be useful if you want to implement a matrix-like selection of protocols MultiHop support ---------------- -.. versionadded:: v7r3p21 - .. |trade| unicode:: U+2122 .. warning:: @@ -200,3 +190,27 @@ More details on how the intermediate SE selection is done and how the matrix is Work in FTS has a `task `_ to try and bring that feature in. A future solution may come from DIRAC. In the meantime, the best solution is to ask the site to either cleanup themselves (some storages like EOS have that built in) or to give you a dump of the namespace, and then do the cleaning yourself. + + +Token support +---------------- + +.. versionadded:: v8.0.51 + +.. warning:: + Very experimental feature + + +The current state is the one in which LHCb ran the DC24 challenge. It only worked for dCache site, as there is still not a uniform way for storages to understand permissions... +A transfer will happen with token if: + + * ``UseTokens`` is true in the FTSAgent configuration + * ``WLCGTokenBasePath`` is set for both the source and the destination + +The tokens use specific file path, and not generic wildcard permissions. + +.. warning:: + Token support is as experimental as can be in any layer of the stack (DIRAC, storage, FTS... even the model is experimental) + +.. warning:: + The FTS3Agent got occasionaly stuck when tokens were used diff --git a/docs/source/AdministratorGuide/Systems/DataManagement/s3.rst b/docs/source/AdministratorGuide/Systems/DataManagement/s3.rst index 5766f4dfa1f..aa20311aa70 100644 --- a/docs/source/AdministratorGuide/Systems/DataManagement/s3.rst +++ b/docs/source/AdministratorGuide/Systems/DataManagement/s3.rst @@ -4,8 +4,6 @@ S3 support in DIRAC ------------------- -.. versionadded:: v7r0p19 - .. contents:: Table of contents :depth: 2 diff --git a/docs/source/AdministratorGuide/Systems/Framework/ComponentMonitoring/index.rst b/docs/source/AdministratorGuide/Systems/Framework/ComponentMonitoring/index.rst index 059d1273fc4..6f4155909c6 100644 --- a/docs/source/AdministratorGuide/Systems/Framework/ComponentMonitoring/index.rst +++ b/docs/source/AdministratorGuide/Systems/Framework/ComponentMonitoring/index.rst @@ -20,8 +20,6 @@ Installation The service constitutes of one database (InstalledComponentsDB) and one service (Framework/ComponentMonitoring). These service and DB may have been installed already when DIRAC was installed the first time. -The script ``dirac-populate-component-db`` should then be used to populate the DB tables with the necessary information. - Interacting with the static component monitoring ------------------------------------------------ diff --git a/docs/source/AdministratorGuide/Systems/Framework/Monitoring/index.rst b/docs/source/AdministratorGuide/Systems/Framework/Monitoring/index.rst index 4b64847d8fe..6e88bc111a2 100644 --- a/docs/source/AdministratorGuide/Systems/Framework/Monitoring/index.rst +++ b/docs/source/AdministratorGuide/Systems/Framework/Monitoring/index.rst @@ -6,4 +6,4 @@ The Framework/monitoring service The framework system for the monitoring of services and agents has been removed and will no longer be used. -It has been replaced by an ElasticSearch-based monitoring system. You can read about it in :ref:`Monitoring ` +It has been replaced by an OpenSearch-based monitoring system. You can read about it in :ref:`Monitoring ` diff --git a/docs/source/AdministratorGuide/Systems/Framework/Notification/index.rst b/docs/source/AdministratorGuide/Systems/Framework/Notification/index.rst index eeaef5ef8e9..42fe8a05e9b 100644 --- a/docs/source/AdministratorGuide/Systems/Framework/Notification/index.rst +++ b/docs/source/AdministratorGuide/Systems/Framework/Notification/index.rst @@ -5,7 +5,7 @@ The Framework/Notification service ================================== -The Framework/Notification service is responsible for notification, like as send mail, sms or alarm window on DIRAC portal. +The Framework/Notification service is responsible for sending mail. Send an email with supplied body to the specified address using the Mail utility. Emails with the same address, subject, and content are only sent once every 24h. diff --git a/docs/source/AdministratorGuide/Systems/Framework/index.rst b/docs/source/AdministratorGuide/Systems/Framework/index.rst index 5a83fef0f06..e51159de3a3 100644 --- a/docs/source/AdministratorGuide/Systems/Framework/index.rst +++ b/docs/source/AdministratorGuide/Systems/Framework/index.rst @@ -15,7 +15,7 @@ and a monitoring system that accounts for CPU and memory usage, queries served, Another very important functionality provided by the framework system is proxies management, via the ProxyManager service and database. -ComponentMonitoring, SecurityLogging, and ProxyManager services are only part of the services that constitute the +ComponentMonitoring and ProxyManager services are only part of the services that constitute the Framework of DIRAC. The following sections add some details for some of the Framework systems. diff --git a/docs/source/AdministratorGuide/Systems/MonitoringSystem/index.rst b/docs/source/AdministratorGuide/Systems/MonitoringSystem/index.rst index 2588e7850b6..53133c20ff8 100644 --- a/docs/source/AdministratorGuide/Systems/MonitoringSystem/index.rst +++ b/docs/source/AdministratorGuide/Systems/MonitoringSystem/index.rst @@ -18,21 +18,20 @@ The Monitoring system is used to monitor various components of DIRAC. Currently, - Service Monitoring: for monitoring the activity of DIRAC services. - RMS Monitoring: for monitoring the DIRAC RequestManagement System (mostly the Request Executing Agent). - PilotSubmission Monitoring: for monitoring the DIRAC pilot submission statistics from SiteDirector agents. - - DataOperation Monitoring: for monitoring the DIRAC data operation statistics. + - DataOperation Monitoring: for monitoring the DIRAC data operation statistics as well as individual failures from interactive use of ``StorageElement``. -It is based on Elasticsearch distributed search and analytics NoSQL database. -If you want to use it, you have to install the Monitoring service, and of course connect to a ElasticSearch instance. +It is based on OpenSearch distributed search and analytics NoSQL database. +If you want to use it, you have to install the Monitoring service, and of course connect to a OpenSearch instance. -Install Elasticsearch -====================== +Install OpenSearch +================== -This is not covered here, as installation and administration of ES are not part of DIRAC guide. -Just a note on the ES versions supported: only ES7+ versions are currently supported, and are later to be replaced by OpenSearch services. +This is not covered here. Configure the MonitoringSystem =============================== -You can run your Elastic cluster even without authentication, or using User name and password. You have to add the following parameters: +You can run your OpenSearch cluster even without authentication, or using User name and password. You have to add the following parameters: - User - Password @@ -55,7 +54,7 @@ For example:: } -The following option can be set in `Systems/Monitoring//Databases/MonitoringDB`: +The following option can be set in `Systems/Monitoring/Databases/MonitoringDB`: *IndexPrefix*: Prefix used to prepend to indexes created in the ES instance. If this is not present in the CS, the indices are prefixed with the setup name. @@ -82,7 +81,7 @@ The given periods above are also the default periods in the code. Enable the Monitoring System ============================ -In order to enable the monitoring of all the following types with an ElasticSearch-based backend, you should add the value `Monitoring` to the flag +In order to enable the monitoring of all the following types with an OpenSearch-based backend, you should add the value `Monitoring` to the flag `MonitoringBackends/Default` in the Operations section of the CS. If you want to override this flag for a specific type, say, you want to only have Monitoring (and no Accounting) for WMSHistory, you just create a flag `WMSHistory` set to `Monitoring`. If, for example, you want both Monitoring and Accounting for WMSHistory (but not for other types), you set `WMSHistory = Accounting, Monitoring`. If no flag is set for `WMSHistory`, the `Default` flag will be used. @@ -93,7 +92,7 @@ This can be done either via the CS or directly in the web app in the Configurati Operations { - + { MonitoringBackends { @@ -111,10 +110,10 @@ This can be done either via the CS or directly in the web app in the Configurati WMSHistory & PilotsHistory Monitoring ===================================== -The WorkloadManagement/StatesAccountingAgent creates, every 15 minutes, a snapshot with the contents of JobDB and PilotAgentsDB and sends it to an Elasticsearch-based database. +The WorkloadManagement/StatesAccountingAgent creates, every 15 minutes, a snapshot with the contents of JobDB and PilotAgentsDB and sends it to an OpenSearch-based database. This same agent can also report the WMSHistory to the MySQL backend used by the Accounting system (which is in fact the default). -Optionally, you can use an MQ system (like RabbitMQ) for failover, even though the agent already has a simple failover mechanism. +Optionally, you can use an MQ system (like ActiveMQ) for failover, even though the agent already has a simple failover mechanism. You can configure the MQ in the local dirac.cfg file where the agent is running:: Resources @@ -161,17 +160,20 @@ Data Operation Monitoring This monitoring enables the reporting of information about the data operation such as the cumulative transfer size or the number of succeded and failed transfers. +It will also fill an index called ``faileddataoperation_index`` containing entries for individual interactive failures (CLI, Job, etc). + Accessing the Monitoring information ===================================== -After you installed and configured the Monitoring system, you can use the Monitoring web application for the types WMSHistory, PilotSubmission, DataOperation and RMS. +After you installed and configured the Monitoring system, you can use the Monitoring web application for the types WMSHistory and RMS. -However, every type can directly be monitored in Kibana dashboards that can be imported into your Elasticsearch (or Opensearch) instance. You can find and import these dashboards from DIRAC/dashboards as per the following example. +However, every type can directly be monitored in Kibana dashboards that can be imported into your Opensearch instance. You can find and import these dashboards from DIRAC/dashboards as per the following example. +Grafana dashboards are also provided for some of the types. *Kibana dashboard for WMSHistory* - A dashboard for WMSHistory monitoring ``WMSDashboard`` is available `here `__ for import as a NDJSON (as support for JSON is being removed in the latest versions of Kibana). - The dashboard may not be compatible with older versions of ElasticSearch. + A dashboard for WMSHistory monitoring ``WMSDashboard`` is available `here `__ for import as a NDJSON (as support for JSON is being removed in the latest versions of Kibana). + The dashboard may not be compatible with older versions of OpenSearch. To import it in the Kibana UI, go to Management -> Saved Objects -> Import and import the JSON file. Note: the JSON file already contains the index patterns needed for the visualizations. You may need to adapt the index patterns to your existing ones. diff --git a/docs/source/AdministratorGuide/Systems/RequestManagement/rmsObjects.rst b/docs/source/AdministratorGuide/Systems/RequestManagement/rmsObjects.rst index 2718f404826..0b75d437c5e 100644 --- a/docs/source/AdministratorGuide/Systems/RequestManagement/rmsObjects.rst +++ b/docs/source/AdministratorGuide/Systems/RequestManagement/rmsObjects.rst @@ -188,6 +188,7 @@ Details: :py:mod:`~DIRAC.DataManagementSystem.Agent.RequestOperations.ReplicateA Extra configuration options: * `FTSMode`: If True, will use FTS to transfer files +* `DMMode`: if False, will not use DataManager transfer as FTS failover * `FTSBannedGroups` : list of groups for which not to use FTS ------ diff --git a/docs/source/AdministratorGuide/Systems/ResourceStatus/advanced_configuration.rst b/docs/source/AdministratorGuide/Systems/ResourceStatus/advanced_configuration.rst index 0756582e401..fa1a02ca8f8 100644 --- a/docs/source/AdministratorGuide/Systems/ResourceStatus/advanced_configuration.rst +++ b/docs/source/AdministratorGuide/Systems/ResourceStatus/advanced_configuration.rst @@ -25,7 +25,7 @@ This section describes the policies and the conditions to match elements. :: - /Operations/[Defaults|SetupName]/ResourceStatus + /Operations/Defaults/ResourceStatus /Policies /PolicyName policyType = policyType @@ -70,7 +70,7 @@ It applies the same idea as in `Policies`_, but the number of options is larger. :: - /Operations/[Defaults|SetupName]/ResourceStatus + /Operations/Defaults/ResourceStatus /PolicyActions /PolicyActionName actionType = actionType @@ -110,7 +110,7 @@ This section defines the notification groups ( right now, only for EmailAction ) :: - /Operations/[Defaults|SetupName]/ResourceStatus + /Operations/Defaults/ResourceStatus /Notification /NotificationGroupName users = email@address, email@address diff --git a/docs/source/AdministratorGuide/Systems/ResourceStatus/install.rst b/docs/source/AdministratorGuide/Systems/ResourceStatus/install.rst index 2c7a7f56ce0..fca866687a4 100644 --- a/docs/source/AdministratorGuide/Systems/ResourceStatus/install.rst +++ b/docs/source/AdministratorGuide/Systems/ResourceStatus/install.rst @@ -8,19 +8,17 @@ This page describes the basic steps to install, configure, activate and start us *WARNING*: If you have doubts about the success of any step, DO NOT ACTIVATE RSS. -*WARNING*: REPORT FIRST to the DIRAC FORUM ! - ---------------- CS Configuration ---------------- The configuration for RSS sits under the following path on the CS following the usual /Operations section convention:: - /Operations/[Defaults|SetupName]/ResourceStatus + /Operations/Defaults/ResourceStatus Please, make sure you have the following schema:: - /Operations/[Defaults|SetupName]/ResourceStatus + /Operations/Defaults/ResourceStatus /Config State = InActive Cache = 300 @@ -87,26 +85,17 @@ Copy over the values that we had on the CS for the StorageElements:: $ dirac-rss-sync --init -o LogLevel=VERBOSE -*WARNING*: If the StorageElement does not have a particular StatusType declared - -*WARNING*: on the CS, this script will set it to Banned. If that happens, you will +*WARNING*: If the StorageElement does not have a particular StatusType declared, on the CS, this script will set it to Active. -*WARNING*: have to issue the dirac-rss-status script over the elements that need - -*WARNING*: to be fixed. +You can check the status of the resources with the following command:: + $ dirac-rss-list-status --element Resource --elementType StorageElement -------------------- Set statuses by HAND -------------------- -In case you entered the WARNING ! on point 4, you may need to identify the -status of your StorageElements. Try to detect the Banned SEs using the -following:: - - $ dirac-rss-list-status --element Resource --elementType StorageElement --status Banned - -If is there any SE to be modified, you can do it as follows:: +If there is any SE status to be modified, you can do it as follows:: $ dirac-rss-set-status --element Resource --name CERN-USER --statusType ReadAccess --status Active --reason "Why not?" # This matches all StatusTypes @@ -114,13 +103,6 @@ If is there any SE to be modified, you can do it as follows:: .. _activateRSS: ------------- -Activate RSS ------------- - -If you did not see any problem, activate RSS by setting the CS option:: - - /Operations/[Defaults|SetupName]/ResourceStatus/Config/State = Active ------ Agents diff --git a/docs/source/AdministratorGuide/Systems/ResourceStatus/introduction.rst b/docs/source/AdministratorGuide/Systems/ResourceStatus/introduction.rst index f7a7896d49e..9282373097e 100644 --- a/docs/source/AdministratorGuide/Systems/ResourceStatus/introduction.rst +++ b/docs/source/AdministratorGuide/Systems/ResourceStatus/introduction.rst @@ -87,11 +87,11 @@ And if we take a look to the ComputingElement Resources, we can see the pattern :: - .../Computing/some.cream.ce - /CEType = CREAM - /Host = some.cream.ce + .../Computing/some.htcondor.ce + /CEType = HTCondorCE + /Host = some.htcondor.ce /Queues - /cream-sge-long + /condor-long /Communities = VO1, VO2 /Domains = Grid1, Grid2 /MaxCPUTime = diff --git a/docs/source/AdministratorGuide/Systems/ResourceStatus/monitoring.rst b/docs/source/AdministratorGuide/Systems/ResourceStatus/monitoring.rst index ca376f239da..d85c24a2404 100644 --- a/docs/source/AdministratorGuide/Systems/ResourceStatus/monitoring.rst +++ b/docs/source/AdministratorGuide/Systems/ResourceStatus/monitoring.rst @@ -146,7 +146,6 @@ Actions DIRAC.RSS has the following actions: * **EmailAction** : sends an email notification -* **SMSAction** : sends a sms notification ( not certified yet ). * **LogStatusAction** : updates the Status table with the new computed status * **LogPolicyResultAction** : updates the PolicyResult table with the results of the policies in singlePolicyResults. diff --git a/docs/source/AdministratorGuide/Systems/Transformation/index.rst b/docs/source/AdministratorGuide/Systems/Transformation/index.rst index a5373836c68..1fefb4646db 100644 --- a/docs/source/AdministratorGuide/Systems/Transformation/index.rst +++ b/docs/source/AdministratorGuide/Systems/Transformation/index.rst @@ -46,10 +46,7 @@ Within the TS a user can (for example): Disadvantages: -- For very large installations, the submission may be perceived as slow, since there is no use (not yet) of Parametric jobs. - - .. versionadded:: v6r20p3 - Bulk submission of jobs is working for the transformations, so job submission can be sped up considerably. +- For very large installations, the submission may be perceived as slow, since there is no use (not yet) of Parametric jobs. Bulk submission of jobs is working for the transformations, so job submission can be sped up considerably. Several improvements have been made in the TS to handle scalability, and extensibility issues. While the system structure remains intact, "tricks" like threading and caching have been extensively applied. @@ -691,9 +688,6 @@ Multi VO Configuration ---------------------- - -.. versionadded:: v6r20p5 - There are two possibilities to configure the agents of the transformation system for the use in a multi VO installation. - Use the same WorkflowTaskAgent and RequestTaskAgents for multiple VOs, no diff --git a/docs/source/AdministratorGuide/Systems/WorkloadManagement/InputDataResolution.rst b/docs/source/AdministratorGuide/Systems/WorkloadManagement/InputDataResolution.rst new file mode 100644 index 00000000000..8b9ffa1cb9c --- /dev/null +++ b/docs/source/AdministratorGuide/Systems/WorkloadManagement/InputDataResolution.rst @@ -0,0 +1,35 @@ +.. _InputDataResolution: + +================================================== +InputDataResolution: giving job access to the data +================================================== + +When a job needs access to data, there are two ways data can be accessed: + +* either by downloading the file on the local worker node +* or by reading the data remotely, aka ``streaming``. + +The resolution is done in the ``JobWrapper`` (see :ref:`jobs`). By default, the resolution logic is implemented in :py:mod:`~DIRAC.WorkloadManagementSystem.Client.InputDataResolution`. It can be overwritten by the Job JDL (see ``InputDataModule`` in :ref:`jdlDescription`), or by the ``/Operations/<>/InputDataPolicy/InputDataModule`` parameter. + + +You can look into this class for more details, but to summarize: + +* it will look into the ``job`` JDL if it can find ``InputDataPolicy`` option. If so, it will use that as the module. +* If not, it will check whether a policy is defined for the site we are running on (in ``/Operations/InputDataPolicy/``). +* If not, it will run the default policy specified in ``/Operations/InputDataPolicy/Default`` + +The ``InputDataPolicy`` parameter can either be set directly in the JDL, in which case it should be a full module, or it can be set using the ``Job`` class (see :py:meth:`~DIRAC.Interfaces.API.Job.Job.setInputDataPolicy`) + +DownloadInputData +================= + +This module will download the files locally on the worker node for processing. + +See :py:mod:`~DIRAC.WorkloadManagementSystem.Client.DownloadInputData` for details. + +InputDataByProtocol +=================== + +This module will generate the URLs necessary to access the files remotely. + +See :py:mod:`~DIRAC.WorkloadManagementSystem.Client.InputDataByProtocol` for details. diff --git a/docs/source/AdministratorGuide/Systems/WorkloadManagement/JobsMatching.rst b/docs/source/AdministratorGuide/Systems/WorkloadManagement/JobsMatching.rst index d7ba8823c79..ba6397c90f0 100644 --- a/docs/source/AdministratorGuide/Systems/WorkloadManagement/JobsMatching.rst +++ b/docs/source/AdministratorGuide/Systems/WorkloadManagement/JobsMatching.rst @@ -24,11 +24,10 @@ The JobAgent running on the Worker Node and started by the pilot presents capabi OwnerGroup: diracAdmin,test,user PilotBenchmark: 19.5 PilotInfoReportedFlag: False - PilotReference: https://ce-01.somewhere.org:8443/CREAM155256908 + PilotReference: htcondorce://ce-01.somewhere.org:8443/155256908.0 Platform: x86_64_glibc-2.21 ReleaseProject: VO ReleaseVersion: 7.2.13 - Setup: VO-Certification Site: DIRAC.somewhere.org Tag: GPU } @@ -40,9 +39,8 @@ An example of requirements include the following:: JobRequirements = [ - OwnerDN = "/some/DN/"; + Owner = "user_x"; VirtualOrganization = "VO"; - Setup = "VO-Certification"; CPUTime = 17800; OwnerGroup = "user"; UserPriority = 1; diff --git a/docs/source/AdministratorGuide/Systems/WorkloadManagement/JobsPriorities.rst b/docs/source/AdministratorGuide/Systems/WorkloadManagement/JobsPriorities.rst index 6d0f2946adb..5aee9a18914 100644 --- a/docs/source/AdministratorGuide/Systems/WorkloadManagement/JobsPriorities.rst +++ b/docs/source/AdministratorGuide/Systems/WorkloadManagement/JobsPriorities.rst @@ -93,8 +93,8 @@ DIRAC includes a priority correction mechanism. The idea behind it is to look at the past history and alter the priorities assigned based on it. It can have multiple plugins but currently it only has one. All correctors have a CS section to configure themselves under -`/Operations///JobScheduling/ShareCorrections`. The option -`/Operations///JobScheduling/ShareCorrections/ShareCorrectorsToStart` +`/Operations//JobScheduling/ShareCorrections`. The option +`/Operations//JobScheduling/ShareCorrections/ShareCorrectorsToStart` defines witch correctors will be used in each iteration. WMSHistory corrector diff --git a/docs/source/AdministratorGuide/Systems/WorkloadManagement/Pilots/Pilots3.rst b/docs/source/AdministratorGuide/Systems/WorkloadManagement/Pilots/Pilots3.rst index f1860b4ded9..f6d1446e85d 100644 --- a/docs/source/AdministratorGuide/Systems/WorkloadManagement/Pilots/Pilots3.rst +++ b/docs/source/AdministratorGuide/Systems/WorkloadManagement/Pilots/Pilots3.rst @@ -47,11 +47,11 @@ Other options that can be set also in the Operations part of the CS include: | *pilotVORepoBranch* | Branch to use, inside the Git repository, | pilotVORepoBranch = master | | | of the pilot code extension to be used | The value above is the default | +------------------------------------+--------------------------------------------+-------------------------------------------------------------------------+ -| *uploadToWebApp* | Whether to try to upload the files to the | uploadToWebApp = True | -| | list of server specified | The value above is the default | +| *CVMFS_locations* | CVMFS locations to use for finding pilot | CVMFS_locations = /cvmfs/lhcb.cern.ch/,/cvmfs/lhcbdev.cern.ch | +| | files, DIRAC installations, CAs, CRLS | | +------------------------------------+--------------------------------------------+-------------------------------------------------------------------------+ -| *workDir* | Local directory of the master CS where the | workDir = /tmp/pilotSyncDir | -| | files will be downloaded before the upload | There is no default (so /opt/dirac/runit/Configuration/Server) | +| *ArchitectureScript* | The script to be used for | ArchitectureScript = dirac-apptainer-exec dirac-platform | +| | discovering the local architecture. | Default: dirac-platform | +------------------------------------+--------------------------------------------+-------------------------------------------------------------------------+ @@ -79,7 +79,7 @@ For example:: dirac-pilot.py --modules=https://github.com/chaen/DIRAC.git:::DIRAC:::rel-v7r3_FEAT_proxyStrength -would end up installing a specific branch (``rel-v7r3_FEAT_proxyStrength``) pushed to github. +would end up installing a specific branch (``rel-v7r3_FEAT_proxyStrength``) pushed to github by user `chaen`. An easy way to try is to add the ``Modules`` option in the CS configuration of one of your ComputingElements:: diff --git a/docs/source/AdministratorGuide/Systems/WorkloadManagement/Pilots/index.rst b/docs/source/AdministratorGuide/Systems/WorkloadManagement/Pilots/index.rst index 3e36ba39543..bbc5b50154a 100644 --- a/docs/source/AdministratorGuide/Systems/WorkloadManagement/Pilots/index.rst +++ b/docs/source/AdministratorGuide/Systems/WorkloadManagement/Pilots/index.rst @@ -8,7 +8,7 @@ DIRAC pilots :keywords: Pilots3, Pilot3, Pilot This page describes what are DIRAC pilots, and how they work. -To know how to develop DIRAC pilots, please refer to the Developers documentation +To know how to develop DIRAC pilots, please refer to the Developers documentation. Pilot development is done in https://github.com/DIRACGrid/Pilot @@ -23,22 +23,24 @@ First of all, a definition: A pilot can be sent, as a script to be run. Or, it can be fetched. -A pilot can run on every computing resource, e.g.: on CREAM Computing elements, +A pilot can run on every computing resource, e.g.: on HTCondor Computing elements, on DIRAC Computing elements, on Virtual Machines in the form of contextualization script, or IAAC (Infrastructure as a Client) provided that these machines are properly configured. A pilot has, at a minimum, to: -- install DIRAC +- install or setup DIRAC, or an extension of it - configure DIRAC - run the JobAgent -A pilot has to run on each and every computing resource type, provided that: +where: -- Python 2.6+ on the WN -- It is an OS onto which we can install a DIRAC client. - - if that's not possible, we plan to add support for singularity +- install means installing DIRAC like described in :ref:`dirac_install` +- setup means that DIRAC code can already be found in the current file system, and it is only a matter of invoking a rc file that would add DIRAC paths +- configure means adding dirac specific configuration files (which, at a minimum, should include the location of a DIRAC configuration service) + +A pilot has to run on each and every computing resource type, provided that Python 2.6+ is on the WN. The same pilot script can be used everywhere. .. image:: Pilots2.png @@ -72,18 +74,20 @@ Administration The following CS section is used for administering the DIRAC pilots:: - Operations//Pilot + Operations/Defaults/Pilot These parameters will be interpreted by the WorkloadManagementSystem/SiteDirector agents, and by the WorkloadManagementSystem/Matcher. They can also be accessed by other services/agents, e.g. for syncing purposes. Inside this section, you should define the following options, and give them a meaningful value (here, an example is given):: - # Needed by the SiteDirector: - Version = v7r2p1 # DIRAC version(s) + # For the SiteDirector: + Version = 8.0.32 # DIRAC version(s) -- a comma-separated list can be provided Project = myVO # Your project name: this will end up in the /LocalSite/ReleaseProject option of the pilot cfg, and will be used at matching time Extensions = myVO # The DIRAC extension (if any) Installation = mycfg.cfg # For an optional configuration file, used by the installation script. + PreInstalledEnv = /cvmfs/some/where/specific/bashrc # A specific rc file to source for setting up DIRAC + PreInstalledEnvPrefix = /cvmfs/some/where/ # Location where DIRAC installations can be found. The Pilot will then try and find the following: /cvmfs/some/where/{Version/}{platform}/diracosrc # For the Matcher CheckVersion = False # True by default, if false any version would be accepted at matching level (this is a check done by the WorkloadManagementSystem/Matcher service). @@ -92,7 +96,7 @@ Further details: - *Version* is the version of DIRAC that the pilots will install. Add the version of your DIRAC extension if you have one. A list of versions can also be added here, meaning that all these versions will be accepted by the Matcher (see below), while only the first in the list will be the one used by the pilots for knowing which DIRAC version to install (e.g. if Version=v7r0p2,v7r0p1 then pilots will install version v7r0p2) - *Project* is, normally, the same as *Extensions* - When the *CheckVersion* option is "True", the version checking done at the Matcher level will be strict, which means that pilots running different versions from those listed in the *Versions* option will refuse to match any job. There is anyway the possibility to list more than one version in *Versions*; in this case, all of them will be accepted by the Matcher. - +- DIRAC versions are pre-installed on CVMFS in the following location: `/cvmfs/dirac.egi.eu`. From there `/cvmfs/dirac.egi.eu/dirac` contains DIRAC installations, like `/cvmfs/dirac.egi.eu/dirac/v8.0.32`, which can be sourced with `. /cvmfs/dirac.egi.eu/dirac/v8.0.32/Linux-x86_64/diracosrc` Pilot Commands @@ -158,22 +162,33 @@ you should carefully read the RFC 18, and what follows. Pilot commands can be extended. A custom list of commands can be added starting the pilot with the -X option. +Pilots started when controlled by the SiteDirector +================================================== + +The :py:mod:`~DIRAC.WorkloadManagementSystem.Agent.SiteDirector` is a central component in DIRAC, +responsible for managing and optimizing the submission of pilot jobs to various computing resources. It features: + +- *Parallel Submission*: Capable of submitting pilot jobs in parallel across different Computing Elements (CEs) to enhance throughput. +- *Monitoring and Accounting*: Features parallel monitoring and accounting for efficient tracking and management of pilot jobs. +- *Pilot Wrapping*: Creates pilot wrappers that facilitate the execution of pilot scripts in diverse environments, including Grid, cloud, and virtualized resources. +- *Resource Status Handling*: Integrates with the Resource Status System to ensure that pilots are only submitted to operational and enabled resources. + +The Site Director is controlled through different parameters set in the DIRAC configuration. More details in :py:mod:`~DIRAC.WorkloadManagementSystem.Agent.SiteDirector`. Pilots started when not controlled by the SiteDirector ====================================================== You should keep reading if your resources include IAAS and IAAC type of resources, like Virtual Machines. If this is the case, then you need to: - - provide a certificate, or a proxy, to start the pilot; - such certificate/proxy should have the `GenericPilot` property; - in case of multi-VO environment, the Pilot should set the `/Resources/Computing/CEDefaults/VirtualOrganization` (as done e.g. by `vm-pilot `_); -- find a way to start the pilots: VMDIRAC extension will make sure to create VirtualMachine contextualized to start Pilot3. +- find a way to start the pilots: DIRAC will make sure to create VirtualMachine contextualized to start DIRAC Pilots. We have introduced a special command named "GetPilotVersion" that you should use, and possibly extend, in case you want to send/start pilots that don't know beforehand the (VO)DIRAC version they are going to install. In this case, you have to provide a json file freely accessible that contains the pilot version. -This is tipically the case for VMs in IAAS and IAAC. +This is typically the case for VMs in IAAS and IAAC. The files to consider are in https://github.com/DIRACGrid/Pilot @@ -181,10 +196,8 @@ The main file in which you should look is dirac-pilot.py that also contains a good explanation on how the system works. You have to provide in this case a pilot wrapper script (which can be written in bash, for example) that will start your pilot script -with the proper environment. If you are on a cloud site, often contextualization of your virtual machine is done by supplying -a script like the following: https://github.com/DIRACGrid/Pilot/blob/master/Pilot/user_data_vm - -A simpler example using the LHCbPilot extension follows:: +with the proper environment. +A simple example using the LHCbPilot extension follows:: #!/bin/sh # @@ -200,10 +213,6 @@ A simpler example using the LHCbPilot extension follows:: DIRAC_SITE="${i#*=}" shift ;; - --lhcb-setup=*) - LHCBDIRAC_SETUP="${i#*=}" - shift - ;; --ce-name=*) CE_NAME="${i#*=}" shift @@ -222,8 +231,6 @@ A simpler example using the LHCbPilot extension follows:: esac done - # Default if not given explicitly - LHCBDIRAC_SETUP=${LHCBDIRAC_SETUP:-LHCb-Production} # JOB_ID is used by when reporting LocalJobID by DIRAC watchdog #export JOB_ID="$VMTYPE:$VM_UUID" @@ -244,9 +251,6 @@ A simpler example using the LHCbPilot extension follows:: DIRAC_PILOT='https://lhcb-portal-dirac.cern.ch/pilot/dirac-pilot.py' DIRAC_PILOT_TOOLS='https://lhcb-portal-dirac.cern.ch/pilot/pilotTools.py' DIRAC_PILOT_COMMANDS='https://lhcb-portal-dirac.cern.ch/pilot/pilotCommands.py' - DIRAC_PILOT_LOGGER='https://lhcb-portal-dirac.cern.ch/pilot/PilotLogger.py' - DIRAC_PILOT_LOGGERTOOLS='https://lhcb-portal-dirac.cern.ch/pilot/PilotLoggerTools.py' - DIRAC_PILOT_MESSAGESENDER='https://lhcb-portal-dirac.cern.ch/pilot/MessageSender.py' LHCbDIRAC_PILOT_COMMANDS='https://lhcb-portal-dirac.cern.ch/pilot/LHCbPilotCommands.py' # @@ -254,16 +258,82 @@ A simpler example using the LHCbPilot extension follows:: wget --no-check-certificate -O dirac-pilot.py $DIRAC_PILOT wget --no-check-certificate -O pilotTools.py $DIRAC_PILOT_TOOLS wget --no-check-certificate -O pilotCommands.py $DIRAC_PILOT_COMMANDS - wget --no-check-certificate -O PilotLogger.py $DIRAC_PILOT_LOGGER - wget --no-check-certificate -O PilotLoggerTools.py $DIRAC_PILOT_LOGGERTOOLS - wget --no-check-certificate -O MessageSender.py $DIRAC_PILOT_MESSAGESENDER wget --no-check-certificate -O LHCbPilotCommands.py $LHCbDIRAC_PILOT_COMMANDS #run the dirac-pilot script python dirac-pilot.py \ - --setup $LHCBDIRAC_SETUP \ --project LHCb \ --Name "$CE_NAME" \ --name "$1" \ --cert \ --certLocation=/scratch/dirac/etc/grid-security \ + +Centralised Pilot Logging +=========================== +The pilot jobs generate log files which are primarily accessed for debugging if +there are issues with a particular resource; these (*classic*) log files are stored in a +resource dependent manner. On a grid CE, the pilot writes logs to stdout/stderr +which are captured by the batch system and can later be retrieved using a CE +specific tool. For a cloud resource the logs are typically written to a file on +a given virtual machine instance where there is no standard or simple way for +them to be retrieved. + +The centralised (*remote*) pilot logging system offers a new resource agnostic logging +to ensure that the pilot logs are captured and made readily accessible for all +resources as an extra debugging facility in parallel with the existing CE-based +logging system. It also offers the ability to preview logs while the pilot +is running. + +The design of the new pilot logging system for DIRAC is based around having the +pilot jobs periodically send their logs back to a central storage service based on the +Tornado web server. For this to work *TornadoPilotLoggingHandler* has to be installed on Tornado. +Further processing of the log entries is done by a back-end plugin; +the plugin to use is selected by the collector service configuration. Currently only +a plugin which stores logs in a file on Tornado is implemented (*FileCacheLoggingPlugin*). +When a pilot job marks a log file finalised, it can be copied by the *PilotLoggingAgent* +to a selected SE. + +The centralised logger can be enabled on a VO-by-VO basis. In addition a CE whitelist can +also be provided to restrict pilot logging to those CEs. + +Remote logger *FileCacheLoggingPlugin* requires following obligatory configuration parameters set in *Operations//Pilot* or *Operations/Defaults/Pilot*: + +- *RemoteLogging* - Enable remote logging (default False - disabled). +- *RemoteLoggerURL* - to be set to the Tornado endpoint, e.g. *https://:8443/WorkloadManagement/TornadoPilotLogging*. +- *UploadSE* - Dirac SE name, where complete logs will be periodically uploaded to by the *PilotLoggingAgent*. +- *UploadPath* - VO-specific upload path on the SE (e.g. *//pilotlogs/*). + +To fine-tune the logger the following parameters could also be adjusted, if necessary: + +- *PilotLogLevel* - log level, default INFO. +- *RemoteLoggerBufsize* - client-side buffer size in lines; default=1000. If the buffer is full it is flushed, causing log records + to be sent to the server. The buffer + is also flushed when an initial pilot activity is finished (i.e. before pilot commands are run) and when a pilot command + finishes (successfully ot not). +- *RemoteLoggerTimerInterval* - a client-side timer interval in seconds. The logs are + periodically flushed. Default: 0 - disabled. The idea behind this option is to make logs available for inspection, should a pilot get stuck. +- *RemoteLoggerCEsWhiteList* - a list of CEs for which the logger records are sent. + Default: no CE restriction. +- *RemoteLogsPriority* - which logs to get first, default False; this will attempt to retrieve + classic logs first. + +*PilotLoggingAgent* configuration: + +- in *Operations* - *Shifter/DataManager* User and Group of a shifter proxy used to upload data. + +Agent's options: + +- *ClearPilotsDelay* - logs lifetime in days on Tornado document area, default: 30. +- *proxyTimeleftLimit* - time limit in seconds, before we get a new one; default: 600. + +The administrator interface for retrieving pilot log files has also been +connected to the collector. When the admin requests a pilot log from the DIRAC +PilotManager service, the default resource-based method for fetching the log +file is tried first; if for any reason this fails (e.g. log not available or +the resource is offline) then the remote log collector is queried instead. The +collector uses the configured plugin to try to retrieve the log file from the +store. The order of the log sources is configurable by the DIRAC administrator (see +*RemoteLogsPriority* flag above) +allowing the collector to be queried before the resource-based system. This +fallback mechanism is completely transparent to the administrator, the log is +simply fetched from whichever source has it available. diff --git a/docs/source/AdministratorGuide/Systems/WorkloadManagement/PilotsLogging/PilotsLoggingDB.png b/docs/source/AdministratorGuide/Systems/WorkloadManagement/PilotsLogging/PilotsLoggingDB.png deleted file mode 100644 index ee54b10f936..00000000000 Binary files a/docs/source/AdministratorGuide/Systems/WorkloadManagement/PilotsLogging/PilotsLoggingDB.png and /dev/null differ diff --git a/docs/source/AdministratorGuide/Systems/WorkloadManagement/PilotsLogging/PilotsLoggingDiagram.png b/docs/source/AdministratorGuide/Systems/WorkloadManagement/PilotsLogging/PilotsLoggingDiagram.png deleted file mode 100644 index 750f67d8c2a..00000000000 Binary files a/docs/source/AdministratorGuide/Systems/WorkloadManagement/PilotsLogging/PilotsLoggingDiagram.png and /dev/null differ diff --git a/docs/source/AdministratorGuide/Systems/WorkloadManagement/PilotsLogging/index.rst b/docs/source/AdministratorGuide/Systems/WorkloadManagement/PilotsLogging/index.rst deleted file mode 100644 index f2efac53728..00000000000 --- a/docs/source/AdministratorGuide/Systems/WorkloadManagement/PilotsLogging/index.rst +++ /dev/null @@ -1,74 +0,0 @@ -.. contents:: Table of contents - :depth: 3 - -======================================================================================= - -======================================================================================= - -============================== -Pilots Logging system overview -============================== - -Pilots Loggins system is designed to allow logging of pilot state on every stage of lifecycle, including before installing -DIRAC client and starting pilot process. - -Each logging entry includes: - -- current status of the Pilot - has to be one of predefined list of possible states, -- additional information about status, -- timestamp of logging the status - if there is no timestamp of actual event provided, time of adding entry to database will be used, -- source of the logging message to distinguish updates from Pilot itself and other services. - -.. image:: PilotsLoggingDiagram.png - :alt: PilotsLogging system - :align: center - - -Server side -================================ - -Server elements of Pilots Logging system is build using five elements: - -- message queue (RabbitMQ) server, -- message queue consumer, -- DIRAC Client, -- DIRAC Service, -- database. - -Message queue --------------------------------- - -Message works as a interface between Pilot and Pilots Logging service. Pilot puts status related messages into queue then -messages are handled by message queue consumer. - -Message queue consumer --------------------------------- - -Consumer registers itself into message queue. When new messages arrive they are handled by callback function. In consumer -messages are processed and passed to DIRAC Service using DIRAC Client. - -DIRAC Client --------------------------------- - -Client handles RPC communication with Service. This is 'thin-client', all business logic is in Service. - -DIRAC Service --------------------------------- - -Service exports functions to be called by Clients. It handles all operations on databases. All server side logic of -Pilots Logging system is defined here. Two databases are accessed to gather all required information. - -Database --------------------------------- - -Database class handles operation on the database. Object-relational mapping is done using SQLAlchemy. Single table stores -record for every status reported by Pilot: - -.. image:: PilotsLoggingDB.png - :alt: PilotsLogging database schema - :align: center - -Pilot side -================================ - -TBD diff --git a/docs/source/AdministratorGuide/Systems/WorkloadManagement/VMDIRAC.rst b/docs/source/AdministratorGuide/Systems/WorkloadManagement/VMDIRAC.rst deleted file mode 100644 index 84277256fd9..00000000000 --- a/docs/source/AdministratorGuide/Systems/WorkloadManagement/VMDIRAC.rst +++ /dev/null @@ -1,309 +0,0 @@ -.. _installingVMDIRAC: - -================== -Installing VMDIRAC -================== - -.. toctree:: - :maxdepth: 2 - -.. contents:: Table of contents - :depth: 4 - ------------- -Introduction ------------- - -VMDIRAC has now been merged into the main DIRAC release (as of v7r3/7.3). It -consists of the following parts: - -VirtualMachineDB (Database) - This is the database that stores the details of each VM -managed by VMDIRAC. There should be one instance of this on your system. - -CloudDirector (Agent) - This is analogous to the core DIRAC SiteDirector. It inspects -the TaskQueues, works out if there are any compatible jobs waiting (without -existing VMs) for cloud resources and starts VM instances as needed. The VM -details are stored in the database for future reference. You need at least one -CloudDirector, for multi community/VO services, it's advisable to have one -CloudDirector per community. - -VirtualMachineManager (Service) - This service provides the virtual machine -life-cycle management interface; the CLI tools contact this service to -list/kill running VMs. It also has an inbuilt thread that will inspect existing -VMs, tidying up any that are stopped or haven't reported back for a long time. - -VirtualMachineMonitor (Agent) - This agent monitors the health of the current -VM it is running on an reports it back to the VirtualMachineManager. It should -not be installed on the central DIRAC instance, but started in the cloud VM -instances themselves. This lets the VirtualMachineManager know that the -instance is still alive and also handles stopping the instance after any -tasks/jobs have finished. - -VMDIRAC WebApp Extention - This WebApp plugin adds an extra VirtualMachines tab -to the usual DIRAC webinterface for summarising the contents of the database. - ---------------- -Install VMDIRAC ---------------- - -On a DIRAC server (generally collocated with the other WorkloadManagement -system components), configure and install the extra components: - -* Resource configuration - - In the configuration add the User/Password information to connect to the Cloud endpoint. Also you should put a valid path for the host certificate, *e.g.*: - - :: - - Resources - { - Sites - { - Cloud - { - Cloud.LUPM.fr - { - Cloud - { - example.cloud.hostname - { - User = xxxx - Password = xxxx - HostCert = /opt/dirac/etc/grid-security/hostcert.pem - HostKey = /opt/dirac/etc/grid-security/hostkey.pem - CreatePublicIP = True - } - } - } - } - } - } - -* Install the following components: - - * DB: VirtualMachineDB - * Service: WorkloadManagement_VirtualMachineManager - * Agent: WorkloadManagement_CloudDirector - ------------------------------------- -Setup for using cloudinit and Pilot3 ------------------------------------- - -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Using OpenStack with an application credential -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -* The user must have access to the OpenStack cloud and be allowed to start up instances. -* Login to the cloud webinterface with your account. -* Go to the identity -> Application credentials panel -* Create a new credential with any name, no expire and no roles selected -* Copy the ID and secret strings somewhere safe for a moment -* Put the ID and string into a new file on the DIRAC server running the - CloudDirectors in the following format (one line, separated by a space): - -* Make sure the file is owned by the user running dirac and has 0600 permissions. -* Add the location of this file to the Resource Settings. - -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -OpenStack Resource Configuration -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -On the OpenStack Resource you will need the following: - -* user account -* user's ssh key uploaded to the OpenStack server: Associated with the instance for debugging. - This key is user specific, not project specific. -* flavour (ID or name) -* image (ID or name) -* network (ID or name) - - :: - - CLOUD.ExampleName.uk - { - Name = [ExampleName] - # must be unique, use e.g. hostname of the OpenStack webinterface - CE = [hostname.example.ac.uk] - Cloud - { - [hostname.example.ac.uk] - { - # assuming your cloud is using a standard CA - CAPath = /etc/pki/tls/cert.pem - # list your favourite VOs here - VO = gridpp - VO += lz - NetworkID = [network uuid] - Network = [name_of_network] - CEType = OpenStack - MaxInstances = [maximum number of instances] - AuthURL = [https://keystone.example.ac.uk:5000/v3] - Appcred = [path to appcred file created earlier] - # this might be optional - CVMFSProxy = http://[your cvmfs proxy cache]:3128 - Images - { - [image name, e.g. CentOS-7-x86_64-GenericCloud-1905] - { - ImageID = [image uuid] - FlavorName = [flavour name] - # this is currently a dummy value - Platform = [DIRACPlatForm] - } - } - OSKeyName = [ssh key name of the OpenStack user] - Tenant = [Openstack Project Name] - } - } - - CEs - { - [hostname.example.ac.uk] - { - CEType = Cloud - Architecture = x86_64 - Queues - { - [image name] - { - maxCPUTime = 24000000 - } - } - } - } - } - ------------------------------- -Configuration - other examples ------------------------------- - -* In the CS Resources section, configure the cloud endpoint as in this example - - :: - - Resources - { - Sites - { - Cloud - { - Cloud.LUPM.fr - { - CE = 194.214.86.244 - Cloud - { - 194.214.86.244 - { - CEType = Cloud - ex_security_groups = default - ex_force_auth_url = http://194.214.86.244:5000/v3/auth/tokens - ex_force_service_region = LUPM-CLOUD - # This is the max number of VM instances that will be running in parallel - # Each VM can have multiple cores, each one executing a job - MaxInstances = 4 - ex_force_auth_version = 3.x_password - ex_tenant_name = dirac - ex_domain_name = msfg.fr - networks = dirac-net - # This is the public key previously uploaded to the Cloud provider - # It's needed to ssh connect to VMs - keyname = cta_cloud_lupm - # If this option is set, public IP are assigned to VMs - # It's needed to ssh connect to VMs - ipPool = ext-net - } - Images - { - # It can be a public or a private image - Centos6-Officielle - { - ImageID = 35403255-f5f1-4c61-96dc-e59678942c6d - FlavorName = m1.medium - } - } - } - } - } - } - } - - -* CS Operation section - - :: - - Operations - { - CTA - { - Cloud - { - GenericCloudGroup = cta_genpilot - GenericCloudUser = arrabito - user_data_commands = vm-bootstrap - user_data_commands += vm-bootstrap-functions - user_data_commands += vm-pilot - user_data_commands += vm-monitor-agent - user_data_commands += pilotCommands.py - user_data_commands += pilotTools.py - user_data_commands += power.sh - user_data_commands += parse-jobagent-log - user_data_commands += dirac-pilot.py - user_data_commands += save-payload-logs - # url from which command scripts are downloaded. Usually the url of the web server - user_data_commands_base_url = http://cta-dirac.in2p3.fr/DIRAC/defaults - Project = CTA - Version = v1r40 - } - } - } - -* CS Registry section - - The host where VMDIRAC is installed and the certificate of which is used for the VMs, it should have these 2 properties set (as in the example below): - - * Properties = GenericPilot (needed to make pilots running on the VM matching jobs in the TaskQueue) - * Properties = VmRpcOperation (needed by the VirtualMachineMonitorAgent running on the VM to be authorized to send Heartbeats to the VirtualMachineManager service) - - :: - - Registry - { - Hosts - { - dcta-agents01.pic.es - { - DN = /DC=org/DC=terena/DC=tcs/C=ES/ST=Barcelona/L=Bellaterra/O=Institut de Fisica dAltes Energies/CN=dcta-agents01.pic.es - CA = /C=NL/ST=Noord-Holland/L=Amsterdam/O=TERENA/CN=TERENA eScience SSL CA 3 - Properties = FullDelegation - Properties += CSAdministrator - Properties += ProxyManagement - Properties += SiteManager - Properties += Operator - Properties += JobAdministrator - Properties += CSAdministrator - Properties += TrustedHost - Properties += GenericPilot - Properties += VmRpcOperation - } - } - } - ----------------------- -Install VMDIRAC WebApp ----------------------- - -* The VirtualMachines panel is now included in the main WebApp release. -* If using the old (non cloudinit) bootstrap, you must Create sym links for the - old bootstrap scripts: - - :: - - $ ll /opt/dirac/webRoot/www/defaults/bootstrap - total 0 - lrwxrwxrwx 1 dirac dirac 76 Feb 21 08:50 parse-jobagent-log -> /opt/dirac/pro/DIRAC/WorkloadManagementSystem/Utilities/CloudBootstrap/parse-jobagent-log - lrwxrwxrwx 1 dirac dirac 66 Feb 21 08:52 power.sh -> /opt/dirac/pro/DIRAC/WorkloadManagementSystem/CloudBootstrap/power.sh - lrwxrwxrwx 1 dirac dirac 75 Feb 21 08:52 save-payload-logs -> /opt/dirac/pro/DIRAC/WorkloadManagementSystem/CloudBootstrap/save-payload-logs - lrwxrwxrwx 1 dirac dirac 70 Feb 21 11:47 vm-bootstrap -> /opt/dirac/pro/DIRAC/WorkloadManagementSystem/CloudBootstrap/vm-bootstrap - lrwxrwxrwx 1 dirac dirac 80 Feb 21 08:52 vm-bootstrap-functions -> /opt/dirac/pro/DIRAC/WorkloadManagementSystem/CloudBootstrap/vm-bootstrap-functions - lrwxrwxrwx 1 dirac dirac 74 Feb 21 08:53 vm-monitor-agent -> /opt/dirac/pro/DIRAC/WorkloadManagementSystem/CloudBootstrap/vm-monitor-agent - lrwxrwxrwx 1 dirac dirac 66 Feb 21 08:53 vm-pilot -> /opt/dirac/pro/DIRAC/WorkloadManagementSystem/CloudBootstrap/vm-pilot diff --git a/docs/source/AdministratorGuide/Systems/WorkloadManagement/architecture.rst b/docs/source/AdministratorGuide/Systems/WorkloadManagement/architecture.rst index b8e26be1637..a575146a806 100644 --- a/docs/source/AdministratorGuide/Systems/WorkloadManagement/architecture.rst +++ b/docs/source/AdministratorGuide/Systems/WorkloadManagement/architecture.rst @@ -25,14 +25,7 @@ SandboxMetadataDB TaskQueueDB The TaskQueueDB is used to organize jobs requirements into task queues, for easier matching. -All the DBs above are MySQL DBs, and should be installed using the :ref:`system administrator console `. - - -.. versionadded:: v7r0 - The JobDB MySQL table *JobParameters* can be replaced by an JobParameters backend built in ElasticSearch. - To enable it, set the following flag:: - - /Operations/[Defaults | Setup]/Services/JobMonitoring/useESForJobParametersFlag=True +All the DBs above are MySQL DBs with the only exception of the Elastic/OpenSearch backend for storing job parameters. Services @@ -81,12 +74,8 @@ All these agents are necessary for the WMS, and each of them should be installed You can duplicate some of these agents as long as you provide the correct configuration. A typical example is the SiteDirector, for which you may want to deploy even 1 for each of the sites managed. -Optional agents are: - -StatesAccountingAgent or StatesMonitoringAgent - Use one or the other. - StatesMonitoringAgent is used for producing Monitoring plots through the :ref:`Monitoring System `. (so, using ElasticSearch as backend), - while StatesAccountingAgent does the same job but using the Accounting system (so, MySQL as backend). +StatesAccountingAgent. + Used for producing Monitoring plots through the :ref:`Monitoring System `, or using the Accounting system (so, MySQL as backend). A very different type of agent is the *JobAgent*, which is run by the pilot jobs and should NOT be run in a server installation. diff --git a/docs/source/AdministratorGuide/Systems/WorkloadManagement/index.rst b/docs/source/AdministratorGuide/Systems/WorkloadManagement/index.rst index 22d1e0bdbcf..2b0ab2319a7 100644 --- a/docs/source/AdministratorGuide/Systems/WorkloadManagement/index.rst +++ b/docs/source/AdministratorGuide/Systems/WorkloadManagement/index.rst @@ -83,10 +83,9 @@ The following sections add some detail for the WMS systems. architecture Pilots/index Pilots/Pilots3 - PilotsLogging/index Jobs/index JobsPriorities JobsMatching tagsAndJobs multiProcessorJobs - VMDIRAC + InputDataResolution diff --git a/docs/source/AdministratorGuide/Systems/WorkloadManagement/tagsAndJobs.rst b/docs/source/AdministratorGuide/Systems/WorkloadManagement/tagsAndJobs.rst index 6c1e087cca4..1e85686cb96 100644 --- a/docs/source/AdministratorGuide/Systems/WorkloadManagement/tagsAndJobs.rst +++ b/docs/source/AdministratorGuide/Systems/WorkloadManagement/tagsAndJobs.rst @@ -35,7 +35,6 @@ Let's take an example:: maxCPUTime = 200 MaxTotalJobs = 5 MaxWaitingJobs = 10 - BundleProxy = True RemoveOutput = True } # This queue has Tag = GPU. So it will accept: diff --git a/docs/source/AdministratorGuide/Tutorials/basicTutoSetup.rst b/docs/source/AdministratorGuide/Tutorials/basicTutoSetup.rst deleted file mode 100644 index bbecc895594..00000000000 --- a/docs/source/AdministratorGuide/Tutorials/basicTutoSetup.rst +++ /dev/null @@ -1,528 +0,0 @@ -.. _tuto_basic_setup: - -==================== -Basic Tutorial setup -==================== - -.. set highlighting to console input/output -.. highlight:: console - -Tutorial goal -============= - -The aim of the tutorial is to have a self contained DIRAC setup. You will be guided through the whole installation process both of the server part and the client part. -By the end of the tutorial, you will have: - -* a Configuration service, to serve other servers and clients -* a ComponentMonitoring service to keep track of other services and agents installed -* a SystemAdministrator service to manage the DIRAC installation in the future -* the WebApp, to allow for web interface access - -The setup you will have at the end is the base for all the other tutorials. - - -More links -========== - -* :ref:`server_installation` - -Basic requirements -================== - -This section is to be executed as ``root`` user. - -In this tutorial, we will use a freshly installed CC7 x86_64 virtual machine, with all the default options, except the hostname being ``dirac-tuto``. Make sure that the hostname of the machine is set to ``dirac-tuto``. Modify the ``HOSTNAME`` variable in the ``/etc/sysconfig/network`` file as such:: - - HOSTNAME=dirac-tuto - -Then reboot the machine and check the hostname. You should get the following output:: - - [root@dirac-tuto ~]# hostname - dirac-tuto - - -Machine setup -============= - -This section is to be executed as ``root`` user. - -Make sure that the machine can address itself using the ``dirac-tuto`` alias. Modify the ``/etc/hosts`` file as such:: - - 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 dirac-tuto - ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 dirac-tuto - - -------------------------- -Create the ``dirac`` user -------------------------- - -The user that will run the server will be ``dirac``. Set the password for that user to ``password``, -and ensure that files below ``/opt/dirac/`` belong to this user: - -.. literalinclude:: basicTutoSetup.sh - :language: bash - :start-after: # START add_dirac - :end-before: # END add_dirac - -------------- -Install runit -------------- - -The next step is to install ``runit``, which is responsible for supervising DIRAC processes - -First, install the `RPM `_: - -.. literalinclude:: basicTutoSetup.sh - :language: bash - :start-after: # START runit - :end-before: # END runit - - -Create the file ``/opt/dirac/sbin/runsvdir-start``, which is responsible for starting runit, with the following content: - -.. literalinclude:: basicTutoSetup.sh - :language: bash - :start-after: # START runsvdir-start - :end-before: # END runsvdir-start - :caption: /opt/dirac/sbin/runsvdir-start - -Then, edit the systemd ``runsvdir-start`` service to match the following: - -.. literalinclude:: basicTutoSetup.sh - :language: bash - :start-after: # START systemd-runsvdir - :end-before: # END systemd-runsvdir - :caption: /usr/lib/systemd/systemd/runsvdir-start.service - -make ``runsvdir-start`` executable and (re)start ``runsvdir``: - -.. literalinclude:: basicTutoSetup.sh - :language: bash - :start-after: # START restartrunsv - :end-before: # END restartrunsv - - -------------- -Install MySQL -------------- - -First of all, remove the existing (outdated) installation, and install all the necessary RPMs for MySQL 5.7: - -.. literalinclude:: basicTutoSetup.sh - :language: bash - :start-after: # START mysqlInstall - :end-before: # END mysqlInstall - -Start the mysql service, which will then initialize itself, and, among other things, create temporary password for the -mysql ``root`` account, which needs to be changed during the first login: - -.. literalinclude:: basicTutoSetup.sh - :language: bash - :start-after: # START mysqlStart - :end-before: # END mysqlStart - -To change the root password, create a ``mysqlSetup.sql`` file, which changes the password to a strong password, removes -a plugin to enforce the strong password (only for tutorial purposes, of course), and then sets the password to -``password``, which is easier to remember: - -.. literalinclude:: basicTutoSetup.sh - :language: mysql - :start-after: # START mysqlSetup - :end-before: # END mysqlSetup - :caption: mysqlSetup.sql - -Now get the temporary password from the ``/var/log/mysqld.log``, and change it using the ``mysqlSetup.sql`` file: - -.. literalinclude:: basicTutoSetup.sh - :language: bash - :start-after: # START mysqlInit - :end-before: # END mysqlInit - -Server installation -=================== - -This section is to be executed as ``dirac`` user - ------------------- -CA and certificate ------------------- - -DIRAC relies on TLS for securing its connections and for authorization and authentication. Since we are using a self contained installation, we will be using our own CA. There are a bunch of utilities that we will be using to generate the necessary files. - -We create a script ``setupCA`` to download utilities from the DIRAC repository and source ``utilities.sh``, and then -create the CA and certificates both for the server and the client: - -.. literalinclude:: basicTutoSetup.sh - :language: bash - :start-after: # START setupCA - :end-before: # END setupCA - :caption: setupCA - -Execute the script:: - - bash setupCA - -At this point, you should find: - -* The CA in ``/opt/dirac/etc/grid-security/certificates``:: - - [dirac@dirac-tuto caUtilities]$ ls /opt/dirac/etc/grid-security/certificates/ - 855f710d.0 ca.cert.pem - -* The host certificate (``hostcert.pem``) and key (``hostkey.pem``) in ``/opt/dirac/etc/grid-security``:: - - [dirac@dirac-tuto caUtilities]$ ls /opt/dirac/etc/grid-security/ - ca certificates hostcert.pem hostkey.pem openssl_config_host.cnf request.csr.pem - -* The user credentials for later in ``/opt/dirac/user/``:: - - [dirac@dirac-tuto caUtilities]$ ls /opt/dirac/user/ - client.key client.pem client.req openssl_config_user.cnf - --------------------- -Install DIRAC Server --------------------- - -This section is to be run as ``dirac`` user in its home folder:: - - sudo su dirac - cd ~ - -First we create the ``install.cfg`` file, which is used to tell the installation script we obtain in a moment what to -install and how to configure the server with the following content: - -.. literalinclude:: basicTutoSetup.sh - :language: bash - :start-after: # START install.cfg - :end-before: # END install.cfg - :caption: install.cfg - -Then we download the installer, make it executable, and run it with the ``install.cfg`` file (assuming the file is in -the user's home folder): - -.. literalinclude:: basicTutoSetup.sh - :language: bash - :start-after: # START installDirac - :end-before: # END installDirac - - -The output should look something like this:: - - Status of installed components: - - Name Runit Uptime PID - ================================================= - 1 Web_WebApp Run 4 24338 - 2 Configuration_Server Run 53 24142 - 3 Framework_ComponentMonitoring Run 36 24207 - 4 Framework_SystemAdministrator Run 20 24247 - - -You can verify that the components are running:: - - [dirac@dirac-tuto DIRAC]$ runsvstat /opt/dirac/startup/* - /opt/dirac/startup/Configuration_Server: run (pid 24142) 288 seconds - /opt/dirac/startup/Framework_ComponentMonitoring: run (pid 24207) 271 seconds - /opt/dirac/startup/Framework_SystemAdministrator: run (pid 24247) 255 seconds - /opt/dirac/startup/Web_WebApp: run (pid 24338) 239 seconds - - -The logs are to be found in ``/opt/dirac/runit/``, grouped by component. - -The installation created the file ``/opt/dirac/etc/dirac.cfg``. The content is the same as the ``install.cfg``, with the addition of the following:: - - DIRAC - { - Setup = MyDIRAC-Production - VirtualOrganization = tutoVO - Extensions = WebApp - Security - { - } - Setups - { - MyDIRAC-Production - { - Configuration = Production - Framework = Production - } - } - Configuration - { - Master = yes - Name = MyDIRAC-Production - Servers = dips://dirac-tuto:9135/Configuration/Server - } - } - LocalSite - { - Site = dirac-tuto - } - Systems - { - Databases - { - User = Dirac - Password = Dirac - Host = localhost - Port = 3306 - } - NoSQLDatabases - { - Host = dirac-tuto - Port = 9200 - } - } - -This part is used as configuration for all your services and agents that you will run. It contains two important information: - -* The database credentials -* The address of the configuration server: ``Servers = dips://dirac-tuto:9135/Configuration/Server`` - -The Configuration service will serve the content of the file ``/opt/dirac/etc/MyDIRAC-Production.cfg`` to every client, be it a service, an agent, a job, or an interactive client. The content looks like such:: - - DIRAC - { - Extensions = WebApp - VirtualOrganization = tutoVO - Configuration - { - Name = MyDIRAC-Production - Version = 2019-04-11 06:52:18.414086 - MasterServer = dips://dirac-tuto:9135/Configuration/Server - } - Setups - { - MyDIRAC-Production - { - Configuration = Production - Framework = Production - } - } - } - Registry - { - Users - { - ciuser - { - DN = /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - Email = adminUser@cern.ch - } - } - Groups - { - dirac_user - { - Users = ciuser - Properties = NormalUser - } - dirac_admin - { - Users = ciuser - Properties = AlarmsManagement - Properties += ServiceAdministrator - Properties += CSAdministrator - Properties += JobAdministrator - Properties += FullDelegation - Properties += ProxyManagement - Properties += Operator - } - } - Hosts - { - dirac-tuto - { - DN = /C=ch/O=DIRAC/OU=DIRAC CI/CN=dirac-tuto - Properties = TrustedHost - Properties += CSAdministrator - Properties += JobAdministrator - Properties += FullDelegation - Properties += ProxyManagement - Properties += Operator - } - } - DefaultGroup = dirac_user - } - Operations - { - Defaults - { - EMail - { - Production = adminUser@cern.ch - Logging = adminUser@cern.ch - } - } - } - WebApp - { - Access - { - upload = TrustedHost - } - } - Systems - { - Framework - { - Production - { - Services - { - ComponentMonitoring - { - Port = 9190 - Authorization - { - Default = ServiceAdministrator - componentExists = authenticated - getComponents = authenticated - hostExists = authenticated - getHosts = authenticated - installationExists = authenticated - getInstallations = authenticated - updateLog = Operator - } - } - SystemAdministrator - { - Port = 9162 - Authorization - { - Default = ServiceAdministrator - storeHostInfo = Operator - } - } - } - URLs - { - ComponentMonitoring = dips://dirac-tuto:9190/Framework/ComponentMonitoring - SystemAdministrator = dips://dirac-tuto:9162/Framework/SystemAdministrator - } - FailoverURLs - { - } - Databases - { - InstalledComponentsDB - { - DBName = InstalledComponentsDB - Host = localhost - Port = 3306 - } - } - } - } - } - - -This configuration will be used for example by Services in order to: - -* know their configuration (for example the ``ComponentMonitoring`` Service will use everything under ``Systems/Framework/Production/Services/ComponentMonitoring`` ) -* Identify host and persons (``Registry`` section) - -Or by clients to get the URLs of given services (for example ``ComponentMonitoring = dips://dirac-tuto:9190/Framework/ComponentMonitoring``) - -Since this configuration is given as a whole to every client, you understand why no database credentials are in this file. Services and Agents running on the machine will have their configuration as a merge of what is served by the Configuration service and the ``/opt/dirac/etc/dirac.cfg``, and thus have access to these private information. - -The file ``/opt/dirac/bashrc`` is to be sourced whenever you want to use the server installation. - -Client installation -=================== - -Now we will create another linux account ``diracuser`` and another installation to be used as client - --------------------- -Setup client session --------------------- - -This section has to be ran as ``root`` - -Create an account ``diracuser`` with password ``password``, and add in its ``~/.globus/`` directory the user -certificate you created earlier: - -.. literalinclude:: basicTutoSetup.sh - :language: bash - :start-after: # START user_diracuser - :end-before: # END user_diracuser - - --------------------- -Install DIRAC client --------------------- - -This section has to be ran as ``diracuser`` in its home directory:: - - sudo su diracuser - cd - -We will do the installation in the ``~/DiracInstallation`` directory. For a client, the configuration is really minimal, -so we will just install the code and its dependencies. Create the structure, download the installer, and then install -the same version as for the server: - -.. literalinclude:: basicTutoSetup.sh - :language: bash - :start-after: # START installClient1 - :end-before: # END installClient1 - -In principle, your system administrator will have managed the CA for you. In this specific case, since we have our own CA, we will just link the client installation CA with the server one: - -.. literalinclude:: basicTutoSetup.sh - :language: bash - :start-after: # START installClient2 - :end-before: # END installClient2 - -The last step is to configure the client to talk to the proper configuration service. This is easily done by creating a ``~/DiracInstallation/etc/dirac.cfg`` file with the following content: - -.. literalinclude:: basicTutoSetup.sh - :language: bash - :caption: ~/DiracInstallation/etc/dirac.cfg - :start-after: # START dirac.cfg - :end-before: # END dirac.cfg - -You should now be able to get a proxy:: - - [diracuser@dirac-tuto DIRAC]$ source ~/DiracInstallation/bashrc - [diracuser@dirac-tuto DIRAC]$ dirac-proxy-init - Generating proxy... - Proxy generated: - subject : /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser/CN=460648814 - issuer : /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - identity : /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - timeleft : 23:59:59 - DIRAC group : dirac_user - rfc : True - path : /tmp/x509up_u501 - username : ciuser - properties : NormalUser - - -And you can observe that the Configuration Service has served the client:: - - [diracuser@dirac-tuto DIRAC]$ grep ciuser /opt/dirac/runit/Configuration/Server/log/current - 2019-04-11 14:54:10 UTC Configuration/Server NOTICE: Executing action ([::1]:33394)[dirac_user:ciuser] RPC/getCompressedDataIfNewer() - 2019-04-11 14:54:10 UTC Configuration/Server NOTICE: Returning response ([::1]:33394)[dirac_user:ciuser] (0.00 secs) OK - --------------- -Use the WebApp --------------- - -This section is to be executed as ``diracuser``. - -First you need to convert your user certificate into a ``p12`` format (you will be prompt for a password, you can leave it empty):: - - cd ~/.globus/ - openssl pkcs12 -export -out certificate.p12 -inkey userkey.pem -in usercert.pem - -This will create the file ``~/.globus/certificate.p12``. - -Use your favorite browser, and add this certificate. - -You should be able to access the WebApp using the following address ``https://localhost:8443/DIRAC/`` - - -Conclusion -========== - -We have seen how to install a DIRAC server and client using a personal CA, and how to access the WebApp. Starting from here, you will be able to extend on further tutorials. diff --git a/docs/source/AdministratorGuide/Tutorials/basicTutoSetup.sh b/docs/source/AdministratorGuide/Tutorials/basicTutoSetup.sh deleted file mode 100644 index 865a06d2ad5..00000000000 --- a/docs/source/AdministratorGuide/Tutorials/basicTutoSetup.sh +++ /dev/null @@ -1,259 +0,0 @@ -#!/bin/bash - -set -euv - -# this doesn't work on cern SL6 machines in openstack, there we still have `hostname` -# Sed the /etc/hosts file to append dirac-tuto -sed -i "s/\(127.*\)/\1 dirac-tuto/" /etc/hosts -sed -i "s/\(::1.*\)/\1 dirac-tuto/" /etc/hosts - -# on cern SL6 image on openstack just use HOSTNAME -HOSTNAME=`hostname` - - -# avoid calling passwd, which might be the cern passwd in /usr/sue/bin -# START add_dirac -adduser -s /bin/bash -d /home/dirac dirac || echo "User dirac already exists." -echo password | /usr/bin/passwd --stdin dirac -mkdir -p /opt/dirac/sbin -chown -R dirac:dirac /opt/dirac/ -# END add_dirac - - -#localinstall does not error when rpm is already installed -# START runit -yum localinstall -y http://diracproject.web.cern.ch/diracproject/rpm/runit-2.1.2-1.el7.cern.x86_64.rpm -# END runit - -cat > /opt/dirac/sbin/runsvdir-start <<'EOF' -# START runsvdir-start -#!/bin/bash -cd /opt/dirac -RUNSVCTRL='/sbin/runsvctrl' -chpst -u dirac $RUNSVCTRL d /opt/dirac/startup/* -killall runsv svlogd -RUNSVDIR='/sbin/runsvdir' -exec chpst -u dirac $RUNSVDIR -P /opt/dirac/startup 'log: DIRAC runsv' -# END runsvdir-start -EOF - -# runsvdir-start can fail to start/restart if it does not contain the shebang at the top of the file -# we remove the first line of the script -sed -i '1d' /opt/dirac/sbin/runsvdir-start - -cat > /lib/systemd/system/runsvdir-start.service < mysqlSetup.sql < install.cfg < setupCA < installDirac < InstallDiracClient < ~diracuser/DiracInstallation/etc/dirac.cfg < /tmp/dummy.txt - -Now create a file called ``/tmp/testSE.py``, with the following content - -.. code-block:: python - - import DIRAC - - DIRAC.initialize() # Initialize configuration - - localFile = '/tmp/dummy.txt' - lfn = '/tutoVO/myFirstFile.txt' - - from DIRAC.Resources.Storage.StorageElement import StorageElement - - - se = StorageElement('StorageElementOne') - - print "Putting file" - print se.putFile({lfn: localFile}) - - print "Listing directory" - print se.listDirectory('/tutoVO') - - print "Getting file" - print se.getFile(lfn, '/tmp/') - - print "Removing file" - print se.removeFile(lfn) - - print "Listing directory" - print se.listDirectory('/tutoVO') - - - - -This script uploads ``/tmp/dummy.txt`` on the StorageElement as ``myFirstFile.txt``, list the directory, downloads the uploaded file and removes it from the StorageElement. The output should be something like that:: - - [diracuser@dirac-tuto ~]$ python /tmp/testSE.py - Putting file - {'OK': True, 'Value': {'Successful': {'/tutoVO/myFirstFile.txt': 10}, 'Failed': {}}} - Listing directory - {'OK': True, 'Value': {'Successful': {'/tutoVO': {'Files': {'myFirstFile.txt': {'Accessible': True, 'Migrated': 0, 'Unavailable': 0, 'Lost': 0, 'Exists': True, 'Cached': 1, 'Checksum': '166203b7', 'Mode': 420, 'File': True, 'Directory': True, 'TimeStamps': (1555342476, 1555342476, 1555342476), 'Type': 'File', 'Size': 10}}, 'SubDirs': {}}}, 'Failed': {}}} - Getting file - {'OK': True, 'Value': {'Successful': {'/tutoVO/myFirstFile.txt': 10}, 'Failed': {}}} - Removing file - {'OK': True, 'Value': {'Successful': {'/tutoVO/myFirstFile.txt': True}, 'Failed': {}}} - Listing directory - {'OK': True, 'Value': {'Successful': {'/tutoVO': {'Files': {}, 'SubDirs': {}}}, 'Failed': {}}} - -The list of files within ``tmp`` should also contain ``dummy.txt`` as well as ``myFirstFile.txt``. - -Adding a second DIRAC SE -======================== - -It is often interesting to have a second SE. - -As ``dirac`` user, create a new directory:: - - [dirac@dirac-tuto ~]$ mkdir /opt/dirac/storageElementTwo/ - -Now the rest is to be installed with ``diracuser`` and a proxy with ``dirac_admin`` group. - -We need another StorageElement service. However, it has to have a different *name*, *Port* and *BasePath* than the first one, so we will just call this service ``StorageElementTwo``:: - - [diracuser@dirac-tuto ~]$ dirac-admin-sysadmin-cli --host dirac-tuto - Pinging dirac-tuto... - [dirac-tuto]$ install service DataManagement StorageElementTwo -m StorageElement -p Port=9147 -p BasePath=/opt/dirac/storageElementTwo/ - Loading configuration template /home/diracuser/DIRAC/DIRAC/DataManagementSystem/ConfigTemplate.cfg - Adding to CS service DataManagement/StorageElementTwo - service DataManagement_StorageElementTwo is installed, runit status: Run - - -Using the WebApp, add the new StorageElement definition in the ``/Resources/StorageElements`` section:: - - StorageElementTwo - { - BackendType = DISET - DIP - { - Host = dirac-tuto - Port = 9147 - Protocol = dips - Path = /DataManagement/StorageElementTwo - Access = remote - } - } - - -In order to test it, just re-use ``/tmp/testSE.py``, replacing ``StorageElementOne`` with ``StorageElementTwo`` diff --git a/docs/source/AdministratorGuide/Tutorials/dmsWithTs.rst b/docs/source/AdministratorGuide/Tutorials/dmsWithTs.rst deleted file mode 100644 index 2ce51e9f109..00000000000 --- a/docs/source/AdministratorGuide/Tutorials/dmsWithTs.rst +++ /dev/null @@ -1,281 +0,0 @@ -========================================================= -Large Scale DataManagement with the Transformation System -========================================================= - -Pre-Requisite -============= - -You should: - -* have a machine setup as described in :ref:`tuto_basic_setup` -* have installed two DIRAC SE using the tutorial (:ref:`tuto_install_dirac_se`). -* have installed the DFC using the tutorial (:ref:`tuto_install_dfc`). -* have followed the tutorial on identity management (:ref:`tuto_managing_identities`) -* have installed the RMS using the tutorial (:ref:`tuto_install_rms`) -* have installed the TS using the tutorial (:ref:`tuto_install_ts`) - - -Tutorial Goal -============= - -The aim of the tutorial is to demonstrate how large scale data management operations (removals, replications, etc.) can -be achieved using the Transformation System. By the end of the tutorial, you will be able to: - -* Submit simple transformation for manipulating a given list of files -* Have transformations automatically fed thanks to metadata -* Write your own plugin for the TransformationSystem - -The transformations can be monitored and controlled with the ``Transformation Monitor`` in the ``WebApp`` when you use -the ``dirac_prod`` group. - - -More Links -========== - -* :ref:`adminTS` - - -Creating a Transformation with a DIRAC Command -============================================== - -.. highlight:: console - -This section is to be performed as ``diracuser`` with a proxy in ``dirac_prod`` group. - -First we need to create some files and upload them to ``StorageElementOne``:: - - [diracuser@dirac-tuto ~]$ for ID in {1..10}; do echo "MyContent $ID" > File_${ID} ; dirac-dms-add-file /tutoVO/data/Trans_01/File_${ID} File_${ID} StorageElementOne ; done - -Then we create the list of LFNs we just uploaded:: - - [diracuser@dirac-tuto ~]$ dirac-dms-find-lfns Path=/tutoVO/data/Trans_01 > trans01.lfns - -The easiest way to create a transformation to replicate files is by using the :ref:`dirac-transformation-replication` command:: - - [diracuser@dirac-tuto ~]$ dirac-transformation-replication 0 StorageElementTwo --Plugin Broadcast --Enable - Created transformation NNN - Successfully created replication transformation - -This created transformation with the unique transformation ID *NNN* (e.g., 1). - -By default this transformation uses *Metadata* information to obtain the input files using the -``InputDataAgent``. Instead we can also just add files manually with the :ref:`dirac-transformation-add-files` command and using the list we created previously, -replace NNN by the ID of the transformation that was just created:: - - [diracuser@dirac-tuto ~]$ dirac-transformation-add-files NNN trans01.lfns - Successfully added 10 files - - -Now we have to wait until the ``TransformationAgent`` runs again and creates a *Task* for each of the files. Once the -tasks are created, the ``RequestTaskAgent`` creates a request out of each task, which is then processed in the -``RequestExecutingAgent`` of the RMS. - - -Creating a Transformation with a Script -======================================= - - -In this step we want to remove the replicas of our files from ``StorageElementOne``, for this purpose we have to write a -script that creates a removal transformation: - -.. code-block:: python - :caption: createRemoval.py - :linenos: - - #!/bin/env python - - # set up the DIRAC configuration, parse command line arguments - from DIRAC import gLogger, S_OK, S_ERROR - from DIRAC.Core.Base.Script import Script - Script.parseCommandLine() - - from DIRAC.TransformationSystem.Client.Transformation import Transformation - - # create a Transformation instance - myTrans = Transformation() - - # transformation names need to be unique - uniqueIdentifier = "Trans1" - transformationName = "RemoveReplicas_%s" % uniqueIdentifier - myTrans.setTransformationName(transformationName) - - # describe what the transformation will do - description = "Remove replicas from StorageElementOne" - myTrans.setDescription(description) - myTrans.setLongDescription(description) - - # 'Replication' type means we do data management - myTrans.setType('Removal') - - # group transformations that belong together, these can be selected in the WebApp - transGroup = "myRemovals" - myTrans.setTransformationGroup(transGroup) - - # groupSize defines the number of files each request will treat - groupSize = 1 - myTrans.setGroupSize(groupSize) - - # the transformation plugin defines which input files are treated, and how they are grouped, for example - plugin = 'Broadcast' - myTrans.setPlugin(plugin) - - # the 'body' of the transformation, defines a list of Request Operations - # that are executed in order for each file added to the transformation - targetSE = 'StorageElementOne' - transBody = [("RemoveReplica", {"TargetSE": targetSE})] - - myTrans.setBody(transBody) - - res = myTrans.setTargetSE(targetSE) - if not res['OK']: - gLogger.error("TargetSE not valid: %s" % res['Message']) - exit(1) - - res = myTrans.addTransformation() - if not res['OK']: - gLogger.error("Failed to add the transformation: %s" % res['Message']) - exit(1) - - # now activate the transformation - myTrans.setStatus('Active') - myTrans.setAgentType('Automatic') - transID = myTrans.getTransformationID()['Value'] - gLogger.notice('Created RemoveReplica transformation: %r' % transID) - exit(0) - -When we execute the script, the transformation is created with the ID MMM (e.g. 2):: - - [diracuser@dirac-tuto ~]$ python createRemoval.py - Created transformation MMM - Created RemoveReplica transformation: MMML - -To remove a replica from StorageElementOne, we just have to add files to this transformation:: - - [diracuser@dirac-tuto ~]$ dirac-transformation-add-files MMM /tutoVO/data/Trans_01/File_10 - Successfully added 1 files - -And then wait again for the ``TransformationAgent``, ``RequestTaskAgent``, ``RequestExecutingAgent`` chain to complete. - -After a short while, you should see that the folder ``/opt/dirac/storageElementOne/tutoVO/data/Trans_01/``, no longer -contains ``File_10``. - - -Using Metadata Queries to Add Files to Transformations -====================================================== - -Adding files manually to transformations can be useful, but if we want to automatically add files to transformations we -can make use of metadata queries in combination with the ``InputDataAgent``, which executes the queries and adds new -files to the corresponding transformation. - -To benefit from metadata query, we first have to create a metadata key, and add the key to a directory. These -operations can be done with the ``dirac-dms-filecatalog-cli``:: - - [diracuser@dirac-tuto ~]$ dirac-dms-filecatalog-cli - Starting FileCatalog client - - File Catalog Client $Revision: 1.17 $Date: - - FC:/$ ls -l - drwxrwxr-x 0 ciuser dirac_user 0 2019-05-06 14:30:36 tutoVO - -In the ``dirac-dms-filecatalog-cli``, like in the other DIRAC CLIs you can use ``help`` and ``help `` to see -information about the available commands. - -Initially there are no metadata keys defined:: - - FC:/$ meta show - FileMetaFields : {} - DirectoryMetaFields : {} - -We now create in integer directory metadata called ``TransformationID``:: - - FC:/$ meta index -d TransformationID int - Added metadata field TransformationID of type int - FC:/$ meta show - FileMetaFields : {} - DirectoryMetaFields : {'TransformationID': 'INT'} - -Let's add the ``TransformationID=1`` to the files we uploaded earlier:: - - FC:/$ meta set /tutoVO/data/Trans_01/ TransformationID 1 - /tutoVO/data/Trans_01 {'TransformationID': '1'} - -You can see the metadata set for a given diretory with the ``meta get`` command, and you can use the ``find`` command -inside the ``dirac-dms-filecatalog-cli`` to search for files with metadata:: - - FC:/$ meta get /tutoVO/data/Trans_01/ - !TransformationID : 1 - FC:/$ find / TransformationID=1 - Query: {'TransformationID': 1} - /tutoVO/data/Trans_01/File_1 - [..snip..] - /tutoVO/data/Trans_01/File_9 - QueryTime 0.00 sec - -Now let us create another directory, and set a different metadata value, before we create another transformation -including an inputdata query:: - - FC:/$ mkdir /tutoVO/data/Trans_02/ - Successfully created directory: /tutoVO/data/Trans_02 - FC:/$ meta set /tutoVO/data/Trans_02/ TransformationID 2 - /tutoVO/data/Trans_02 {'TransformationID': '2'} - FC:/$ meta get /tutoVO/data/Trans_02/ - !TransformationID : 2 - -Now upload some files to this folder:: - - [diracuser@dirac-tuto ~]$ for ID in {1..10}; do echo "MyContent $ID" > File_${ID} ; dirac-dms-add-file /tutoVO/data/Trans_02/File_${ID} File_${ID} StorageElementOne ; done - -We can also use the command ``dirac-dms-find-lfns`` to search for files with given metadata:: - - [diracuser@dirac-tuto ~]$ dirac-dms-find-lfns Path=/ TransformationID=2 - - -Now we create a transformation, which uses the metadata to pick up the files:: - - [diracuser@dirac-tuto ~]$ dirac-transformation-replication 2 StorageElementTwo --Plugin=Broadcast --Enable - Created transformation LLL - Successfully created replication transformation - -In fact the command ``dirac-transformation-replication`` already uses metadata, the first argument is the value for the -``TransformationID`` metadata. Now we have to wait for the ``InputDataAgent``, ``TransformationAgent``, -``RequestTaskAgent``, ``RequestExecutingAgent`` chain to run its course. - -In the log file of the ``InputDataAgent`` in ``/opt/dirac/pro/runit/Transformation/InputDataAgent/log/current`` -eventually this line should appear:: - - Transformation/InputDataAgent INFO: 10 files returned for transformation LLL from the metadata catalog - - -You may add some more files to ``/tutoVO/data/Trans_02/`` and see them appearing in your transformation:: - - [diracuser@dirac-tuto ~]$ for ID in {11..20}; do echo "MyContent $ID" > File_${ID} ; dirac-dms-add-file /tutoVO/data/Trans_02/File_${ID} File_${ID} StorageElementOne ; done - - -InputDataQuery in the Script ----------------------------- - -To add the metadata query functionality to our ``createRemoval.py`` script from above, we just need to insert a couple -of lines - -.. code-block:: python - :lineno-start: 44 - - metaQuery = {'TransformationID': 2} - myTrans.setInputMetaQuery(metaQuery) - - ... - -Adapt the script by inserting the lines and changing the ``uniqueIdentifier`` and execute it:: - - [diracuser@dirac-tuto ~]$ python createRemoval.py - Created transformation JJJ - Created RemoveReplica transformation: JJJL - -Conclusion -========== - -You now have all the knowledge to perform DataManagement in DIRAC with the TransformationSystem. - -To learn how to extend the system by creating new transformation plugins, please see how to -:ref:`dev-ts-transformationagent-plugins` and :ref:`dev-ts-body-plugins` diff --git a/docs/source/AdministratorGuide/Tutorials/index.rst b/docs/source/AdministratorGuide/Tutorials/index.rst deleted file mode 100644 index 2e905d6157d..00000000000 --- a/docs/source/AdministratorGuide/Tutorials/index.rst +++ /dev/null @@ -1,20 +0,0 @@ -.. _dirac-admin-tutorials: - -============================= -DIRAC Administrator tutorials -============================= - -Each of this tutorial is a step by step guide. - -.. toctree:: - :maxdepth: 1 - :numbered: - - basicTutoSetup - managingIdentities - diracSE - installDFC - installRMS - installTS - dmsWithTs - installWMS diff --git a/docs/source/AdministratorGuide/Tutorials/installDFC.rst b/docs/source/AdministratorGuide/Tutorials/installDFC.rst deleted file mode 100644 index bcf24170bd8..00000000000 --- a/docs/source/AdministratorGuide/Tutorials/installDFC.rst +++ /dev/null @@ -1,133 +0,0 @@ -.. _tuto_install_dfc: - -================================= -Installing the DIRAC File Catalog -================================= - -.. set highlighting to console input/output -.. highlight:: console - -Pre-requisite -============= - -You should: - -* have a machine setup as described in :ref:`tuto_basic_setup` -* be able to install dirac components -* have installed a DIRAC SE using the tutorial (:ref:`tuto_install_dirac_se`). - -Tutorial goal -============= - -The aim of the tutorial is to install the DIRAC FileCatalog (DFC) -By the end of the tutorial, you will be able to do all sort of simple Data Management operations. - -More links -========== - -More information can be found at the following places: - -* Introduction to DataManagement: :ref:`data-management-system` -* Catalog resource definition :ref:`resourcesCatalog` -* How-to datamanagement for user :ref:`howto_user_dms` - -Installing the DFC -================== - -This section is to be executed as ``diracuser`` with a proxy with ``dirac_admin`` group. - -The DFC is no different than any other DIRAC service with a database. The installation step are thus very simple:: - - [diracuser@dirac-tuto ~]$ dirac-admin-sysadmin-cli --host dirac-tuto - Pinging dirac-tuto... - [dirac-tuto]$ install db FileCatalogDB - Adding to CS DataManagement/FileCatalogDB - Database FileCatalogDB from DIRAC/DataManagementSystem installed successfully - [dirac-tuto]$ install service DataManagement FileCatalog - Loading configuration template /home/diracuser/DIRAC/DIRAC/DataManagementSystem/ConfigTemplate.cfg - Adding to CS service DataManagement/FileCatalog - service DataManagement_FileCatalog is installed, runit status: Run - - -Adding the FileCatalog resource -=============================== - -In order to be used as a FileCatalog by clients, the DFC needs to be declared. This happens in two places: - -* ``/Resources/FileCatalogs/``: in this section, you define how to access the catalog -* ``/Operations/Defaults/Services/Catalogs/``: in this section, you define how to use the catalog (for example read/write) - - -Since we have only one catalog, we will use it as ``Read-Write`` and as ``Master``. - -Using the WebApp (group ``dirac_admin``), add the following in ``/Resources/FileCatalogs/`` (all options to defaults):: - - FileCatalog - { - } - - -Using the WebApp, add the following in ``/Operations/Defaults/Services/Catalogs``:: - - FileCatalog - { - AccessType = Read-Write - Status = Active - Master = True - } - -From this moment onward, the catalog is totally usable. - -Test the catalog -================ - -Since we have a StorageElement at our disposal, we can use the standard ``dirac-dms-*`` script. - -First, let us create a file and then "put it on the grid":: - - - [diracuser@dirac-tuto ~]$ echo "Hello" > /tmp/world.txt - [diracuser@dirac-tuto ~]$ dirac-dms-add-file /tutoVO/user/c/ciuser/world.txt /tmp/world.txt StorageElementOne - - Uploading /tutoVO/user/c/ciuser/world.txt - Successfully uploaded file to StorageElementOne - - -Now, let's check its replicas and metadata:: - - [diracuser@dirac-tuto ~]$ dirac-dms-lfn-replicas /tutoVO/user/c/ciuser/world.txt - LFN StorageElement URL - ===================================================== - /tutoVO/user/c/ciuser/world.txt StorageElementOne dips://dirac-tuto:9148/DataManagement/StorageElement/tutoVO/user/c/ciuser/world.txt - - [diracuser@dirac-tuto ~]$ dirac-dms-lfn-metadata /tutoVO/user/c/ciuser/world.txt - {'Failed': {}, - 'Successful': {'/tutoVO/user/c/ciuser/world.txt': {'Checksum': '078b01ff', - 'ChecksumType': 'Adler32', - 'CreationDate': datetime.datetime(2019, 4, 16, 9, 5, 58), - 'FileID': 1L, - 'GID': 1, - 'GUID': '09F7E02F-1290-BE21-1DA7-07A266F153B3', - 'Mode': 509, - 'ModificationDate': datetime.datetime(2019, 4, 16, 9, 5, 58), - 'Owner': 'ciuser', - 'OwnerGroup': 'dirac_admin', - 'Size': 6L, - 'Status': 'AprioriGood', - 'UID': 1}}} - -Note that these metadata are those registered in the catalog (which hopefully should match the physical one !) - -We can also check all the user files that belong to us on the grid:: - - [diracuser@dirac-tuto ~]$ dirac-dms-user-lfns - Will search for files in /tutoVO/user/c/ciuser - /tutoVO/user/c/ciuser: 1 files, 0 sub-directories - 1 matched files have been put in tutoVO-user-c-ciuser.lfns - [diracuser@dirac-tuto ~]$ cat tutoVO-user-c-ciuser.lfns - /tutoVO/user/c/ciuser/world.txt - -Finally, let's remove the file:: - - [diracuser@dirac-tuto ~]$ dirac-dms-remove-files /tutoVO/user/c/ciuser/world.txt - Successfully removed 1 files diff --git a/docs/source/AdministratorGuide/Tutorials/installRMS.rst b/docs/source/AdministratorGuide/Tutorials/installRMS.rst deleted file mode 100644 index 946b138fca2..00000000000 --- a/docs/source/AdministratorGuide/Tutorials/installRMS.rst +++ /dev/null @@ -1,138 +0,0 @@ -.. _tuto_install_rms: - -======================================= -Installing the RequestManagement System -======================================= - -.. set highlighting to console input/output -.. highlight:: console - -Pre-requisite -============= - -You should: - -* have a machine setup as described in :ref:`tuto_basic_setup` -* be able to install dirac components -* have installed two DIRAC SE using the tutorial (:ref:`tuto_install_dirac_se`). -* have installed the DFC (:ref:`tuto_install_dfc`) -* have followed the tutorial on identity management (:ref:`tuto_managing_identities`) - -Tutorial goal -============= - -The aim of the tutorial is to install the RequestManagement system components and to use it to perform a simple replication of file. - -More links -========== - -More information can be found at the following places: - -* :ref:`data-management-system` -* :ref:`requestManagementSystem` - -Installing the RMS -================== - -This section is to be executed as ``diracuser`` with a proxy with ``dirac_admin`` group. - -The RMS needs the ``ReqManager`` service and the ``RequestExecutingAgent`` to work (you may want to add the ``CleanReqDBAgent`` if you scale...). - -The RMS is no different than any other DIRAC system. The installation step are thus very simple:: - - [diracuser@dirac-tuto ~]$ dirac-admin-sysadmin-cli --host dirac-tuto - Pinging dirac-tuto... - [dirac-tuto]$ add instance RequestManagement Production - Adding RequestManagement system as Production self.instance for MyDIRAC-Production self.setup to dirac.cfg and CS - RequestManagement system instance Production added successfully - [dirac-tuto]$ restart * - All systems are restarted, connection to SystemAdministrator is lost - [dirac-tuto]$ install db ReqDB - MySQL root password: - Adding to CS RequestManagement/ReqDB - Database ReqDB from DIRAC/RequestManagementSystem installed successfully - [dirac-tuto]$ install service RequestManagement ReqManager - Loading configuration template /home/diracuser/DiracInstallation/DIRAC/RequestManagementSystem/ConfigTemplate.cfg - Adding to CS service RequestManagement/ReqManager - service RequestManagement_ReqManager is installed, runit status: Run - [dirac-tuto]$ install agent RequestManagement RequestExecutingAgent - Loading configuration template /home/diracuser/DiracInstallation/DIRAC/RequestManagementSystem/ConfigTemplate.cfg - Adding to CS agent RequestManagement/RequestExecutingAgent - agent RequestManagement_RequestExecutingAgent is installed, runit status: Run - [dirac-tuto]$ quit - - -By default, the installation of the ``RequestExecutingAgent`` will configure it with a whole bunch of default Operations possible. You can see that in the Agent configuration in ``/Systems/RequestManagement/Production/Agents/RequestExecutingAgent/OperationHandlers`` - - -Testing the RMS -=============== - -This section is to be executed with a proxy with `dirac_data` group. - -The test we are going to do consists in transferring a file from one storage element to another, using the RequestExecutingAgent. - -First, let's add a file:: - - [diracuser@dirac-tuto ~]$ echo "My Test File" > /tmp/myTestFile.txt - [diracuser@dirac-tuto ~]$ dirac-dms-add-file /tutoVO/user/c/ciuser/myTestFile.txt /tmp/myTestFile.txt StorageElementOne - - Uploading /tutoVO/user/c/ciuser/myTestFile.txt - Successfully uploaded file to StorageElementOne - - -We can see that our file is indeed in the ``StorageElementOne``:: - - [diracuser@dirac-tuto ~]$ dirac-dms-lfn-replicas /tutoVO/user/c/ciuser/myTestFile.txt - LFN StorageElement URL - ========================================================== - /tutoVO/user/c/ciuser/myTestFile.txt StorageElementOne dips://dirac-tuto:9148/DataManagement/StorageElement/tutoVO/user/c/ciuser/myTestFile.txt - -Let's replicate it to ``StorageElementTwo`` using the RMS:: - - [diracuser@dirac-tuto ~]$ dirac-dms-replicate-and-register-request myFirstRequest /tutoVO/user/c/ciuser/myTestFile.txt StorageElementTwo - Request 'myFirstRequest' has been put to ReqDB for execution. - RequestID(s): 8 - You can monitor requests' status using command: 'dirac-rms-request ' - - -The Request has a name (``myFirstRequest``) that we chose, but also an ID, returned by the system (here ``8``). The ID is guaranteed to be unique, while the name is not, so it is recommended to use the ID when you interact with the RMS. You can see the status of your Request, using its name or ID:: - - [diracuser@dirac-tuto ~]$ dirac-rms-request myFirstRequest - Request name='myFirstRequest' ID=8 Status='Waiting' - Created 2019-04-23 14:37:05, Updated 2019-04-23 14:37:05, NotBefore 2019-04-23 14:37:05 - Owner: '/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser', Group: dirac_data - [0] Operation Type='ReplicateAndRegister' ID=8 Order=0 Status='Waiting' - TargetSE: StorageElementTwo - Created 2019-04-23 14:37:05, Updated 2019-04-23 14:37:05 - [01] ID=2 LFN='/tutoVO/user/c/ciuser/myTestFile.txt' Status='Waiting' Checksum='1e750431' - - [diracuser@dirac-tuto ~]$ dirac-rms-request 8 - Request name='myFirstRequest' ID=8 Status='Waiting' - Created 2019-04-23 14:37:05, Updated 2019-04-23 14:37:05, NotBefore 2019-04-23 14:37:05 - Owner: '/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser', Group: dirac_data - [0] Operation Type='ReplicateAndRegister' ID=8 Order=0 Status='Waiting' - TargetSE: StorageElementTwo - Created 2019-04-23 14:37:05, Updated 2019-04-23 14:37:05 - [01] ID=2 LFN='/tutoVO/user/c/ciuser/myTestFile.txt' Status='Waiting' Checksum='1e750431' - - -You can here clearly see that the Request consists of one ``ReplicateAndRegister`` operation (which does what it says) targeting the LFN ``/tutoVO/user/c/ciuser/myTestFile.txt``. The RequestExecutingAgent will pick up the request and execute it. And shortly you should be able to see it done:: - - [diracuser@dirac-tuto ~]$ dirac-rms-request 8 - Request name='myFirstRequest' ID=8 Status='Done' - Created 2019-04-23 14:37:05, Updated 2019-04-23 14:37:29, NotBefore 2019-04-23 14:37:05 - Owner: '/C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser', Group: dirac_data - [0] Operation Type='ReplicateAndRegister' ID=8 Order=0 Status='Done' - TargetSE: StorageElementTwo - Created 2019-04-23 14:37:05, Updated 2019-04-23 14:37:29 - [01] ID=2 LFN='/tutoVO/user/c/ciuser/myTestFile.txt' Status='Done' Checksum='1e750431' - - [diracuser@dirac-tuto ~]$ dirac-dms-lfn-replicas /tutoVO/user/c/ciuser/myTestFile.txt - LFN StorageElement URL - ========================================================== - /tutoVO/user/c/ciuser/myTestFile.txt StorageElementTwo dips://dirac-tuto:9147/DataManagement/StorageElementTwo/tutoVO/user/c/ciuser/myTestFile.txt - StorageElementOne dips://dirac-tuto:9148/DataManagement/StorageElement/tutoVO/user/c/ciuser/myTestFile.txt - - -Conclusion -========== - -You now have an RMS in place, which is the base for all the asynchronous operations in DIRAC. This is used for big scale operations, failover, or even more ! diff --git a/docs/source/AdministratorGuide/Tutorials/installTS.rst b/docs/source/AdministratorGuide/Tutorials/installTS.rst deleted file mode 100644 index a4c27370940..00000000000 --- a/docs/source/AdministratorGuide/Tutorials/installTS.rst +++ /dev/null @@ -1,134 +0,0 @@ -.. _tuto_install_ts: - -=================================== -Installing the TransformationSystem -=================================== - -.. set highlighting to console input/output -.. highlight:: console - -Pre-Requisite -============= - -You should: - -* have a machine setup as described in :ref:`tuto_basic_setup` -* have installed two DIRAC SE using the tutorial (:ref:`tuto_install_dirac_se`). -* have installed the DFC using the tutorial (:ref:`tuto_install_dfc`). -* have followed the tutorial on identity management (:ref:`tuto_managing_identities`) -* have installed the RMS using the tutorial (:ref:`tuto_install_rms`) - - -Tutorial Goal -============= - -The aim of the tutorial is to install the Transformation system components and to use it to perform an automatic replication. - - -More Links -========== - -* :ref:`adminTS` -* See the options for services and agents in code documentation of the :mod:`~DIRAC.TransformationSystem` - -Installing the TransformationSystem -=================================== - -.. highlight:: console - -This section is to be executed as ``diracuser`` with a proxy int the ``dirac_admin`` group. - -The Transformation System(TS) needs the ``TransformationManager`` service and the ``TransformationAgent``, ``InputDataAgent``, -``RequestTaskAgent`` to work for data management purposes. The ``WorkflowTaskAgent`` is needed to submit jobs. -Finally the ``TransformationCleaning`` cleans up if transformations are finished. As the agents need to run one after -the other we set the PollingTime to 30 seconds to reduce the waiting time once we create transformations. - -The TS is no different than any other DIRAC system. The installation steps are thus very simple:: - - [diracuser@dirac-tuto ~]$ dirac-admin-sysadmin-cli --host dirac-tuto - Pinging dirac-tuto... - [dirac-tuto]$ add instance Transformation Production - Adding Transformation system as Production self.instance for MyDIRAC-Production self.setup to dirac.cfg and CS - Transformation system instance Production added successfully - [dirac-tuto]$ restart * - All systems are restarted, connection to SystemAdministrator is lost - [dirac-tuto]$ install db TransformationDB - MySQL root password: - Adding to CS Transformation/TransformationDB - Database TransformationDB from DIRAC/TransformationSystem installed successfully - [dirac-tuto]$ install service Transformation TransformationManager - Loading configuration template /home/diracuser/DiracInstallation/DIRAC/TransformationSystem/ConfigTemplate.cfg - Adding to CS service Transformation/TransformationManager - service Transformation_TransformationManager is installed, runit status: Run - [dirac-tuto]$ install agent Transformation TransformationAgent -p PollingTime=30 - Loading configuration template /home/diracuser/DiracInstallation/DIRAC/TransformationSystem/ConfigTemplate.cfg - Adding to CS agent Transformation/TransformationAgent - agent Transformation_TransformationAgent is installed, runit status: Run - [dirac-tuto]$ install agent Transformation InputDataAgent -p PollingTime=30 - Loading configuration template /home/diracuser/DiracInstallation/DIRAC/TransformationSystem/ConfigTemplate.cfg - Adding to CS agent Transformation/InputDataAgent - agent Transformation_InputDataAgent is installed, runit status: Run - [dirac-tuto]$ install agent Transformation WorkflowTaskAgent -p PollingTime=30 -p MonitorTasks=True -p MonitorFiles=True - Loading configuration template /home/diracuser/DiracInstallation/DIRAC/TransformationSystem/ConfigTemplate.cfg - Adding to CS agent Transformation/WorkflowTaskAgent - agent Transformation_WorkflowTaskAgent is installed, runit status: Run - [dirac-tuto]$ install agent Transformation RequestTaskAgent -p PollingTime=30 -p MonitorTasks=True -p MonitorFiles=True - Loading configuration template /home/diracuser/DiracInstallation/DIRAC/TransformationSystem/ConfigTemplate.cfg - Adding to CS agent Transformation/RequestTaskAgent - agent Transformation_RequestTaskAgent is installed, runit status: Run - -Add a ProductionManagement Group -================================ - -We create a new group ``dirac_prod``, which will be used to manage transformations - -Using the ``Configuration Manager`` application in the WebApp, create a new section ``dirac_prod`` in ``/Registry/Groups``:: - - Users = ciuser - Properties = ProductionManagement, NormalUser - AutoUploadProxy = True - - -After restarting the ``ProxyManager``, you should now be able to get a proxy belonging to the ``dirac_prod`` group that -will be automatically uploaded. - -Add a ProdManager Shifter -========================= - -Using the ``Configuration Manager`` application in the WebApp, create a new shifter ``ProdManager`` in the -``/Operations/Defaults/Shifter`` section:: - - ProdManager - { - User = ciuser - Group = dirac_prod - } - - - -Add a Sites which the StorageElements belong to -=============================================== - -Using the ``Configuration Manager`` application in the WebApp, create a new section ``Sites`` in ``/Resources``, which -contains a *Grid* with two *Sites*, to which the two SEs are associated:: - - Sites - { - MyGrid - { - MyGrid.Site1.uk - { - SE = StorageElementOne - } - MyGrid.Site2.de - { - SE = StorageElementTwo - } - } - } - - -Conclusion -========== - -You now have a Transformation System in place, which is the base for all automatic operations in DIRAC. diff --git a/docs/source/AdministratorGuide/Tutorials/installWMS.rst b/docs/source/AdministratorGuide/Tutorials/installWMS.rst deleted file mode 100644 index 6a1e6e510c3..00000000000 --- a/docs/source/AdministratorGuide/Tutorials/installWMS.rst +++ /dev/null @@ -1,336 +0,0 @@ -.. _tuto_install_wms: - - -======================================== -Installing the WorkloadManagement System -======================================== - -.. set highlighting to console input/output -.. highlight:: console - -Pre-Requisite -============= - -You should: - -* have a machine setup as described in :ref:`tuto_basic_setup` -* have installed two DIRAC SE using the tutorial (:ref:`tuto_install_dirac_se`). -* have followed the tutorial on identity management (:ref:`tuto_managing_identities`) -* have installed the TS using the tutorial (:ref:`tuto_install_ts`) - -Tutorial Goal -============= - -The aim of the tutorial is to install the WorkloadManagement system components and to use them to generate and submit a simple job. - - -More Links -========== - -* :ref:`WMS` -* Information about the types and options of the :ref:`Computing Elements` -* Information about the user jobs using the DIRAC API: :ref:`user jobs` - -Installing the WorkloadManagementSystem -======================================= - -.. highlight:: console - -This section is to be executed as ``diracuser`` with the ``dirac_admin`` proxy (reminder: ``dirac-proxy-init -g dirac_admin``). - -Basically, the WorkloadManagement System (WMS) needs the ``SiteDirector`` agent to install pilots on Computing Elements (CEs) as well as -different services and agents such as the ``JobManager``, the ``JobMonitoring`` and the ``Matcher`` to manage the jobs and their status. -The executors are used to check the jobs and schedule them on Task Queues. - -The WMS is no different than any other DIRAC system. The installation steps are thus very simple:: - - [diracuser@dirac-tuto ~]$ dirac-proxy-init -g dirac_admin - [diracuser@dirac-tuto ~]$ dirac-admin-sysadmin-cli --host dirac-tuto - Pinging dirac-tuto... - [dirac-tuto]> add instance WorkloadManagement Production - Adding WorkloadManagement system as Production self.instance for MyDIRAC-Production self.setup to dirac.cfg and CS WorkloadManagement system instance Production added successfully - [dirac-tuto]> restart * - All systems are restarted, connection to SystemAdministrator is lost - [dirac-tuto]> install db JobDB - MySQL root password: - Adding to CS WorkloadManagement/JobDB - Database JobDB from DIRAC/WorkloadManagementSystem installed successfully - [dirac-tuto]> install db JobLoggingDB - MySQL root password: - Adding to CS WorkloadManagement/JobLoggingDB - Database JobLoggingDB from DIRAC/WorkloadManagementSystem installed successfully - [dirac-tuto]> install db PilotAgentsDB - MySQL root password: - Adding to CS WorkloadManagement/PilotAgentsDB - Database PilotAgentsDB from DIRAC/WorkloadManagementSystem installed successfully - [dirac-tuto]> install db SandboxMetadataDB - MySQL root password: - Adding to CS WorkloadManagement/SandboxMetadataDB - Database SandboxMetadataDB from DIRAC/WorkloadManagementSystem installed successfully - [dirac-tuto]> install db TaskQueueDB - MySQL root password: - Adding to CS WorkloadManagement/TaskQueueDB - Database TaskQueueDB from DIRAC/WorkloadManagementSystem installed successfully - [dirac-tuto]> install service WorkloadManagement PilotManager - Loading configuration template /home/diracuser/DiracInstallation/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg - Adding to CS service WorkloadManagement/PilotManager - service WorkloadManagement_PilotManager is installed, runit status: Run - [dirac-tuto]> install service WorkloadManagement JobManager - Loading configuration template /home/diracuser/DiracInstallation/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg - Adding to CS service WorkloadManagement/JobManager - service WorkloadManagement_JobManager is installed, runit status: Run - [dirac-tuto]> install service WorkloadManagement JobMonitoring - Loading configuration template /home/diracuser/DiracInstallation/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg - Adding to CS service WorkloadManagement/JobMonitoring - service WorkloadManagement_JobMonitoring is installed, runit status: Run - [dirac-tuto]> install service WorkloadManagement JobStateUpdate - Loading configuration template /home/diracuser/DiracInstallation/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg - Adding to CS service WorkloadManagement/JobStateUpdate - service WorkloadManagement_JobStateUpdate is installed, runit status: Run - [dirac-tuto]> install service WorkloadManagement Matcher - Loading configuration template /home/diracuser/DiracInstallation/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg - Adding to CS service WorkloadManagement/Matcher - service WorkloadManagement_Matcher is installed, runit status: Run - [dirac-tuto]> install service WorkloadManagement OptimizationMind - Loading configuration template /home/diracuser/DiracInstallation/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg - Adding to CS service WorkloadManagement/OptimizationMind - service WorkloadManagement_OptimizationMind is installed, runit status: Run - [dirac-tuto]> install service WorkloadManagement SandboxStore - Loading configuration template /home/diracuser/DiracInstallation/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg - Adding to CS service WorkloadManagement/SandboxStore - service WorkloadManagement_SandboxStore is installed, runit status: Run - [dirac-tuto]> install service WorkloadManagement WMSAdministrator - Loading configuration template /home/diracuser/DiracInstallation/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg - Adding to CS service WorkloadManagement/WMSAdministrator - service WorkloadManagement_WMSAdministrator is installed, runit status: Run - [dirac-tuto]> install service Framework BundleDelivery - Loading configuration template /home/diracuser/DiracInstallation/DIRAC/Framework/ConfigTemplate.cfg - Adding to CS service WorkloadManagement/BundleDelivery - service WorkloadManagement_BundleDelivery is installed, runit status: Run - [dirac-tuto]> install service Framework Monitoring - Loading configuration template /home/diracuser/DiracInstallation/DIRAC/Framework/ConfigTemplate.cfg - Adding to CS service WorkloadManagement/Monitoring - service WorkloadManagement_BundleDelivery is installed, runit status: Run - [dirac-tuto]> install agent WorkloadManagement SiteDirector - Loading configuration template /home/diracuser/DiracInstallation/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg - Adding to CS service WorkloadManagement/SiteDirector - agent WorkloadManagement_SiteDirector is installed, runit status: Run - [dirac-tuto]> install agent WorkloadManagement JobCleaningAgent - Loading configuration template /home/diracuser/DiracInstallation/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg - Adding to CS service WorkloadManagement/JobCleaningAgent - agent WorkloadManagement_JobCleaningAgent is installed, runit status: Run - [dirac-tuto]> install agent WorkloadManagement PilotStatusAgent - Loading configuration template /home/diracuser/DiracInstallation/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg - Adding to CS service WorkloadManagement/PilotStatusAgent - agent WorkloadManagement_PilotStatusAgent is installed, runit status: Run - [dirac-tuto]> install agent WorkloadManagement StalledJobAgent - Loading configuration template /home/diracuser/DiracInstallation/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg - Adding to CS service WorkloadManagement/StalledJobAgent - agent WorkloadManagement_StalledJobAgent is installed, runit status: Run - [dirac-tuto]> install executor WorkloadManagement Optimizers - Loading configuration template /home/diracuser/DiracInstallation/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg - Adding to CS service WorkloadManagement/Optimizers - Loading configuration template /home/diracuser/DiracInstallation/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg - Adding to CS service WorkloadManagement/JobPath - Loading configuration template /home/diracuser/DiracInstallation/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg - Adding to CS service WorkloadManagement/JobSanity - Loading configuration template /home/diracuser/DiracInstallation/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg - Adding to CS service WorkloadManagement/InputData - Loading configuration template /home/diracuser/DiracInstallation/DIRAC/WorkloadManagementSystem/ConfigTemplate.cfg - Adding to CS service WorkloadManagement/JobScheduling - executor WorkloadManagement_Optimizers is installed, runit status: Run - [dirac-tuto]> restart WorkloadManagement * - -Create and submit a job -======================= - -This section is to be executed as ``diracuser`` with the ``dirac_user`` proxy (reminder: ``dirac-proxy-init``). - -Create a Python script to generate and submit a simple job. Copy paste the following lines into a new file called ``job.py`` - -.. code-block:: python - - #!/bin/env python - # Magic lines necessary to activate the DIRAC Configuration System - # to discover all the required services - from DIRAC.Core.Base.Script import Script - Script.parseCommandLine(ignoreErrors=True) - from DIRAC.Interfaces.API.Job import Job - from DIRAC.Interfaces.API.Dirac import Dirac - - j = Job() - dirac = Dirac() - - j.setName('MyFirstJob') - j.setJobGroup('MyJobs') - - # Specify CPU requirements - j.setCPUTime(21600) - - # Specify the log level of the job execution: INFO (default), DEBUG, VERBOSE - j.setLogLevel('DEBUG') - - # Executabe and arguments can be given in one call - j.setExecutable('echo', arguments='Hello world!') - - result = dirac.submitJob(j) - if not result['OK']: - print("ERROR:", result['Message']) - else: - print(result['Value']) - -This script creates a new job called ``MyFirstJob`` and aims at executing ``echo "Hello World!"``. The output should be something like that:: - - [diracuser@dirac-tuto ~]$ python job.py - - [diracuser@dirac-tuto ~]$ dirac-wms-job-status - JobID= Status=Waiting; MinorStatus=Pilot Agent Submission; Site=ANY; - -As we have not defined any CE yet, the job cannot run and remains ``Waiting``. - -Adding a CE -=========== - -First, as ``root``, we create a new user ``diracpilot`` that is going to simulate an SSH Computing Element on ``dirac-tuto``:: - - adduser -s /bin/bash -d /home/diracpilot diracpilot - echo password | /usr/bin/passwd --stdin diracpilot - -As ``diracuser``, connect to ``diracpilot`` through SSH a first time to initialize the connection and make sure everything works:: - - ssh diracpilot@dirac-tuto - -Then, as ``diracuser`` with the ``dirac_admin`` proxy, we need to define a CE in a ``/Resources/Sites//`` section of the configuration file using the WebApp (create the sections if necessary):: - - Resources - { - Sites - { - MyGrid - { - MyGrid.Site1.uk - { - CE = dirac-tuto - CEs - { - dirac-tuto - { - CEType = SSH - SSHHost = dirac-tuto - SSHUser = diracpilot - SSHPassword = password - SSHType = ssh - Queues - { - queue - { - CPUTime = 40000 - MaxTotalJobs = 5 - MaxWaitingJobs = 10 - BundleProxy = True - BatchError = /home/diracpilot/localsite/error - ExecutableArea = /home/diracpilot/localsite/submission - RemoveOutput = True - } - } - } - } - } - } - } - } - -We set the type of the CE, ``SSH`` in our case, as well as the required parameters to access the Element. -Then we configure the queue that is going to receive the jobs. A queue corresponds to a set of Worker Nodes in practice. - -Note: make sure the ``CPUTime`` of the queue is above the ``CPUTime`` of the job, else the job will not be scheduled to run on this Worker Node. - -Configuring the pilots -====================== - -A job is not able to run directly on a Worker Node and needs to be executed by a pilot that has the knowledge of its environment and knows how to run jobs within it. -The pilot is the first job to be deployed on a Worker Node and it installs and configures DIRAC and asks for pending jobs in Task Queues that would match the environment of the Worker Node. Add the following lines in the ``/Operations/MyDIRAC-Production`` section using the WebApp:: - - Pilot - { - Version = v7r0p36 - CheckVersion = False - Command - { - Test = GetPilotVersion - Test += CheckWorkerNode - Test += InstallDIRAC - Test += ConfigureBasics - Test += ConfigureCPURequirements - Test += ConfigureArchitecture - Test += CheckCECapabilities - Test += LaunchAgent - } - GenericPilotGroup = dirac_user - GenericPilotUser = ciuser - pilotFileServer = dirac-tuto:8443 - } - -We pass our credentials information to the pilot so that it can interact with DIRAC as it needs to execute the commands defined in ``Commands``. -Only a small script called ``pilotWrapper`` is directly passed to the CE, most of the files used by the pilot will be downloaded from ``pilotFileServer`` during the script execution. -These files can be uploaded and updated at each commit done to the configuration, we just need to create the directory that is going to contain the files required by the pilot and add the information within the configuration. First, add the option below to the configuration, in the ``/WebApp`` section:: - - StaticDirs = pilot - -As ``dirac``, create the pilot repository that will contain all the pilot files that will be updated whenever a CS update is triggered:: - - mkdir -p /opt/dirac/webRoot/www/pilot - - -..warning:: Do not put the Pilot configuration in ``Operations/Defaults``, DIRAC would not be able to get it. - -Configuring the Sandbox -======================= - -We need to define a Sandbox to pass input files related to the job to the Worker Node and then to get the results of the execution. -A Sandbox is represented as a StorageElement and can be installed in this way. As ``diracuser`` with the ``dirac_admin`` proxy, executes :: - - [diracuser@dirac-tuto ~]$ dirac-admin-sysadmin-cli --host dirac-tuto - Pinging dirac-tuto... - [dirac-tuto]> install service DataManagement ProductionSandboxSE -m StorageElement -p Port=9146 -p BasePath=/opt/dirac/storage/sandboxes - -Then the following lines have to be added to the configuration in the ``/Resources/StorageElements`` section using the WebApp:: - - ProductionSandboxSE - { - BackendType = DISET - DIP - { - Host = dirac-tuto - Port = 9146 - Protocol = dips - Path = /DataManagement/ProductionSandboxSE - Access = remote - } - } - -The Storage Element is then used by the ``SandboxStore`` service. -If it is not defined (it should in practice), add the following option in ``Systems/WorkloadManagement/Production/Services/SandboxStore``:: - - LocalSE = ProductionSandboxSE - -Make the Site available for receiving jobs -========================================== - -By default, the Site previously created is not allowed to receive any job from DIRAC. Execute the following command to add it to the list of available Sites:: - - [diracuser@dirac-tuto ~]$ dirac-admin-allow-site MyGrid.Site1.uk "test" -E False - Site MyGrid.Site1.uk status is set to Active - -Finally restart the WorkloadManagement system to apply the configuration changes to the components:: - - [diracuser@dirac-tuto ~]$ dirac-admin-sysadmin-cli --host dirac-tuto - Pinging dirac-tuto... - [dirac-tuto]> restart WorkloadManagement * - -After a moment we should get a result performing these commands:: - - [diracuser@dirac-tuto ~]$ dirac-wms-job-status - JobID= Status=Done; MinorStatus=Execution Complete; Site=MyGrid.Site1.uk; - [diracuser@dirac-tuto ~]$ dirac-wms-job-get-output - Job output sandbox retrieved in /home/diracuser// diff --git a/docs/source/AdministratorGuide/Tutorials/managingIdentities.rst b/docs/source/AdministratorGuide/Tutorials/managingIdentities.rst deleted file mode 100644 index 57b793a4a6f..00000000000 --- a/docs/source/AdministratorGuide/Tutorials/managingIdentities.rst +++ /dev/null @@ -1,153 +0,0 @@ -.. _tuto_managing_identities: - -=================== -Managing identities -=================== - -.. set highlighting to console input/output -.. highlight:: console - -Pre-requisite -============= - -You should: - -* have a machine setup as described in :ref:`tuto_basic_setup` -* be able to install dirac components - - -Tutorial goal -============= - -Very quickly when using DIRAC, you will need to manage identities of people and their proxies. This is done with the ``ProxyManager`` service and with several configuration options. -In this tutorial, we will install the ``ProxyManager``, create a new group, and define some ``Shifter``. - - -Further reading -=============== - -* :ref:`compAuthNAndAutZ` -* :ref:`manageAuthNAndAuthZ` - -Installing the ``ProxyManager`` -=============================== - -This section is to be performed as ``diracuser`` with ``dirac_admin`` group proxy:: - - [diracuser@dirac-tuto ~]$ source ~/DiracInstallation/bashrc - [diracuser@dirac-tuto ~]$ dirac-proxy-init -g dirac_admin - - -The ``ProxyManager`` will host delegated proxies of the users. As any other service, it is very easy to install with the ``dirac-admin-sysadmin-cli``:: - - [diracuser@dirac-tuto ~]$ dirac-admin-sysadmin-cli -H dirac-tuto - -And then in the CLI:: - - [dirac-tuto]$ install db ProxyDB - MySQL root password: - Adding to CS Framework/ProxyDB - Database ProxyDB from DIRAC/FrameworkSystem installed successfully - [dirac-tuto]$ install service Framework ProxyManager - Loading configuration template /home/diracuser/DiracInstallation/DIRAC/FrameworkSystem/ConfigTemplate.cfg - Adding to CS service Framework/ProxyManager - service Framework_ProxyManager is installed, runit status: Run - - - -.. note:: The ProxyDB contains sensitive information. For production environment, it is recommended that you keep this in a separate database with different credentials and strict access control. - - -Testing the ``ProxyManager`` -============================ - -The simplest way to test it is to upload your user proxy:: - - [diracuser@dirac-tuto ~]$ dirac-proxy-init - Generating proxy... - Uploading proxy for dirac_user... - Proxy generated: - subject : /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser/CN=6045995638 - issuer : /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - identity : /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - timeleft : 23:59:59 - DIRAC group : dirac_user - rfc : True - path : /tmp/x509up_u501 - username : ciuser - properties : NormalUser - - Proxies uploaded: - DN | Group | Until (GMT) - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser | dirac_user | 2020/04/09 14:43 - -As you can see, the ProxyDB now contains a delegated proxy for the ``ciuser`` with the group ``dirac_user``. - -If you use a proxy with the ``ProxyManagement`` permission, like the ``dirac_admin`` group has, you can retrieve proxies stored in the DB:: - - [diracuser@dirac-tuto ~]$ dirac-proxy-init -g dirac_admin - Generating proxy... - Proxy generated: - subject : /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser/CN=5472309786 - issuer : /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - identity : /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - timeleft : 23:59:59 - DIRAC group : dirac_admin - rfc : True - path : /tmp/x509up_u501 - username : ciuser - properties : AlarmsManagement, ServiceAdministrator, CSAdministrator, JobAdministrator, FullDelegation, ProxyManagement, Operator - [diracuser@dirac-tuto ~]$ dirac-admin-get-proxy ciuser dirac_user - Proxy downloaded to /home/diracuser/proxy.ciuser.dirac_user - - -Adding a new group -================== - -Groups are useful to manage permissions and separate activities. For example, we will create a new group ``dirac_data``, and decide to use that group for all the data centrally managed. - -Using the ``Configuration Manager`` application in the WebApp using the ``dirac_admin`` group, create a new section ``dirac_data`` in ``/Registry/Groups``:: - - Users = ciuser - Properties = NormalUser - AutoUploadProxy = True - -You should now be able to get a proxy belonging to the `dirac_data` group that will be automatically uploaded:: - - [diracuser@dirac-tuto ~]$ dirac-proxy-init -g dirac_data - Generating proxy... - Uploading proxy for dirac_data... - Proxy generated: - subject : /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser/CN=6009266000 - issuer : /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - identity : /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser - timeleft : 23:59:59 - DIRAC group : dirac_data - rfc : True - path : /tmp/x509up_u501 - username : ciuser - properties : NormalUser - - Proxies uploaded: - DN | Group | Until (GMT) - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser | dirac_data | 2020/04/09 14:43 - /C=ch/O=DIRAC/OU=DIRAC CI/CN=ciuser | dirac_user | 2020/04/09 14:43 - - -.. note:: if you get ``Unauthorized query ( 1111 : Unauthorized query)``, it means the ProxyManager has not yet updated its internal configuration. Just restart it to save time, or wait. - - -Adding a Shifter -================ - -``Shifter`` is basically a role, to which you associate a given proxy, for example ``DataManager`` (it could be anything). You can then tell your Components to use the ``DataManager`` identity to perform certain operations (at random: data management operations ? :-) ). - -Using the ``Configuration Manager`` application in the WebApp, create a new section ``Shifter`` in ``/Operations/Defaults``:: - - DataManager - { - User = ciuser - Group = dirac_data - } - -You can now force any agent (don't, unless you know what you are doing) to use a proxy instead of the host certificate by specifying the ``shifterProxy`` option. diff --git a/docs/source/AdministratorGuide/index.rst b/docs/source/AdministratorGuide/index.rst index 56f6f7cb3c8..6bfeb791c3a 100644 --- a/docs/source/AdministratorGuide/index.rst +++ b/docs/source/AdministratorGuide/index.rst @@ -1,3 +1,5 @@ +.. _administrator_guide: + =================== Administrator Guide =================== @@ -5,7 +7,7 @@ Administrator Guide DIRAC has been developed with extensibility and flexibility in mind. A DIRAC release is composed by few projects, like in the following picture. This administration documentation refers to the "Core" DIRAC project. -.. image:: ../_static/horizontalAndVertical-ext.png +.. image:: ../_static/py3Stack.png :alt: DIRAC projects interaction overview :align: center @@ -20,8 +22,6 @@ This administration documentation refers to the "Core" DIRAC project. HowTo/index Configuration/index Resources/index - Tutorials/index - technologyPreviews Systems/index .. toctree:: diff --git a/docs/source/AdministratorGuide/technologyPreviews.rst b/docs/source/AdministratorGuide/technologyPreviews.rst deleted file mode 100644 index 294efeeec73..00000000000 --- a/docs/source/AdministratorGuide/technologyPreviews.rst +++ /dev/null @@ -1,38 +0,0 @@ -=================== -Technology Previews -=================== - - -When new technologies are introduced within DIRAC, there are cases when we allow this technology not to be used. -The reason might be that it is not completely mature yet, or that it can be disturbing. These technologies are toggled by either CS flags or environment variables. -They are meant to stay optional for a couple of releases, and then they become the default. -This page keeps a list of such technologies. - -.. _jsonSerialization: - -JSON Serialization -================== - -We aim at replacing the DEncode serialization over the network with json serialization. In order to be smooth, the transition needs to happen in several steps: - -* DISET -> DISET: as it is now. We encode DISET, and decode DISET -* DISET -> DISET, JSON: we send DISET, try to decode with DISET, fallback to JSON if it fails. This step is to make sure all clients will be ready to understand JSON whenever it comes. -* JSON -> DISET, JSON: we send JSON, attempt to decode with DISET, fallback to JSON if it fails. This step is to make sure that if we still have some old clients lying around, we are still able to understand them. -* JSON -> JSON: final step, goodbye DISET. - -The changes from one stage to the next is controlled by environment variables, and it can go to your own pace: - -* ``DIRAC_USE_JSON_DECODE``: must be the first one. Enables the DISET,JSON decoding -* ``DIRAC_USE_JSON_ENCODE``: ``DIRAC_USE_JSON_DECODE`` must still be enabled ! Sends JSON instead of DISET. - -The last stage (JSON only) will be the default of the following release, so before upgrading you will have to go through the previous steps. - -HTTPS Services -============== - -The aim is to replace the DISET services with HTTPS services. The changes should be almost transparent for users/admins. However, because it is still very much in the state of preview, we do not yet describe how/what to change. If you really want to play around, please check :ref:`httpsTornado`. - -OAuth2 authorization -===================== - -The main idea is to start using access tokens to communicate with DIRAC systems and third-party services. Tokens can be delivered to DIRAC using Identity Providers such as Indigo IaM (WLCG) and EGI CheckIn. As a result of this integration, a new component type, named :ref:`apis` was developed. diff --git a/docs/source/DeveloperGuide/AddingNewComponents/CheckYourInstallation/index.rst b/docs/source/DeveloperGuide/AddingNewComponents/CheckYourInstallation/index.rst index a3dc7c49205..050eba9014e 100644 --- a/docs/source/DeveloperGuide/AddingNewComponents/CheckYourInstallation/index.rst +++ b/docs/source/DeveloperGuide/AddingNewComponents/CheckYourInstallation/index.rst @@ -102,15 +102,15 @@ We will now play with a **dirac.cfg** file. For these exercises you can use the Try this:: >>> from DIRAC import gConfig - >>> gConfig.getValue('/DIRAC/Setup') - 'DeveloperSetup' + >>> gConfig.getValue('/DIRAC/DefaultGroup') + 'dirac_user' -Where does 'DeveloperSetup' come from? Open that dirac.cfg and search for it. Got it? it's in:: +Where does 'dirac_user' come from? Open that dirac.cfg and search for it. Got it? it's in:: DIRAC { ... - Setup = DeveloperSetup + DefaultGroup = dirac_user ... } diff --git a/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/dirac_my_great_script.py b/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/dirac_my_great_script.py index 08507bd6c57..20fb764a3f0 100644 --- a/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/dirac_my_great_script.py +++ b/docs/source/DeveloperGuide/AddingNewComponents/DevelopingCommands/dirac_my_great_script.py @@ -73,7 +73,7 @@ def parseSwitchesAndPositionalArguments(): # Get arguments allArgs = Script.getPositionalArgs() - gLogger.debug("All arguments: %s" % ", ".join(allArgs)) + gLogger.debug(f"All arguments: {', '.join(allArgs)}") # Get unprocessed switches switches = dict(Script.getUnprocessedSwitches()) @@ -111,9 +111,9 @@ def main(): if services == "no elements": gLogger.error("No services defined") DIRACExit(1) - gLogger.notice("Your %s is:" % ("DN" if user.startswith("/") else "name"), user) + gLogger.notice(f"Your {'DN' if user.startswith('/') else 'name'} is:", user) gLogger.notice("This is the servicesList:", ", ".join(services)) - gLogger.notice("We are done with %s report." % repType) + gLogger.notice(f"We are done with {repType} report.") DIRACExit(0) diff --git a/docs/source/DeveloperGuide/AddingNewComponents/DevelopingDatabases/index.rst b/docs/source/DeveloperGuide/AddingNewComponents/DevelopingDatabases/index.rst index dcad8ed8d18..4f44e4a9ee3 100644 --- a/docs/source/DeveloperGuide/AddingNewComponents/DevelopingDatabases/index.rst +++ b/docs/source/DeveloperGuide/AddingNewComponents/DevelopingDatabases/index.rst @@ -3,7 +3,7 @@ Developing Databases ==================== This is a quick guide about developing classes interacting with MySQL databases. -DIRAC supports also Elasticsearch NoSQL database, but it is not part of this document. +DIRAC supports also OpenSearch NoSQL database, but it is not part of this document. Before starting developing databases, you have to make sure that MySQL is installed, as well as python-mysql, as explained in :ref:`editing_code`, and make sure that MySQL service is on. @@ -36,7 +36,7 @@ The *DB* class includes all the methods necessary to access, query, modify... th The first line in the ``__init__`` method should be the initialization of the parent (*DB*) class. That initialization requires 2 or 3 parameters: 1. Logging name of the database. This name will be used in all the logging messages generated by this class. - 2. Full name of the database. With *System/Name*. So it can know where in the CS look for the initialization parameters. In this case it would be */Systems/Test//Databases/AtomDB*. + 2. Full name of the database. With *System/Name*. So it can know where in the CS look for the initialization parameters. In this case it would be */Systems/Test/Databases/AtomDB*. 3. Boolean for the debug flag After the initialization of the *DB* parent class we call our own ``__initializeDB`` method. @@ -50,34 +50,29 @@ The *addStuff* method simply inserts into the created table the argument value. Configure the database access ============================== -The last step is to configure the database credentials for DIRAC to be able to connect. In our previous example the CS path was */Systems/Test//Databases/AtomDB*. That section should contain:: +The last step is to configure the database credentials for DIRAC to be able to connect. In our previous example the CS path was */Systems/Test/Databases/AtomDB*. That section should contain:: Systems { Test { - + Databases { - Databases + AtomDB { - AtomDB - { - Host = localhost - User = yourusername - Password = yourpasswd - DBName = yourdbname - } - } - } + Host = localhost + User = yourusername + Password = yourpasswd + DBName = yourdbname + } + } } In a production environment, the "Password" should be defined in a non-accessible file, while the rest of the configuration can go in the central Configuration Service. -If you encounter any problem with sockets, you should replace "localhost" (DIRAC/Systems/Test//AtomDB/Host) by 127.0.0.1. - -Keep in mind that is the name of the instance defined under */DIRAC/Setups//Test* and is defined under */DIRAC/Setup*. +If you encounter any problem with sockets, you should replace "localhost" (DIRAC/Systems//AtomDB/Host) by 127.0.0.1. Once that is defined you're ready to go. diff --git a/docs/source/DeveloperGuide/AddingNewComponents/DevelopingExecutors/PingPongMindHandler.py b/docs/source/DeveloperGuide/AddingNewComponents/DevelopingExecutors/PingPongMindHandler.py index 5655d7bbde8..a6bff2e8a32 100644 --- a/docs/source/DeveloperGuide/AddingNewComponents/DevelopingExecutors/PingPongMindHandler.py +++ b/docs/source/DeveloperGuide/AddingNewComponents/DevelopingExecutors/PingPongMindHandler.py @@ -13,7 +13,6 @@ class PingPongMindHandler(ExecutorMindHandler): - MSG_DEFINITIONS = {"StartReaction": {"numBounces": int}} auth_msg_StartReaction = ["all"] @@ -28,7 +27,7 @@ def msg_StartReaction(self, msgObj): def export_startPingOfDeath(self, numBounces): taskData = {"bouncesLeft": numBounces} - sLog.info("START TASK", "%s" % taskData) + sLog.info("START TASK", f"{taskData}") return self.executeTask(int(time.time() + random.random()), taskData) @classmethod @@ -38,7 +37,7 @@ def exec_executorConnected(cls, trid, eTypes): eTypes is a list of executor modules the reactor runs """ - sLog.info("EXECUTOR CONNECTED OF TYPE", "%s" % eTypes) + sLog.info("EXECUTOR CONNECTED OF TYPE", f"{eTypes}") return S_OK() @classmethod @@ -53,7 +52,7 @@ def exec_dispatch(cls, taskid, taskData, pathExecuted): """ Before a task can be executed, the mind has to know which executor module can process it """ - sLog.info("IN DISPATCH", "%s" % taskData) + sLog.info("IN DISPATCH", f"{taskData}") if taskData["bouncesLeft"] > 0: sLog.info("SEND TO PLACE") return S_OK("Test/PingPongExecutor") @@ -66,12 +65,12 @@ def exec_prepareToSend(cls, taskId, taskData, trid): @classmethod def exec_serializeTask(cls, taskData): - sLog.info("SERIALIZE", "%s" % taskData) + sLog.info("SERIALIZE", f"{taskData}") return S_OK(DEncode.encode(taskData)) @classmethod def exec_deserializeTask(cls, taskStub): - sLog.info("DESERIALIZE", "%s" % taskStub) + sLog.info("DESERIALIZE", f"{taskStub}") return S_OK(DEncode.decode(taskStub)[0]) @classmethod @@ -79,7 +78,7 @@ def exec_taskProcessed(cls, taskid, taskData, eType): """ This function will be called when a task has been processed and by which executor module """ - sLog.info("PROCESSED", "%s" % taskData) + sLog.info("PROCESSED", f"{taskData}") taskData["bouncesLeft"] -= 1 return cls.executeTask(taskid, taskData) diff --git a/docs/source/DeveloperGuide/AddingNewComponents/Resources/Computing.rst b/docs/source/DeveloperGuide/AddingNewComponents/Resources/Computing.rst new file mode 100644 index 00000000000..3463eab5f48 --- /dev/null +++ b/docs/source/DeveloperGuide/AddingNewComponents/Resources/Computing.rst @@ -0,0 +1,5 @@ +---------------- +ComputingElement +---------------- + +The full code documentation is available here :py:class:`~DIRAC.Resources.Computing.ComputingElement` diff --git a/docs/source/DeveloperGuide/AddingNewComponents/Resources/index.rst b/docs/source/DeveloperGuide/AddingNewComponents/Resources/index.rst index d12c00bf497..16af11ebcbc 100644 --- a/docs/source/DeveloperGuide/AddingNewComponents/Resources/index.rst +++ b/docs/source/DeveloperGuide/AddingNewComponents/Resources/index.rst @@ -16,5 +16,6 @@ DIRAC applications :maxdepth: 2 Catalog + Computing MessageQueues/index Storage diff --git a/docs/source/DeveloperGuide/AddingNewComponents/Utilities/CSHelpers/Operations/index.rst b/docs/source/DeveloperGuide/AddingNewComponents/Utilities/CSHelpers/Operations/index.rst index 8cfe5c32ee5..e35093e0d80 100644 --- a/docs/source/DeveloperGuide/AddingNewComponents/Utilities/CSHelpers/Operations/index.rst +++ b/docs/source/DeveloperGuide/AddingNewComponents/Utilities/CSHelpers/Operations/index.rst @@ -4,8 +4,8 @@ Helper for accessing /Operations */Operations* section is *VO* and *setup* aware. That means that configuration for different *VO/setup* will have a different CS path: - * For multi-VO installations */Operations//* should be used. - * For single-VO installations */Operations/* should be used. + * For multi-VO installations */Operations/* should be used. + * For single-VO installations */Operations* should be used. In any case, there is the possibility to define a default configuration, that is valid for all the *setups*. The *Defaults* keyword can be used instead of the setup. For instance */Operations/myvo/Defaults*. diff --git a/docs/source/DeveloperGuide/AddingNewComponents/Utilities/gLogger/Backends/index.rst b/docs/source/DeveloperGuide/AddingNewComponents/Utilities/gLogger/Backends/index.rst index 19c2d0e6351..4b2d96c62bb 100644 --- a/docs/source/DeveloperGuide/AddingNewComponents/Utilities/gLogger/Backends/index.rst +++ b/docs/source/DeveloperGuide/AddingNewComponents/Utilities/gLogger/Backends/index.rst @@ -53,12 +53,12 @@ Parameters +-----------+------------------------------------------------------------------+----------------------+ -ElasticSearchBackend +OpenSearch Backend -------------------- Description ~~~~~~~~~~~ -Used to emit log records in the an ElasticSearch database. +Used to emit log records in the an OpenSearch database. The *Backend* acccepts logs from *Debug* to *Always* level. Parameters @@ -66,15 +66,15 @@ Parameters +-----------+------------------------------------------------------------------+----------------------+ | Option | Description | Default value | +===========+==================================================================+======================+ -| Host | host machine where the ElasticSearch DB is installed | '' | +| Host | host machine where the OpenSearch DB is installed | '' | +-----------+------------------------------------------------------------------+----------------------+ -| Port | port where the ElasticSearch DB listen | 9203 | +| Port | port where the OpenSearch DB listen | 9203 | +-----------+------------------------------------------------------------------+----------------------+ -| User | username of the ElasticSearch DB (optional) | None | +| User | username of the OpenSearch DB (optional) | None | +-----------+------------------------------------------------------------------+----------------------+ -| Password | password of the ElasticSearch DB (optional) | None | +| Password | password of the OpenSearch DB (optional) | None | +-----------+------------------------------------------------------------------+----------------------+ -| Index | ElasticSearch index | '' | +| Index | OpenSearch index | '' | +-----------+------------------------------------------------------------------+----------------------+ | BufferSize| maximum size of the buffer before sending | 1000 | +-----------+------------------------------------------------------------------+----------------------+ diff --git a/docs/source/DeveloperGuide/AddingNewComponents/Utilities/gLogger/Changes/index.rst b/docs/source/DeveloperGuide/AddingNewComponents/Utilities/gLogger/Changes/index.rst index 4fe116cca2b..beef03c9e0a 100644 --- a/docs/source/DeveloperGuide/AddingNewComponents/Utilities/gLogger/Changes/index.rst +++ b/docs/source/DeveloperGuide/AddingNewComponents/Utilities/gLogger/Changes/index.rst @@ -185,10 +185,10 @@ there is the old exception display, at the bottom the new: a = 1/0 ZeroDivisionError: integer division or modulo by zero -*registerBackends() and registerBackend()* for all loggers +*registerBackend()* for all loggers ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Now, each *Logging* can use the *registerBackend(s)* method for their own +Now, each *Logging* can use the *registerBackend* method for their own needs. In this way, you can easily isolate log records from a specific *Logging* object. @@ -203,7 +203,8 @@ these ones to its parent and so on. Thus, all log records from all gLogger.registerBackend('stdout') log = gLogger.getSubLogger('log') - log.registerBackends(['stderr', 'stdout']) + log.registerBackend('stderr') + log.registerBackend('stdout') sublog = log.getSubLogger('sublog') diff --git a/docs/source/DeveloperGuide/AddingNewComponents/Utilities/gLogger/gLogger/Advanced/index.rst b/docs/source/DeveloperGuide/AddingNewComponents/Utilities/gLogger/gLogger/Advanced/index.rst index abea005ed22..c754041aff8 100644 --- a/docs/source/DeveloperGuide/AddingNewComponents/Utilities/gLogger/gLogger/Advanced/index.rst +++ b/docs/source/DeveloperGuide/AddingNewComponents/Utilities/gLogger/gLogger/Advanced/index.rst @@ -111,7 +111,7 @@ its level with the *setLevel* method. Add a *Backend* object on a child *Logging* ------------------------------------------- -*registerBackend(s)* presentation +*registerBackend* presentation ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Now, it is possible to add some *Backend* objects to any *Logging* via @@ -124,12 +124,8 @@ names and their values associated. Here is an example of use: logger = gLogger.getSubLogger("logger") logger.registerBackend('stdout') logger.registerBackend('file', {'FileName': 'file.log'}) - # An alternative: - # logger.registerBackends(['stdout', 'file'], {'FileName': 'file.log'}) -This, will create *stdout* and *file Backend* objects in *logger*. The alternative method -named *registerBackends* takes a *Backend* objects list as first argument. This method can be really efficient -to add some *Backend* objects in one time but also restrictive due to the unicity of the dictionary keys. +This, will create *stdout* and *file Backend* objects in *logger*. Log records propagation ~~~~~~~~~~~~~~~~~~~~~~~ @@ -221,6 +217,68 @@ This option can not be modified in the children of *gLogger*, even by *gLogger* itself after the configuration, so the children receive the *gLogger* configuration. +Add variables to different *Logging* objects depending on the context +--------------------------------------------------------------------- + +In complex cases, it can be useful to have loggers that change depending on +the execution context, without having to pass logger instances explicitly +through multiple layers of function calls. + +Python's `contextvars` module provides context-local storage, which can be used +to store and retrieve context-specific data, such as logger instances. + +gLogger supports the use of context variables to manage loggers in a flexible way. + +Provide a Context Logger +~~~~~~~~~~~~~~~~~~~~~~~~ + +When you have a *Logging* instance that you want to use in a specific context, +you can set it in the context variable: + +:: + + # Create a logger instance + logger = gLogger.getSubLogger("MyContextLogger") + + # Set it in the context variable + contextLogger.set(logger) + +Then, the instances within the context block will use the shared *Logging* object +set in the context variable: + +:: + + with setContextLogger(contextualLogger): + # Any logging within this block will use contextualLogger + obj = MyClass() + obj.do_something() # This will use contextualLogger + +Consume a Context Logger +~~~~~~~~~~~~~~~~~~~~~~~~ + +In functions or classes that need to log messages, you can retrieve the logger +from the context variable: + +:: + + class MyClass: + def __init__(self): + # Get the default logger if no context logger is set + self._defaultLogger = gLogger.getSubLogger("MyClass") + + @property + def log(self): + # Return the context logger if set, otherwise the default logger + return contextLogger.get() or self._defaultLogger + + @log.setter + def log(self, value): + # Optionally, allow setting a new default logger + self._defaultLogger = value + + def do_something(self): + self.log.notice("Doing something") + Some examples and summaries --------------------------- diff --git a/docs/source/DeveloperGuide/AddingNewComponents/Utilities/gLogger/gLogger/Basics/index.rst b/docs/source/DeveloperGuide/AddingNewComponents/Utilities/gLogger/gLogger/Basics/index.rst index c4333d98e3f..c33a4a6388d 100644 --- a/docs/source/DeveloperGuide/AddingNewComponents/Utilities/gLogger/gLogger/Basics/index.rst +++ b/docs/source/DeveloperGuide/AddingNewComponents/Utilities/gLogger/gLogger/Basics/index.rst @@ -761,7 +761,7 @@ To summarize, this file configures two agents respectively named *SimplestAgent* In *SimplestAgent*, it sets the level of *gLogger* at *info*, adds 5 *Backend* objects to it, which are *stdout*, *stderr*, two *file Backend* objects and an *ElastiSearch* access. Thus, each log record superior to *info* level, created by a *Logging* object in the agent, will be sent -to 5 different outputs: *stdout*, *stderr*, */tmp/logtmp.log*, */tmp/logtmp2.log* and ElasticSearch. In *AnotherAgent*, the same process is performed, and each log record superior to *notice* level is sent to *stdout* and another ElasticSearch database because of the redifinition. None of the default *Backend* objects of the *Operations* section are used because of the overwriting. +to 5 different outputs: *stdout*, *stderr*, */tmp/logtmp.log*, */tmp/logtmp2.log* and OpenSearch. In *AnotherAgent*, the same process is performed, and each log record superior to *notice* level is sent to *stdout* and another OpenSearch database because of the redifinition. None of the default *Backend* objects of the *Operations* section are used because of the overwriting. In addition, the log records will be not displayed with color. Summary of the command line argument configuration diff --git a/docs/source/DeveloperGuide/AddingNewComponents/dirac.cfg.basic.example b/docs/source/DeveloperGuide/AddingNewComponents/dirac.cfg.basic.example index 29ad986af3d..7e34b27372b 100644 --- a/docs/source/DeveloperGuide/AddingNewComponents/dirac.cfg.basic.example +++ b/docs/source/DeveloperGuide/AddingNewComponents/dirac.cfg.basic.example @@ -2,18 +2,6 @@ LocalSite { Site = DIRAC.DevBox.org } -DIRAC -{ - Setup = DeveloperSetup - Setups - { - DeveloperSetup - { - Framework = DevInstance - Test = DevInstance - } - } -} Systems { Database @@ -72,7 +60,6 @@ Registry Properties += Operator Properties += CSAdministrator Properties += ProductionManagement - Properties += AlarmsManagement Properties += TrustedHost Properties += SiteManager } @@ -84,7 +71,6 @@ Registry Properties += Operator Properties += CSAdministrator Properties += ProductionManagement - Properties += AlarmsManagement Properties += TrustedHost Properties += SiteManager } diff --git a/docs/source/DeveloperGuide/CodeTesting/index.rst b/docs/source/DeveloperGuide/CodeTesting/index.rst index d5c7f168c50..dab9d018f6c 100644 --- a/docs/source/DeveloperGuide/CodeTesting/index.rst +++ b/docs/source/DeveloperGuide/CodeTesting/index.rst @@ -41,7 +41,7 @@ Testing itself could also speed up the development process rapidly tracing probl DIRAC is not different from that scenario, with the exception that service-oriented architecture paradigm, which is one of the basic concepts of the project, making the quality assurance and testing process the real challenge. However as DIRAC becomes more and more popular and now is being used by several different communities, -the main question is not: *to test or not to test?*, but rather: *how to test in an efficient way?* +the main question is not: *to test or not to test?*, but rather: *how to test in an efficient way?* [#]_. The topic of software testing is very complicated by its own nature, but depending on the testing method employed, the testing process itself can be implemented at any time in the development phase and ideally should cover many different levels of the system: @@ -89,13 +89,13 @@ This could be obtained by objects mocking technique, where all fragile component equivalents - test doubles. For that it is recommended to use mock_ module. Hence it is clear that knowledge of mock_ module API is essential. -Unit tests are typically created by the developer who will also write the code that is being tested. The tests may therefore share the same blind spots with the code: for example, a developer does not realize that certain input parameters must be checked, most likely neither the test nor the code will verify these input parameters. If the developer misinterprets the requirements specification for the module being developed, both the tests and the code will be wrong. Hence if the developer is going to prepare her own unit tests, she should pay attention and take extra care to implement proper testing suite, checking for every spot of possible failure (i.e. interactions with other components) and not trusting that someone else's code is always returning proper type and/or values. +Unit tests are typically created by the developer who will also write the code that is being tested. The tests may therefore share the same blind spots with the code: for example, a developer does not realize that certain input parameters must be checked, most likely neither the test nor the code will verify these input parameters [#]_. If the developer misinterprets the requirements specification for the module being developed, both the tests and the code will be wrong. Hence if the developer is going to prepare her own unit tests, she should pay attention and take extra care to implement proper testing suite, checking for every spot of possible failure (i.e. interactions with other components) and not trusting that someone else's code is always returning proper type and/or values. Test doubles ============ -Unit tests should run in *isolation*. Which means that they should run without having DIRAC fully installed, because, remember, they should just test the code logic. If, to run a unit test in DIRAC, you need a dirac.cfg file to be present, you are failing your goal. +Unit tests should run in *isolation*. Which means that they should run without having DIRAC fully installed, because, remember, they should just test the code logic. If, to run a unit test in DIRAC, you need a dirac.cfg file to be present, you are failing your goal [#]_. To isolate the code being tested from depended-on components it is convenient and sometimes necessary to use *test doubles*: simplified objects or procedures, that behaves and looks like the their real-intended counterparts, but are actually simplified versions @@ -248,7 +248,7 @@ Continuous Integration software ------------------------------- There are several tools, on the free market, for so-called *Continuous Integration*, or simply CI_. -One possibility is to use Jenkins, but today (from branch *rel-v7r0*) all DIRAC integration tests are run +One possibility is to use Jenkins, but all DIRAC integration tests are run by `GitHub Actions `_ If you have looked in the `DIRAC/tests `_ @@ -273,7 +273,7 @@ What can you do with those above? You can run the Integration tests you read abo How do I do that? - you need a MySQL DB somewhere, empty, to be used only for testing purposes (in GitHub Actions and GitLab-CI a docker container is instantiated for the purpose) -- you need a ElasticSearch instance running somewhere, empty, to be used only for testing purposes (in GitHub Actions and GitLab-CI a docker container is instantiated for the purpose) +- you need a OpenSearch instance running somewhere, empty, to be used only for testing purposes (in GitHub Actions and GitLab-CI a docker container is instantiated for the purpose) - if you have tests that need to access other DBs, you should also have them ready, again used for testing purposes. The files ``DIRAC/tests/Integration/all_integration_client_tests.sh`` and ``DIRAC/tests/Integration/all_integration_server_tests.sh`` @@ -309,23 +309,33 @@ Running the above might take a while. Supposing you are interested in running on ./integration_tests.py prepare-environment [FLAGS] ./integration_tests.py install-server -which (in some minutes) will give you a fully dockerized server setup (`docker container ls` will list the created container, and you can see what's going on inside with the standard `docker exec -it server /bin/bash`). Now, suppose that you want to run `WorkloadManagementSystem/Test_JobDB.py`. -The first thing to do is that you should first login in the docker container, by doing: +which (in some minutes) will give you a fully dockerized server setup +(`docker container ls` will list the created container, and you can see what's going on inside with the standard `docker exec -it server /bin/bash`. +Now, suppose that you want to run `WorkloadManagementSystem/Test_JobDB.py`, +the first thing to do is that you should first login in the docker container, by doing: .. code-block:: bash ./integration_tests.py exec-server -The installations automatically pick up external changes to the DIRAC code and tests) +(The docker installation automatically picks up external changes to the DIRAC code and tests) Now you can run the test with: .. code-block:: bash - pytest LocalRepo/ALTERNATIVE_MODULES/DIRAC/tests/Integration/WorkloadManagementSystem/Test_JobDB.py + pytest --no-check-dirac-environment LocalRepo/ALTERNATIVE_MODULES/DIRAC/tests/Integration/WorkloadManagementSystem/Test_JobDB.py You can find the logs of the services in `/home/dirac/ServerInstallDIR/diracos/runit/` +You can also login in client and mysql with: + +.. code-block:: bash + + ./integration_tests.py exec-client + ./integration_tests.py exec-mysql + + Validation and System tests --------------------------- @@ -361,9 +371,9 @@ Footnotes ============ .. [#] Or even better software requirements document, if any of such exists. Otherwise this is a great opportunity to prepare one. +.. [#] You may ask: *isn't it silly?* No, in fact it isn't. Validation of input parameters is one of the most important tasks during testing. .. [#] To better understand this term, think about a movie industry: if a scene movie makers are going to film is potentially dangerous and unsafe for the leading actor, his place is taken over by a stunt double. .. [#] And eventually is killing him with a gun. At least in a TV show. -.. [#] You may ask: *isn't it silly?* No, in fact it isn't. Validation of input parameters is one of the most important tasks during testing. .. _Python: http://www.python.org/ diff --git a/docs/source/DeveloperGuide/CodingConvention/index.rst b/docs/source/DeveloperGuide/CodingConvention/index.rst index 9501c7630a9..75be08f1749 100644 --- a/docs/source/DeveloperGuide/CodingConvention/index.rst +++ b/docs/source/DeveloperGuide/CodingConvention/index.rst @@ -30,6 +30,7 @@ Commit messages Commit messages must be between 20 and 150 chars, and follow the format ``(): ``: + * ``type``: docs, feat, fix, refactor, style or test * ``scope`` (optional): any extra info, (like DMS or whatever) diff --git a/docs/source/DeveloperGuide/DevelopmentEnvironment/DeveloperInstallation/editingCode.rst b/docs/source/DeveloperGuide/DevelopmentEnvironment/DeveloperInstallation/editingCode.rst index 018554ba2ef..d91517f9cd7 100644 --- a/docs/source/DeveloperGuide/DevelopmentEnvironment/DeveloperInstallation/editingCode.rst +++ b/docs/source/DeveloperGuide/DevelopmentEnvironment/DeveloperInstallation/editingCode.rst @@ -104,7 +104,7 @@ You can create a development environment in a new directory named ``diracos/`` b .. code-block:: bash :linenos: - wget https://github.com/DIRACGrid/DIRACOS2/releases/download/latest/DIRACOS-Linux-x86_64.sh + wget https://github.com/DIRACGrid/DIRACOS2/releases/latest/download/DIRACOS-Linux-x86_64.sh bash DIRACOS-Linux-x86_64.sh rm DIRACOS-Linux-x86_64.sh diff --git a/docs/source/DeveloperGuide/DevelopmentEnvironment/DeveloperInstallation/interactingWithProductionSetups.rst b/docs/source/DeveloperGuide/DevelopmentEnvironment/DeveloperInstallation/interactingWithProductionSetups.rst index 9f2cb144b20..e474859578f 100644 --- a/docs/source/DeveloperGuide/DevelopmentEnvironment/DeveloperInstallation/interactingWithProductionSetups.rst +++ b/docs/source/DeveloperGuide/DevelopmentEnvironment/DeveloperInstallation/interactingWithProductionSetups.rst @@ -11,7 +11,6 @@ So, the only real thing that you need to have is: - a DIRAC developer installation - a (real) certificate, that is recognized by your server installation -- a dirac.cfg that include the (real) setup of the production environment that you want to connect to (in DIRAC/Setup section) - a dirac.cfg that include the (real) URL of the production Configuration server. The last 2 bullets can be achieved with the following command:: diff --git a/docs/source/DeveloperGuide/DevelopmentEnvironment/DeveloperInstallation/stuffThatRun.rst b/docs/source/DeveloperGuide/DevelopmentEnvironment/DeveloperInstallation/stuffThatRun.rst index 46dc505a6db..df19860b3c7 100644 --- a/docs/source/DeveloperGuide/DevelopmentEnvironment/DeveloperInstallation/stuffThatRun.rst +++ b/docs/source/DeveloperGuide/DevelopmentEnvironment/DeveloperInstallation/stuffThatRun.rst @@ -22,7 +22,7 @@ you better keep reading. A docker-based isolated environment =================================== -position yourself in the DIRAC root directory and then run: +Position yourself in the DIRAC root directory and then run: .. code-block:: bash @@ -47,7 +47,7 @@ Now you can run the test with: .. code-block:: bash - pytest LocalRepo/ALTERNATIVE_MODULES/DIRAC/tests/Integration/WorkloadManagementSystem/Test_JobDB.py +pytest --no-check-dirac-environment LocalRepo/ALTERNATIVE_MODULES/DIRAC/tests/Integration/WorkloadManagementSystem/Test_JobDB.py You can find the logs of the services in `/home/dirac/ServerInstallDIR/diracos/runit/` @@ -56,7 +56,7 @@ The Configuration Server (the CS) ================================= At some point you'll need to understand how the DIRAC -`Configuration Service (CS) :ref:`dirac-cs-structure` works. I'll explain here briefly. +Configuration Service (CS) :ref:`dirac-cs-structure` works. I'll explain here briefly. The CS is a layered structure: whenever you access a CS information (e.g. using a "gConfig" object, see later), @@ -65,81 +65,10 @@ home as ".dirac.cfg", or in *etc/* directory, see the link above). If this will not be found, it will look for such info in the CS servers available. When you develop locally, you don't need to access any CS server: instead, you need to have total control. -So, you need to work a bit on the local dirac.cfg file. There is not much else needed, just create your own etc/dirac.cfg. -The example that follows might not be easy to understand at a first sight, but it will become easy soon. -The syntax is extremely simple, yet verbose: simply, only brackets and equalities are used. - -If you want to create an isolated installation just create a -*$DEVROOT/etc/dirac.cfg* file with:: - - DIRAC - { - Setup = DeveloperSetup - Setups - { - DeveloperSetup - { - Framework = DevInstance - Test = DevInstance - } - } - } - Systems - { - Framework - { - DevInstance - { - URLs - { - } - Services - { - } - } - } - Test - { - DevInstance - { - URLs - { - } - Services - { - } - } - } - } - Registry - { - Users - { - yourusername - { - DN = /your/dn/goes/here - Email = youremail@yourprovider.com - } - } - Groups - { - devGroup - { - Users = yourusername - Properties = CSAdministrator, JobAdministrator, ServiceAdministrator, ProxyDelegation, FullDelegation - } - } - Hosts - { - mydevbox - { - DN = /your/box/dn/goes/here - Properties = CSAdministrator, JobAdministrator, ServiceAdministrator, ProxyDelegation, FullDelegation - } - } - } - -Within the code we also provide a pre-filled example of dirac.cfg. You can get it simply doing (on the host):: +The docker-based setup created by `integration_tests.py` will take care of creating the dirac.cfg file for you. + +In case you want to work outside of the setup created by `integration_tests.py`, +we also provide a pre-filled example of dirac.cfg. You can get it simply doing:: cp $DEVROOT/DIRAC/docs/source/DeveloperGuide/AddingNewComponents/dirac.cfg.basic.example $DEVROOT/etc/dirac.cfg @@ -157,7 +86,6 @@ Still, you CAN run DIRAC services without any certificate. The reason is that, while the use of TLS/SSL and certificates is the default, you can still go away without it, simply disabling TLS/SSL. You'll see how later. So, if you find difficulties with this subsection, the good news is that you don't strictly need it. - Anyway: DIRAC understands certificates in *pem* format. That means that a certificate set will consist of two files. Files ending in *cert.pem* can be world readable but just user writable since it contains the certificate and public key. Files ending in *key.pem* should be only user readable since they contain @@ -165,13 +93,9 @@ the private key. You will need two different sets certificates and the CA certif The following commands should do the trick for you, by creating a fake CA, a fake user certificate, and a fake host certificate:: cd $DEVROOT/DIRAC - git checkout release/integration - source tests/Jenkins/utilities.sh - generateCA - generateCertificates 365 - generateUserCredentials 365 + docker run ghcr.io/diracgrid/diracx/certificates-generation:latest mkdir -p ~/.globus/ - cp $DEVROOT/user/*.{pem,key} ~/.globus/ + docker cp certificates-generation:/ca/certs/client.{pem,key} ~/.globus/ mv ~/.globus/client.key ~/.globus/userkey.pem mv ~/.globus/client.pem ~/.globus/usercert.pem diff --git a/docs/source/DeveloperGuide/Internals/Core/ClientServer.rst b/docs/source/DeveloperGuide/Internals/Core/ClientServer.rst index 944289afe6c..a02da426b28 100644 --- a/docs/source/DeveloperGuide/Internals/Core/ClientServer.rst +++ b/docs/source/DeveloperGuide/Internals/Core/ClientServer.rst @@ -166,10 +166,10 @@ Complete path of packages are not on the diagram for readability: - requestHandler: :py:class:`DIRAC.Core.DISET.RequestHandler` -You can see that the client sends a proposalTuple, proposalTuple contain (service, setup, ClientVO) then (typeOfCall, method) and finaly extra-credentials. +You can see that the client sends a proposalTuple, proposalTuple contain (service, ClientVO) then (typeOfCall, method) and finaly extra-credentials. e.g.:: - (('Framework/serviceName', 'DeveloperSetup', 'unknown'), ('RPC', 'methodName'), '') + (('Framework/serviceName', 'unknown'), ('RPC', 'methodName'), '') diff --git a/docs/source/DeveloperGuide/Internals/Core/componentsAuthNandAuthZ.rst b/docs/source/DeveloperGuide/Internals/Core/componentsAuthNandAuthZ.rst index 5fe8a12bb5a..321f47df620 100644 --- a/docs/source/DeveloperGuide/Internals/Core/componentsAuthNandAuthZ.rst +++ b/docs/source/DeveloperGuide/Internals/Core/componentsAuthNandAuthZ.rst @@ -10,7 +10,7 @@ for authentication and authorization purposes. Components can be instructed to use a "shifter proxy" for authN and authZ of their service calls. A shifter proxy is proxy certificate, which should be: -- specified in the "Operations//Shifter" section of the CS +- specified in the "Operations/Shifter" section of the CS - uploaded to the ProxyManager (i.e. using "--upload" option of dirac-proxy-init) Within an agent, in the "initialize" method, we can specify:: diff --git a/docs/source/DeveloperGuide/Overview/index.rst b/docs/source/DeveloperGuide/Overview/index.rst index 22c4c8353a9..c93b5856728 100644 --- a/docs/source/DeveloperGuide/Overview/index.rst +++ b/docs/source/DeveloperGuide/Overview/index.rst @@ -112,7 +112,6 @@ Interfaces control the ongoing tasks. The Web interfaces are based on the DIRAC Web Portal framework which ensures secure access to the system service using X509 certificates loaded into the user browsers. -.. versionadded:: 8.0 DIRAC also has the ability to implement additional interfaces based on the http protocol, see :ref:`apis`. DIRAC Framework @@ -132,7 +131,7 @@ Configuration Service The Configuration Service is built in the DISET framework to provide static configuration parameters to all the distributed DIRAC components. This is the backbone of the whole system and necessitates excellent reliability. Therefore, it is organized as a single master service where all the parameter -updates are done and multiple read-only slave services which are distributed geographically, on VO-boxes +updates are done and multiple read-only worker services which are distributed geographically, on VO-boxes at Tier-1 LCG sites in the case of LHCb. All the servers are queried by clients in a load balancing way. This arrangement ensures configuration data consistency together with very good scalability properties. diff --git a/docs/source/DeveloperGuide/Systems/Framework/index.rst b/docs/source/DeveloperGuide/Systems/Framework/index.rst index d923ea7c28b..a7eb5c7493d 100644 --- a/docs/source/DeveloperGuide/Systems/Framework/index.rst +++ b/docs/source/DeveloperGuide/Systems/Framework/index.rst @@ -111,6 +111,6 @@ The MonitoringUtilities module provides the functionality needed to store or del Dynamic Component Monitoring ============================ -This system takes care of managing monitoring information of DIRAC component. It is based on ElasticSearch database. It is based on MonitoringSystem. +This system takes care of managing monitoring information of DIRAC component. It is based on OpenSearch. The information is collected by the __storeProfiling periodic task on the SystemAdministartor. The task is disabled by default. The MonitoringReporter is used to propagate the DB whith the collected values. diff --git a/docs/source/DeveloperGuide/Systems/Framework/stableconns/client.py b/docs/source/DeveloperGuide/Systems/Framework/stableconns/client.py index 50bfe5193de..e60040b5dc9 100644 --- a/docs/source/DeveloperGuide/Systems/Framework/stableconns/client.py +++ b/docs/source/DeveloperGuide/Systems/Framework/stableconns/client.py @@ -48,11 +48,11 @@ def disconnectedCB(msgClient): msgClient.subscribeToDisconnect(disconnectedCB) result = msgClient.connect() if not result["OK"]: - print("CANNOT CONNECT: %s" % result["Message"]) + print(f"CANNOT CONNECT: {result['Message']}") sys.exit(1) result = sendPingMsg(msgClient) if not result["OK"]: - print("CANNOT SEND PING: %s" % result["Message"]) + print(f"CANNOT SEND PING: {result['Message']}") sys.exit(1) # Wait 10 secs of pingpongs :P time.sleep(10) diff --git a/docs/source/DeveloperGuide/Systems/Framework/stableconns/service.py b/docs/source/DeveloperGuide/Systems/Framework/stableconns/service.py index 14324287a7d..c1c6d2aaba3 100644 --- a/docs/source/DeveloperGuide/Systems/Framework/stableconns/service.py +++ b/docs/source/DeveloperGuide/Systems/Framework/stableconns/service.py @@ -17,7 +17,6 @@ class PingPongHandler(RequestHandler): - MSG_DEFINITIONS = {"Ping": {"id": int}, "Pong": {"id": int}} auth_conn_connected = ["all"] diff --git a/docs/source/DeveloperGuide/Systems/Monitoring/index.rst b/docs/source/DeveloperGuide/Systems/Monitoring/index.rst index aff34d7239c..c6811bd70d0 100644 --- a/docs/source/DeveloperGuide/Systems/Monitoring/index.rst +++ b/docs/source/DeveloperGuide/Systems/Monitoring/index.rst @@ -14,8 +14,7 @@ The system is storing monitoring information. It means the data stored in the da -computing infrastructures (for example: machines, etc.) -data movement (for example: Data operation etc.) -This system is based on ElasticSearch, RabbitMQ and DIRAC plotting facilities. It allows to introduce new monitoring types by adding -minimal code. +This system is based on OpenSearch. ------------ Architecture @@ -31,9 +30,9 @@ It is based on layered architecture and is based on DIRAC architecture: * **DB** * MonitoringDB: - It is a based on ElasticSearch database and provides all the methods which needed to create the reports. Currently, it supports only + It is a based on OpenSearch database and provides all the methods which needed to create the reports. Currently, it supports only one type of query: It creates a dynamic buckets which will be used to retrieve the data points. The query used to retrieve the data points - is retrieveBucketedData. As you can see it uses the ElasticSearch QueryDSL language. Before you modify this method please learn this language. + is retrieveBucketedData. As you can see it uses the OpenSearch QueryDSL language. Before you modify this method please learn this language. * private: - Plotters: It contains all Plotters used to create the plots. More information will be provided later. diff --git a/docs/source/DeveloperGuide/Systems/RequestManagement/index.rst b/docs/source/DeveloperGuide/Systems/RequestManagement/index.rst index 8dfc4a73d10..27a02dab922 100644 --- a/docs/source/DeveloperGuide/Systems/RequestManagement/index.rst +++ b/docs/source/DeveloperGuide/Systems/RequestManagement/index.rst @@ -201,7 +201,7 @@ The agent will try to execute request as a whole in one go. :alt: Treating of Request in the RequestExecutionAgent. :align: center -The `RequestExecutingAgent` is using the `ProcessPool` utility to create slave workers (subprocesses running `RequestTask`) +The `RequestExecutingAgent` is using the `ProcessPool` utility to create workers (subprocesses running `RequestTask`) designated to execute requests read from `ReqDB`. Each worker is processing request execution using following steps: * downloading and setting up request's owner proxy @@ -238,6 +238,7 @@ The timeout for the operation is then calculated from this value and the number The `ReplicateAndRegister` section accepts extra attributes, specific to FTSTransfers: * FTSMode (default False): if True, delegate transfers to FTS + * DMMode (default True): if False, will not use DataManager as a failover transfer for FTS * FTSBannedGroups: list of DIRAC group whose transfers should not go through FTS. This of course does not cover all possible needs for a specific VO, hence all developers are encouraged to create and keep diff --git a/docs/source/DeveloperGuide/Systems/WorkloadManagement/index.rst b/docs/source/DeveloperGuide/Systems/WorkloadManagement/index.rst index 40e0923f26a..7ec3701e700 100644 --- a/docs/source/DeveloperGuide/Systems/WorkloadManagement/index.rst +++ b/docs/source/DeveloperGuide/Systems/WorkloadManagement/index.rst @@ -78,7 +78,6 @@ It is based on layered architecture and is based on DIRAC architecture: * JobStateUpdateHandler * MatcherHandler * OptimizationMindHandler - * PilotsLoggingHandler * SandboxStoreHandler * WMSAdministratorHandler * WMSUtilities @@ -97,33 +96,9 @@ It is based on layered architecture and is based on DIRAC architecture: This database keeps track of all the submitted grid pilot jobs. It also registers the mapping of the DIRAC jobs to the pilot agents. - * PilotsLoggingDB: - PilotsLoggingDB class is a front-end to the Pilots Logging Database. - This database keeps track of all the submitted grid pilot jobs. - It also registers the mapping of the DIRAC jobs to the pilot agents. - * SandboxMetadataDB SandboxMetadataDB class is a front-end to the metadata for sandboxes. - * ElasticJobParametersDB - ElasticJobParametersDB class is a front-end to the ES-based index providing Job Parameters. - It is used in most of the WMS components and is based on ElasticSearch. - ------------------------------------------- -Using ElasticSearch DB for Job Parameters ------------------------------------------- - -ElasticJobParametersDB is a DB class which is used to interact with an ElasticSearch backend. It contains methods -to retreive (get) information about the Job Parameters along with updating and creating those parameters. - -The class exposes two methods: - - * getJobParameters(JobID, ParamList (optional)): - This method can be used to get information of the Job Parameters based on the JobID. Returns name and value. - Optional ParamList can be given to make the search more specific. - The method uses the search API provided by ElasticSearch-py. - - * setJobParameter(JobID, Name, Value): - This method is used to update the Job Parameters based on the given JobID. Returns result of the operation. - If JobID is not present in the index, it inserts the given values in that day's index. - The method uses the update-by-query and create APIs provided by ElasticSearch-py. + * JobParametersDB + JobParametersDB class is a front-end to the Elastic/OpenSearch based index providing Job Parameters. + It is used in most of the WMS components and is based on Elastic/OpenSearch. diff --git a/docs/source/DeveloperGuide/TornadoServices/index.rst b/docs/source/DeveloperGuide/TornadoServices/index.rst index d9a8c773939..2246fcd5285 100644 --- a/docs/source/DeveloperGuide/TornadoServices/index.rst +++ b/docs/source/DeveloperGuide/TornadoServices/index.rst @@ -17,6 +17,7 @@ This page summarizes the changes between DISET and HTTPS. You can all also see t - `Presentation of HTTPS migration `_. - `Summary presentation (latest) `_. +For installing HTTPs services, please refer to :ref:`https_services` administration page. ******* @@ -101,32 +102,19 @@ How to start server The easy way is to use command ``tornado-start-all`` which will start all services registered in configuration. To register a service you just have to add the service in the CS and ``Protocol = https``. It may look like this:: - DIRAC - { - Setups - { - Tornado = DevInstance - } - } Systems { Tornado { - DevInstance - { - Port = 443 - } + Port = 443 } Framework { - DevInstance + Services { - Services + DummyTornado { - DummyTornado - { - Protocol = https - } + Protocol = https } } } @@ -147,29 +135,8 @@ Options available are: This start method can be useful for developing new service or create starting script for a specific service, like the Configuration System (as master). - -MasterCS special case -********************* - -The master CS is different because it uses the same global variable (``gConfig``) but uses it also to write config. Because of that, it needs to run in a separate process. In order to do so: - -* Do NOT specify ``Protocol=https`` in the service description, otherwise it will be ran with all the other Tornado services -* If you run on the same machine as other TornadoService, specify a ``Port`` in the service description - -Finally, there is no automatic installations script. So just install a CS as you normally would do, and then edit the ``run`` file like that:: - - diff --git a/run b/run.new - index d45dce1..f5f3b55 100755 - --- a/run - +++ b/run.new - @@ -7,6 +7,6 @@ - [ "service" = "agent" ] && renice 20 -p $$ - # - # - - exec python $DIRAC/DIRAC/Core/scripts/dirac-service.py Configuration/Server --cfg /opt/dirac/pro/etc/Configuration_Server.cfg < /dev/null - + export DIRAC_USE_TORNADO_IOLOOP=Yes - + exec tornado-start-CS -ddd - +The master CS is different because it uses the same global variable (``gConfig``) but uses it also to write config. Because of that, it needs to run in a separate process. +It needs to be started with ``tornado-start-CS`` script. TransferClient @@ -355,9 +322,17 @@ Two special python packages are needed: Install a service ***************** -``dirac-install-tornado-service`` is your friend. This will install a runit component running ``tornado-start-all``. +Initial install: first modify one config (with port and so on) before running ``tornado-install`` + +``dirac-install-component`` is your friend. This will install a runit component running ``tornado-start-all``. Nothing is ready yet to install specific tornado service, like the master CS. +Migrate from dips to https +************************** + +comment out port, set Protocol = https, change handler + + Start the server **************** diff --git a/docs/source/DeveloperGuide/index.rst b/docs/source/DeveloperGuide/index.rst index be6369864a9..9d49f03a466 100644 --- a/docs/source/DeveloperGuide/index.rst +++ b/docs/source/DeveloperGuide/index.rst @@ -18,10 +18,8 @@ Detailed instructions on how to develop various types of DIRAC components are gi are discussed as well. More detailes on the available interfaces can be found in the :ref:`code_documentation` part. -For every question, or comment, please open a `GitHub issue `_. -For everything operational, instead, you can write on the `dirac-grid `_ -group. - +For issues, please open a `GitHub issue `_. +For questions, comments, or operational issues, use `GitHub discussions `_. .. toctree:: :maxdepth: 1 diff --git a/docs/source/UserGuide/GettingStarted/GettingUserIdentity/index.rst b/docs/source/UserGuide/GettingStarted/GettingUserIdentity/index.rst index 2cb043e3ec6..76603499119 100644 --- a/docs/source/UserGuide/GettingStarted/GettingUserIdentity/index.rst +++ b/docs/source/UserGuide/GettingStarted/GettingUserIdentity/index.rst @@ -54,9 +54,6 @@ If another non-default user group is needed, the command becomes:: where ``user_group`` is the desired DIRAC group name for which the user is entitled. -.. versionadded:: 8.0 - added the possibility to generate proxy with new `dirac-login` command, use *--help* switch for more information. E.g.: dirac-login - Token authorization ------------------- diff --git a/docs/source/UserGuide/GettingStarted/InstallingClient/index.rst b/docs/source/UserGuide/GettingStarted/InstallingClient/index.rst index 182e303687d..49078f457d5 100644 --- a/docs/source/UserGuide/GettingStarted/InstallingClient/index.rst +++ b/docs/source/UserGuide/GettingStarted/InstallingClient/index.rst @@ -1,8 +1,8 @@ -.. _dirac_install: .. set highlighting to console input/output .. highlight:: console +.. _dirac_install: ======================= Installing DIRAC client @@ -21,9 +21,9 @@ So, you first install DIRACOS2 and only then install DIRAC in it:: and now DIRAC:: - $ pip install DIRAC==8.0 + $ pip install DIRAC -(Just `pip install DIRAC` will install the most recent production version found on https://pypi.org/project/DIRAC/) +will install the most recent production version found on https://pypi.org/project/DIRAC/ And for the configuration:: @@ -34,7 +34,7 @@ Using a user proxy If you want to use a user proxy, we assume that you already have a user certificate, so in this case create a directory *.globus* in your home directory and copy the certificate files -(public and private keys in .pem (Privacy Enhanced Mail format) to this directory:: +`usercert.pem` and `userkey.pem` -- public and private keys in .pem (Privacy Enhanced Mail) format to this directory:: $ mkdir ~/.globus $ cp <> ~/.globus/ diff --git a/docs/source/UserGuide/GettingStarted/UserJobs/CommandLine/index.rst b/docs/source/UserGuide/GettingStarted/UserJobs/CommandLine/index.rst index 58f607fb5f4..0fabf20f62e 100644 --- a/docs/source/UserGuide/GettingStarted/UserJobs/CommandLine/index.rst +++ b/docs/source/UserGuide/GettingStarted/UserJobs/CommandLine/index.rst @@ -30,7 +30,7 @@ is a unique job identifier within the DIRAC Workload Management System. You can the status of the job by giving:: $ dirac-wms-job-status 11758 - JobID=11758 Status=Waiting; MinorStatus=Pilot Agent Submission; Site=CREAM.CNAF.it; + JobID=11758 Status=Waiting; MinorStatus=Pilot Agent Submission; Site=DIRAC.CNAF.it; In the output of the command you get the job Status, Minor Status with more details, and the site to which the job is destinated. diff --git a/docs/source/UserGuide/GettingStarted/UserJobs/JDLReference/index.rst b/docs/source/UserGuide/GettingStarted/UserJobs/JDLReference/index.rst index 7c98925fa55..c0301b624ab 100644 --- a/docs/source/UserGuide/GettingStarted/UserJobs/JDLReference/index.rst +++ b/docs/source/UserGuide/GettingStarted/UserJobs/JDLReference/index.rst @@ -6,84 +6,88 @@ .. role:: subtitle +.. _jdlDescription: + ========================================= Job Description Language Reference ========================================= In this section all the attributes that can be used in the DIRAC JDL job descriptions are presented. -+---------------------+---------------------------------------------+-----------------------------------------------+ -| | -| :subtitle:`The basic JDL parameters` | -| | -| These are the parameters giving the basic job description | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| **Attribute Name** | **Description** | **Example** | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *Executable* | Name of the executable file | Executable = ``"/bin/ls";`` | -| | | | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *Arguments* | String of arguments for the job | Arguments = ``"-ltr";`` | -| | executable | | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *StdError* | Name of the file to get the standard error | StdError = ``"std.err";`` | -| | stream of the user application | | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *StdOutput* | Name of the file to get the standard output | StdOutput = ``"std.out";`` | -| | stream of the user application | | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *InputSandbox* | A list of input sandbox files | InputSandbox = ``{"jobScript.sh"};`` | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *OutputSandbox* | A list of output sandbox files | OutputSandbox = ``{"std.err","std.out"};`` | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| | -| :subtitle:`Job Requirements` | -| | -| These parameters are interpreted as job requirements | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| **Attribute Name** | **Description** | **Example** | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *CPUTime* | Max CPU time required by the job in | CPUTime = 18000; | -| | HEPSPEC06 seconds | | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *Site* | Job destination site | Site = ``{"EGI.CPPM.fr"};`` | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *BannedSites* | Sites where the job must not go | BannedSites = ``{"EGI.LAPP.fr"};`` | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *GridCE* | Job destination CE | GridCE = ``{"some.ce.lapp.fr"};`` | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *Platform* | Target Operating System | Platform = ``"Linux_x86_64_glibc-2.17";`` | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| | -| :subtitle:`Data` | -| | -| Describing job data | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| **Attribute Name** | **Description** | **Example** | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *InputData* | Job input data files | InputData = ``{"/dirac/user/a/atsareg/data1", | -| | | "/dirac/user/a/atsareg/data1"};`` | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *InputDataPolicy* | Job input data policy | InputDataPolicy = ``"Download";`` | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *OutputData* | Job output data files | OutputData = ``{"output1","output2"};`` | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *OutputPath* | The output data path in the File Catalog | OutputPath = ``{"/myjobs/output"};`` | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *OutputSE* | The output data Storage Element | OutputSE = ``{"DIRAC-USER"};`` | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| | -| :subtitle:`Parametric Jobs` | -| | -| Bulk submission parameters | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| **Attribute Name** | **Description** | **Example** | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *Parameters* | Number of parameters or a list of values | Parameters = 10; | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *ParameterStart* | Value of the first parameter | ParameterStart = 0.; | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *ParameterStep* | Parameter increment | ParameterStep = 0.1; (default 0.) | -+---------------------+---------------------------------------------+-----------------------------------------------+ -| *ParameterFactor* | Parameter multiplier | ParameterFactor = 1.1; (default 1.) | -+---------------------+---------------------------------------------+-----------------------------------------------+ ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| | +| :subtitle:`The basic JDL parameters` | +| | +| These are the parameters giving the basic job description | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| **Attribute Name** | **Description** | **Example** | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *Executable* | Name of the executable file | Executable = ``"/bin/ls";`` | +| | | | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *Arguments* | String of arguments for the job | Arguments = ``"-ltr";`` | +| | executable | | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *StdError* | Name of the file to get the standard error | StdError = ``"std.err";`` | +| | stream of the user application | | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *StdOutput* | Name of the file to get the standard output | StdOutput = ``"std.out";`` | +| | stream of the user application | | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *InputSandbox* | A list of input sandbox files | InputSandbox = ``{"jobScript.sh"};`` | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *OutputSandbox* | A list of output sandbox files | OutputSandbox = ``{"std.err","std.out"};`` | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| | +| :subtitle:`Job Requirements` | +| | +| These parameters are interpreted as job requirements | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| **Attribute Name** | **Description** | **Example** | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *CPUTime* | Max CPU time required by the job in | CPUTime = 18000; | +| | HEPSPEC06 seconds | | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *Site* | Job destination site | Site = ``{"EGI.CPPM.fr"};`` | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *BannedSites* | Sites where the job must not go | BannedSites = ``{"EGI.LAPP.fr"};`` | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *GridCE* | Job destination CE | GridCE = ``{"some.ce.lapp.fr"};`` | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *Platform* | Target Operating System | Platform = ``"Linux_x86_64_glibc-2.17";`` | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| | +| :subtitle:`Data` | +| | +| Describing job data | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| **Attribute Name** | **Description** | **Example** | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *InputData* | Job input data files | InputData = ``{"/dirac/user/a/atsareg/data1", | +| | | "/dirac/user/a/atsareg/data1"};`` | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *InputDataModule* | Job input data module | InputDataModule = ``"DIRAC.WorkloadManagementSystem.Client.InputDataResolution"`` | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *InputDataPolicy* | Job input data policy | InputDataPolicy = ``"DIRAC.WorkloadManagementSystem.Client.DownloadInputData";`` | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *OutputData* | Job output data files | OutputData = ``{"output1","output2"};`` | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *OutputPath* | The output data path in the File Catalog | OutputPath = ``{"/myjobs/output"};`` | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *OutputSE* | The output data Storage Element | OutputSE = ``{"DIRAC-USER"};`` | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| | +| :subtitle:`Parametric Jobs` | +| | +| Bulk submission parameters | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| **Attribute Name** | **Description** | **Example** | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *Parameters* | Number of parameters or a list of values | Parameters = 10; | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *ParameterStart* | Value of the first parameter | ParameterStart = 0.; | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *ParameterStep* | Parameter increment | ParameterStep = 0.1; (default 0.) | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ +| *ParameterFactor* | Parameter multiplier | ParameterFactor = 1.1; (default 1.) | ++---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ diff --git a/docs/source/UserGuide/Tutorials/JobManagementAdvanced/index.rst b/docs/source/UserGuide/Tutorials/JobManagementAdvanced/index.rst index 0b70d79c050..a939114b72a 100644 --- a/docs/source/UserGuide/Tutorials/JobManagementAdvanced/index.rst +++ b/docs/source/UserGuide/Tutorials/JobManagementAdvanced/index.rst @@ -345,8 +345,6 @@ using the "setNumberOfProcessors" method of the API:: Calling ``Job().setNumberOfProcessors()``, with a value bigger than 1, will translate into adding also the "MultiProcessor" tag to the job description. -.. versionadded:: v6r20p5 - Users can specify in the job descriptions NumberOfProcessors and WholeNode parameters, e.g.:: NumberOfProcessors = 16; diff --git a/docs/source/UserGuide/Tutorials/ManagingUserCredentials/index.rst b/docs/source/UserGuide/Tutorials/ManagingUserCredentials/index.rst index a6d015976dc..14c1bd8528f 100644 --- a/docs/source/UserGuide/Tutorials/ManagingUserCredentials/index.rst +++ b/docs/source/UserGuide/Tutorials/ManagingUserCredentials/index.rst @@ -86,13 +86,13 @@ Creating a user proxy The switches below will create a proxy of group "dirac_user" (if defined) and will securely upload such proxy to the DIRAC proxy store (ProxyManager), from where it could later be downloaded:: - dirac-proxy-init --group dirac_user --upload + dirac-proxy-init --group dirac_user The additional "--debug" switch (alias of "-ddd") can be used for debugging purposes, and its output would end up being similar to the following:: - $ dirac-proxy-init --group dirac_user --upload --debug + $ dirac-proxy-init --group dirac_user --debug Generating proxy... Enter Certificate password: Contacting CS... @@ -125,14 +125,11 @@ Creating a user proxy As a result of this command, several operations are accomplished: - a long user proxy (with the length of the validity of the certificate) is uploaded to the - DIRAC ProxyManager service, equivalent of the gLite MyProxy service + DIRAC ProxyManager service - a short user proxy is created with the DIRAC extension carrying the DIRAC group name and with the VOMS extension corresponding to the DIRAC group if the gLite UI environment is available. This proxy is stored in the local "/tmp/" directory, as shown. - If the gLite UI environment is not available, the VOMS extensions will not be loaded into the proxy. - This is not a serious problem, still most of the operations will be possible. - Getting the proxy information @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ @@ -164,11 +161,11 @@ Getting the proxy information $ dirac-proxy-get-uploaded-info Checking for DNs /O=GRID-FR/C=FR/O=CNRS/OU=CPPM/CN=Vanessa Hamar - -------------------------------------------------------------------------------------------------------- - | UserDN | UserGroup | ExpirationTime | PersistentFlag | - -------------------------------------------------------------------------------------------------------- - | /O=GRID-FR/C=FR/O=CNRS/OU=CPPM/CN=Vanessa Hamar | dirac_user | 2011-06-29 12:04:25 | True | - -------------------------------------------------------------------------------------------------------- + ------------------------------------------------------------------------- + | UserDN | ExpirationTime | + ------------------------------------------------------------------------- + | /O=GRID-FR/C=FR/O=CNRS/OU=CPPM/CN=Vanessa Hamar | 2011-06-29 12:04:25 | + ------------------------------------------------------------------------- - The same can be checked in the Web Portal at the following location:: diff --git a/docs/source/UserGuide/Tutorials/ShorthandCommands/index.rst b/docs/source/UserGuide/Tutorials/ShorthandCommands/index.rst new file mode 100644 index 00000000000..46fe078129a --- /dev/null +++ b/docs/source/UserGuide/Tutorials/ShorthandCommands/index.rst @@ -0,0 +1,41 @@ +.. _shorthand_commands_tutorial: + +================== +Shorthand Commands +================== + +These commands are aimed at frequent users of a DIRAC UI, especially if you experience latency issues, as this setup offers a caching mechanism. +They do require some initial setup. + + +#. Before using any DIRAC UI commands you need to set up your DIRAC UI environment and get a proxy:: + + $ source diracos/diracosrc + $ dirac-proxy-init -g [your DIRAC group] + +#. Setting up your defaults. + + You only need to do this once:: + + $ dconfig --minimal + + This will create a skeleton config file in ``~/.dirac/dcommands.conf`` + + Now edit the configuration file using your details. If you are a member of more than one VO, you might want to set up different profiles for each. Your current group will be automatically picked up from your proxy. + + .. code-block:: cfg + + [global] + default_profile = myvo_user + + [myvo_user] + group_name = myvo_user + home_dir = /myvo/user/m/mydir + default_se = MYHOME-SE-disk + + [my2vo_prod] + group_name = my2vo_prod + home_dir = /my2vo/prod/mc + default_se = ANOTHER-SE-disk + + Now you are set up to use the DIRAC short commands. For a full list of what is available, please see the :ref:`shorthand_cmd` section. diff --git a/docs/source/UserGuide/Tutorials/index.rst b/docs/source/UserGuide/Tutorials/index.rst index 2b2bfd6bc50..06c9d9a7d93 100644 --- a/docs/source/UserGuide/Tutorials/index.rst +++ b/docs/source/UserGuide/Tutorials/index.rst @@ -12,3 +12,4 @@ Tutorials FileCatalogBasic/index JobManagementAdvanced/index UsingDIRACFromPython/index + ShorthandCommands/index diff --git a/docs/source/UserGuide/WebPortalReference/ManageProxies/index.rst b/docs/source/UserGuide/WebPortalReference/ManageProxies/index.rst index 648d0e9075a..fbdc2242fe4 100644 --- a/docs/source/UserGuide/WebPortalReference/ManageProxies/index.rst +++ b/docs/source/UserGuide/WebPortalReference/ManageProxies/index.rst @@ -37,9 +37,6 @@ Columns Date until user certificate is valid. - **Persistent** - - Show if a proxy is persistent (value=true) or not (value=false). You can choose to display the proxies by group or grouping by field choosing them in the menu, activated by pressing on a menu button. diff --git a/docs/source/UserGuide/WebPortalReference/PilotMonitor/index.rst b/docs/source/UserGuide/WebPortalReference/PilotMonitor/index.rst index 35cf86f8da1..a61627e05bd 100644 --- a/docs/source/UserGuide/WebPortalReference/PilotMonitor/index.rst +++ b/docs/source/UserGuide/WebPortalReference/PilotMonitor/index.rst @@ -90,9 +90,6 @@ The following columns are provided: **Benchmark** Estimation of the power of the Worker Node CPU which is running the Pilot Job. If 0, the estimation was not possible. -**TaskQueueID** - Internal DIRAC WMS identifier of the Task Queue for which the Pilot Job is sent. - **PilotID** Internal DIRAC WMS Pilot Job identifier diff --git a/docs/source/UserGuide/commands.rst b/docs/source/UserGuide/commands.rst index eb33079f0b8..7fd287fd27d 100644 --- a/docs/source/UserGuide/commands.rst +++ b/docs/source/UserGuide/commands.rst @@ -1,8 +1,14 @@ +.. _commands_reference: + ================== Commands Reference ================== -This page is the work in progress. See more material here soon ! +| The commands are grouped by type, except for the shorthand section which contains short commands specifically developed for the interactive use of a DIRAC UI. Please see the :ref:`shorthand_commands_tutorial` for an introduction and the :ref:`shorthand_cmd` section for a complete list of available short commands. + +| Please note that you will need to get a proxy using the dirac-proxy-init command before you are able to use the any of the commands in the sections below. + +| Note that you can use DIRAC commands from all sections as needed, there is no need to stick with one set only. .. toctree:: :maxdepth: 1 @@ -10,3 +16,4 @@ This page is the work in progress. See more material here soon ! CommandReference/DataManagement/index CommandReference/WorkloadManagement/index CommandReference/Others/index + CommandReference/Shorthand/index diff --git a/docs/source/_static/horizontalAndVertical-ext.png b/docs/source/_static/horizontalAndVertical-ext.png deleted file mode 100644 index ed673bcde4e..00000000000 Binary files a/docs/source/_static/horizontalAndVertical-ext.png and /dev/null differ diff --git a/docs/source/_static/pja_application_submission.png b/docs/source/_static/pja_application_submission.png new file mode 100644 index 00000000000..488e715bb61 Binary files /dev/null and b/docs/source/_static/pja_application_submission.png differ diff --git a/docs/source/_static/pja_jobwrapper_submission.png b/docs/source/_static/pja_jobwrapper_submission.png new file mode 100644 index 00000000000..781ba9ac420 Binary files /dev/null and b/docs/source/_static/pja_jobwrapper_submission.png differ diff --git a/docs/source/_static/py3Stack.png b/docs/source/_static/py3Stack.png new file mode 100644 index 00000000000..88d59e6e58e Binary files /dev/null and b/docs/source/_static/py3Stack.png differ diff --git a/docs/source/conf.py b/docs/source/conf.py index 154b4d72864..e17191c8170 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -48,7 +48,8 @@ LOG.info("Building command reference") from diracdoctools.cmd.commandReference import run as buildCommandReference - buildCommandReference(configFile="../docs.conf") + if buildCommandReference(configFile="../docs.conf") != 0: + raise RuntimeError("Something went wrong with the documentation creation") # singlehtml build needs too much memory, so we need to create less code documentation buildType = "limited" if any("singlehtml" in arg for arg in sys.argv) else "full" @@ -103,7 +104,7 @@ "sphinx.ext.graphviz", "recommonmark", "sphinx_rtd_theme", - "sphinx_panels", + "sphinx_design", ] @@ -138,7 +139,7 @@ def setup(app): # General information about the project. project = "DIRAC" -copyright = "%s, DIRAC Project" % datetime.datetime.utcnow().year +copyright = f"{datetime.datetime.utcnow().year}, DIRAC Project" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -305,6 +306,7 @@ def setup(app): intersphinx_mapping = { "python": ("https://docs.python.org/3/", None), "matplotlib": ("https://matplotlib.org/", None), + "requests": ("https://requests.readthedocs.io/en/latest/", None), } # check for :param / :return in html, points to faulty syntax, missing empty lines, etc. diff --git a/docs/source/index.rst b/docs/source/index.rst index 4381960d099..e4fb5458641 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -34,43 +34,64 @@ An alternative description of the DIRAC system can be found in this `presentatio Documentation ============= -.. panels:: - :card: shadow + text-center - :img-top-cls: p-5 - - :img-top: _static/dirac_user.png - .. link-button:: UserGuide/index - :type: ref - :text: User Guide - :classes: btn-link stretched-link font-weight-bold - - including client installation - - --- - :img-top: _static/dirac_dev.png - .. link-button:: DeveloperGuide/index - :type: ref - :text: Developer Guide - :classes: btn-link stretched-link font-weight-bold - - adding new functionality to DIRAC - - --- - :img-top: _static/dirac_admin.png - .. link-button:: AdministratorGuide/index - :type: ref - :text: Administrator Guide - :classes: btn-link stretched-link font-weight-bold - - services administration, server installation - - --- - :img-top: _static/dirac_code.png - .. link-button:: CodeDocumentation/index - :type: ref - :text: Code Documentation - :classes: btn-link stretched-link font-weight-bold - - code reference +.. grid:: 2 + :padding: 3 + :gutter: 3 + + .. grid-item-card:: + :shadow: lg + :text-align: center + :link-type: ref + :class-body: btn-link stretched-link font-weight-bold + :class-img-top: p-5 + :img-top: _static/dirac_user.png + :link: user-guide + :link-alt: User Guide + + User Guide + + including client installation + + .. grid-item-card:: + :shadow: lg + :text-align: center + :link-type: ref + :class-body: btn-link stretched-link font-weight-bold + :class-img-top: p-5 + :img-top: _static/dirac_dev.png + :link: developer_guide + :link-alt: Developer Guide + + Developer Guide + + adding new functionality to DIRAC + + + .. grid-item-card:: + :shadow: lg + :text-align: center + :link-type: ref + :class-body: btn-link stretched-link font-weight-bold + :class-img-top: p-5 + :img-top: _static/dirac_admin.png + :link: administrator_guide + :link-alt: Administrator Guide + + Administrator Guide + + services administration, server installation + + + .. grid-item-card:: + :shadow: lg + :text-align: center + :link-type: ref + :class-body: btn-link stretched-link font-weight-bold + :class-img-top: p-5 + :img-top: _static/dirac_code.png + :link: code_documentation + :link-alt: Code Reference + + code reference :ref:`genindex` diff --git a/environment.yml b/environment.yml index c69c864971e..4f062b9f643 100644 --- a/environment.yml +++ b/environment.yml @@ -7,24 +7,20 @@ channels: dependencies: # Temporary workarounds - - astroid 2.5.6 # https://github.com/PyCQA/astroid/issues/1006 and https://github.com/PyCQA/astroid/issues/1007 # runtime - - python =3.9 + - python =3.11 - pip - apache-libcloud - boto3 - cachetools - certifi - cmreshandler >1.0.0b4 + - cwltool - db12 - - elasticsearch <7.14 - - elasticsearch-dsl - opensearch-py - - opensearch-dsl - fts3 - - future - gitpython >=2.1.0 - - m2crypto >=0.36,!=0.38.0 + - m2crypto >=0.38.0 - matplotlib - numpy - pexpect >=4.0.1 @@ -33,6 +29,7 @@ dependencies: - psutil >=4.2.0 - pyasn1 >0.4.1 - pyasn1-modules + - pydantic >=2 - python-json-logger >=0.1.8 - pytz >=2015.7 - pyyaml @@ -47,7 +44,8 @@ dependencies: - pycurl - voms - python-gfal2 - - mysqlclient + # Workaround for https://github.com/DIRACGrid/DIRAC/issues/6978 + - mysqlclient >=2.0.3,<2.1 - diraccfg - ldap3 - importlib_resources @@ -61,16 +59,16 @@ dependencies: - make - mock - parameterized - - pylint >=1.6.5 + - pylint - pyparsing >=2.0.6 - pytest >=3.6 - pytest-cov >=2.2.0 - pytest-mock + - pytest-rerunfailures - setuptools-scm - shellcheck - typer - typer-cli - - flaky # docs - pygments >=1.5 - sphinx @@ -78,7 +76,7 @@ dependencies: # RTD Sphinx theme - sphinx_rtd_theme # Bootstrap and new elements fo Sphinx - - sphinx-panels + - sphinx-design # unused - funcsigs - jinja2 @@ -90,15 +88,21 @@ dependencies: - simplejson >=3.8.1 #- tornado >=5.0.0,<6.0.0 - typing >=3.6.6 - - pyyaml - - rucio-clients + - rucio-clients >=34.4.2 + # For mypy + - mypy >=0.982 + - types-cachetools + - types-python-dateutil + - types-pytz + - types-PyYAML + - types-requests + - types-setuptools - pip: # Prerelease of the required package for integration of OAuth2 - - Authlib>=1.0.0.a2 + - Authlib >=1.0.0 - dominate - pyjwt # This is a fork of tornado with a patch to allow for configurable iostream - # It should eventually be part of DIRACGrid - git+https://github.com/DIRACGrid/tornado.git@iostreamConfigurable # This is an extension of Tornado to use M2Crypto # It should eventually be part of DIRACGrid diff --git a/integration_tests.py b/integration_tests.py index 7be977f9813..bdca32baaf0 100755 --- a/integration_tests.py +++ b/integration_tests.py @@ -1,7 +1,7 @@ #!/usr/bin/env python import fnmatch +import json import os -from pathlib import Path import re import shlex import shutil @@ -9,11 +9,12 @@ import sys import tempfile import time -from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import ThreadPoolExecutor, as_completed from contextlib import contextmanager +from itertools import chain +from pathlib import Path from typing import Optional -import click import git import typer import yaml @@ -21,20 +22,22 @@ from typer import colors as c # Editable configuration -DEFAULT_HOST_OS = "cc7" -DEFAULT_MYSQL_VER = "mysql:8.0" -DEFAULT_ES_VER = "elasticsearch:7.9.1" +DEFAULT_HOST_OS = "el9" +DEFAULT_MYSQL_VER = "mysql:8.4.4" +DEFAULT_ES_VER = "opensearchproject/opensearch:2.18.0" +DEFAULT_IAM_VER = "indigoiam/iam-login-service:v1.10.2" FEATURE_VARIABLES = { "DIRACOSVER": "master", "DIRACOS_TARBALL_PATH": None, "TEST_HTTPS": "No", + "TEST_DIRACX": "No", "DIRAC_FEWER_CFG_LOCKS": None, "DIRAC_USE_JSON_ENCODE": None, - "DIRAC_USE_JSON_DECODE": None, -} -DEFAULT_MODULES = { - "DIRAC": Path(__file__).parent.absolute(), + "INSTALLATION_BRANCH": "", + "DEBUG": "Yes", } +DIRACX_OPTIONS = () +DEFAULT_MODULES = {"DIRAC": Path(__file__).parent.absolute()} # Static configuration DB_USER = "Dirac" @@ -44,6 +47,17 @@ DB_HOST = "mysql" DB_PORT = "3306" +IAM_INIT_CLIENT_ID = "admin-client-rw" +IAM_INIT_CLIENT_SECRET = "secret" +IAM_SIMPLE_CLIENT_NAME = "simple-client" +IAM_SIMPLE_USER = "jane_doe" +IAM_SIMPLE_PASSWORD = "password" +IAM_ADMIN_CLIENT_NAME = "admin-client" +IAM_ADMIN_USER = "admin" +IAM_ADMIN_PASSWORD = "password" +IAM_HOST = "iam-login-service" +IAM_PORT = "8080" + # Implementation details LOG_LEVEL_MAP = { "ALWAYS": (c.BLACK, c.WHITE), @@ -80,8 +94,10 @@ def list_commands(self, ctx): ./integration_tests.py prepare-environment ./integration_tests.py install-server ./integration_tests.py install-client + ./integration_tests.py install-pilot ./integration_tests.py test-server ./integration_tests.py test-client + ./integration_tests.py test-pilot The test setup can be shutdown using: @@ -98,6 +114,7 @@ def list_commands(self, ctx): HOST_OS: {DEFAULT_HOST_OS!r} MYSQL_VER: {DEFAULT_MYSQL_VER!r} ES_VER: {DEFAULT_ES_VER!r} + IAM_VER: {DEFAULT_IAM_VER!r} {(os.linesep + ' ').join(['%s: %r' % x for x in FEATURE_VARIABLES.items()])} All features can be prefixed with "SERVER_" or "CLIENT_" to limit their scope. @@ -132,14 +149,17 @@ def create( flags: Optional[list[str]] = typer.Argument(None), editable: Optional[bool] = None, extra_module: Optional[list[str]] = None, + diracx_dist_dir: Optional[str] = None, release_var: Optional[str] = None, run_server_tests: bool = True, run_client_tests: bool = True, + run_pilot_tests: bool = True, ): """Start a local instance of the integration tests""" - prepare_environment(flags, editable, extra_module, release_var) + prepare_environment(flags, editable, extra_module, diracx_dist_dir, release_var) install_server() install_client() + install_pilot() exit_code = 0 if run_server_tests: try: @@ -155,6 +175,13 @@ def create( exit_code += e.exit_code else: raise NotImplementedError() + if run_pilot_tests: + try: + test_pilot() + except TestExit as e: + exit_code += e.exit_code + else: + raise NotImplementedError() if exit_code != 0: typer.secho("One or more tests failed", err=True, fg=c.RED) raise typer.Exit(exit_code) @@ -166,8 +193,8 @@ def destroy(): typer.secho("Shutting down and removing containers", err=True, fg=c.GREEN) with _gen_docker_compose(DEFAULT_MODULES) as docker_compose_fn: os.execvpe( - "docker-compose", - ["docker-compose", "-f", docker_compose_fn, "down", "--remove-orphans", "-t", "0"], + "docker", + ["docker", "compose", "-f", docker_compose_fn, "down", "--remove-orphans", "-t", "0", "--volumes"], _make_env({}), ) @@ -177,10 +204,12 @@ def prepare_environment( flags: Optional[list[str]] = typer.Argument(None), editable: Optional[bool] = None, extra_module: Optional[list[str]] = None, + diracx_dist_dir: Optional[str] = None, release_var: Optional[str] = None, ): """Prepare the local environment for installing DIRAC.""" - + if extra_module is None: + extra_module = [] _check_containers_running(is_up=False) if editable is None: editable = sys.stdout.isatty() @@ -188,36 +217,50 @@ def prepare_environment( f"No value passed for --[no-]editable, automatically detected: {editable}", fg=c.YELLOW, ) - typer.echo(f"Preparing environment") + typer.echo("Preparing environment") modules = DEFAULT_MODULES | dict(f.split("=", 1) for f in extra_module) modules = {k: Path(v).absolute() for k, v in modules.items()} - flags = dict(f.split("=", 1) for f in flags) + if not flags: + flags = {} + else: + flags = dict(f.split("=", 1) for f in flags) docker_compose_env = _make_env(flags) server_flags = {} client_flags = {} + pilot_flags = {} for key, value in flags.items(): if key.startswith("SERVER_"): server_flags[key[len("SERVER_") :]] = value elif key.startswith("CLIENT_"): client_flags[key[len("CLIENT_") :]] = value + elif key.startswith("PILOT_"): + pilot_flags[key[len("PILOT_") :]] = value else: server_flags[key] = value client_flags[key] = value + pilot_flags[key] = value server_config = _make_config(modules, server_flags, release_var, editable) client_config = _make_config(modules, client_flags, release_var, editable) + pilot_config = _make_config(modules, pilot_flags, release_var, editable) + + # The dependencies of dirac-server and dirac-client will be automatically + # started but we need to add manually all the extra services + module_configs = _load_module_configs(modules) + extra_services = list(chain(*[config["extra-services"] for config in module_configs.values()])) - typer.secho("Running docker-compose to create containers", fg=c.GREEN) - with _gen_docker_compose(modules) as docker_compose_fn: + typer.secho("Running docker compose to create containers", fg=c.GREEN) + with _gen_docker_compose(modules, diracx_dist_dir=diracx_dist_dir) as docker_compose_fn: subprocess.run( - ["docker-compose", "-f", docker_compose_fn, "up", "-d"], + ["docker", "compose", "-f", docker_compose_fn, "up", "-d", "dirac-server", "dirac-client", "dirac-pilot"] + + extra_services, check=True, env=docker_compose_env, ) - typer.secho("Creating users in server and client containers", fg=c.GREEN) - for container_name in ["server", "client"]: + typer.secho("Creating users in server client and pilot containers", fg=c.GREEN) + for container_name in ["server", "client", "pilot"]: if os.getuid() == 0: continue cmd = _build_docker_cmd(container_name, use_root=True, cwd="/") @@ -225,7 +268,7 @@ def prepare_environment( uid = str(os.getuid()) ret = subprocess.run(cmd + ["groupadd", "--gid", gid, "dirac"], check=False) if ret.returncode != 0: - typer.secho(f"Failed to add add group dirac with id={gid}", fg=c.YELLOW) + typer.secho(f"Failed to add group dirac with id={gid}", fg=c.YELLOW) subprocess.run( cmd + [ @@ -245,7 +288,8 @@ def prepare_environment( subprocess.run(cmd + ["chown", "dirac", "/home/dirac"], check=True) typer.secho("Creating MySQL user", fg=c.GREEN) - cmd = ["docker", "exec", "mysql", "mysql", f"--password={DB_ROOTPWD}", "-e"] + mysql_command = "mariadb" if "mariadb" in docker_compose_env["MYSQL_VER"].lower() else "mysql" + cmd = ["docker", "exec", "mysql", f"{mysql_command}", f"--password={DB_ROOTPWD}", "-e"] # It sometimes takes a while for MySQL to be ready so wait for a while if needed for _ in range(10): ret = subprocess.run( @@ -267,8 +311,10 @@ def prepare_environment( check=True, ) + _prepare_iam_instance() + typer.secho("Copying files to containers", fg=c.GREEN) - for name, config in [("server", server_config), ("client", client_config)]: + for name, config in [("server", server_config), ("client", client_config), ("pilot", pilot_config)]: if path := config.get("DIRACOS_TARBALL_PATH"): path = Path(path) config["DIRACOS_TARBALL_PATH"] = f"/{path.name}" @@ -298,59 +344,55 @@ def prepare_environment( ) subprocess.run(command, check=True, shell=True) + docker_compose_fn_final = Path(tempfile.mkdtemp()) / "ci" + typer.secho("Running docker compose to create DiracX containers", fg=c.GREEN) + typer.secho(f"Will leave a folder behind: {docker_compose_fn_final}", fg=c.YELLOW) + + with _gen_docker_compose(modules, diracx_dist_dir=diracx_dist_dir) as docker_compose_fn: + # We cannot use the temporary directory created in the context manager because + # we don't stay in the contect manager (Popen) + # So we need something that outlives it. + shutil.copytree(docker_compose_fn.parent, docker_compose_fn_final, dirs_exist_ok=True) + # We use Popen because we don't want to wait for this command to finish. + # It is going to start all the diracx containers, including one which waits + # for the DIRAC installation to be over. + subStdout = open(docker_compose_fn_final / "stdout", "w") + subStderr = open(docker_compose_fn_final / "stderr", "w") + + subprocess.Popen( + ["docker", "compose", "-f", docker_compose_fn_final / "docker-compose.yml", "up", "-d", "diracx"], + env=docker_compose_env, + stdin=None, + stdout=subStdout, + stderr=subStderr, + close_fds=True, + ) + @app.command() def install_server(): """Install DIRAC in the server container.""" _check_containers_running() - typer.secho("Running server installation", fg=c.GREEN) - base_cmd = _build_docker_cmd("server", tty=False) - subprocess.run( - base_cmd + ["bash", "/home/dirac/LocalRepo/TestCode/DIRAC/tests/CI/install_server.sh"], - check=True, - ) + # This runs a continuous loop that exports the config in yaml + # for the diracx container to use + # It needs to be started and running before the DIRAC server installation + # because after installing the databases, the install server script + # calls dirac-proxy-init. + # At this point we need the new CS to have been updated + # already else the token exchange fails. - typer.secho("Copying credentials and certificates", fg=c.GREEN) - base_cmd = _build_docker_cmd("client", tty=False) + typer.secho("Starting configuration export loop for diracx", fg=c.GREEN) + base_cmd = _build_docker_cmd("server", tty=False, daemon=True, use_root=True) subprocess.run( - base_cmd - + [ - "mkdir", - "-p", - "/home/dirac/ServerInstallDIR/user", - "/home/dirac/ClientInstallDIR/etc", - "/home/dirac/.globus", - ], + base_cmd + ["bash", "/home/dirac/LocalRepo/ALTERNATIVE_MODULES/DIRAC/tests/CI/exportCSLoop.sh"], check=True, ) - for path in [ - "etc/grid-security", - "user/client.pem", - "user/client.key", - f"/tmp/x509up_u{os.getuid()}", - ]: - source = os.path.join("/home/dirac/ServerInstallDIR", path) - ret = subprocess.run( - ["docker", "cp", f"server:{source}", "-"], - check=True, - text=False, - stdout=subprocess.PIPE, - ) - if path.startswith("user/"): - dest = f"client:/home/dirac/ServerInstallDIR/{os.path.dirname(path)}" - elif path.startswith("/"): - dest = f"client:{os.path.dirname(path)}" - else: - dest = f"client:/home/dirac/ClientInstallDIR/{os.path.dirname(path)}" - subprocess.run(["docker", "cp", "-", dest], check=True, text=False, input=ret.stdout) + + typer.secho("Running server installation", fg=c.GREEN) + base_cmd = _build_docker_cmd("server", tty=False) subprocess.run( - base_cmd - + [ - "bash", - "-c", - "cp /home/dirac/ServerInstallDIR/user/client.* /home/dirac/.globus/", - ], + base_cmd + ["bash", "/home/dirac/LocalRepo/TestCode/DIRAC/tests/CI/install_server.sh"], check=True, ) @@ -367,6 +409,18 @@ def install_client(): ) +@app.command() +def install_pilot(): + """Run a pilot in a container.""" + _check_containers_running() + typer.secho("Running pilot installation", fg=c.GREEN) + base_cmd = _build_docker_cmd("pilot") + subprocess.run( + base_cmd + ["bash", "/home/dirac/LocalRepo/TestCode/DIRAC/tests/CI/run_pilot.sh"], + check=True, + ) + + @app.command() def test_server(): """Run the server integration tests.""" @@ -391,6 +445,18 @@ def test_client(): raise TestExit(ret.returncode) +@app.command() +def test_pilot(): + """Run the pilot integration tests.""" + _check_containers_running() + typer.secho("Running pilot tests", err=True, fg=c.GREEN) + base_cmd = _build_docker_cmd("pilot") + ret = subprocess.run(base_cmd + ["bash", "LocalRepo/TestCode/DIRAC/tests/CI/run_tests.sh"], check=False) + color = c.GREEN if ret.returncode == 0 else c.RED + typer.secho(f"pilot tests finished with {ret.returncode}", err=True, fg=color) + raise TestExit(ret.returncode) + + @app.command() def exec_server(): """Start an interactive session in the server container.""" @@ -419,6 +485,16 @@ def exec_client(): os.execvp(cmd[0], cmd) +@app.command() +def exec_pilot(): + """Start an interactive session in the pilot container.""" + _check_containers_running() + cmd = _build_docker_cmd("pilot") + cmd += ["bash", "-c", ". $HOME/CONFIG && exec bash"] + typer.secho("Opening prompt inside pilot container", err=True, fg=c.GREEN) + os.execvp(cmd[0], cmd) + + @app.command() def exec_mysql(): """Start an interactive session in the server container.""" @@ -473,10 +549,15 @@ def logs(pattern: str = "*", lines: int = 10, follow: bool = True): if follow: base_cmd += ["-f"] with ThreadPoolExecutor(len(services)) as pool: + futures = [] for service in fnmatch.filter(services, pattern): cmd = base_cmd + [f"{runit_dir}/{service}/log/current"] p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=None, text=True) - pool.submit(_log_popen_stdout, p) + futures.append(pool.submit(_log_popen_stdout, p)) + for res in as_completed(futures): + err = res.exception() + if err: + raise err class TestExit(typer.Exit): @@ -484,17 +565,41 @@ class TestExit(typer.Exit): @contextmanager -def _gen_docker_compose(modules): - # Load the docker-compose configuration and mount the necessary volumes +def _gen_docker_compose(modules, *, diracx_dist_dir=None): + # Load the docker compose configuration and mount the necessary volumes input_fn = Path(__file__).parent / "tests/CI/docker-compose.yml" docker_compose = yaml.safe_load(input_fn.read_text()) + # diracx-wait-for-db needs the volume to be able to run the witing script + for ctn in ("dirac-server", "dirac-client", "dirac-pilot", "diracx-wait-for-db"): + if "volumes" not in docker_compose["services"][ctn]: + docker_compose["services"][ctn]["volumes"] = [] volumes = [f"{path}:/home/dirac/LocalRepo/ALTERNATIVE_MODULES/{name}" for name, path in modules.items()] volumes += [f"{path}:/home/dirac/LocalRepo/TestCode/{name}" for name, path in modules.items()] - docker_compose["services"]["dirac-server"]["volumes"] = volumes[:] - docker_compose["services"]["dirac-client"]["volumes"] = volumes[:] + docker_compose["services"]["dirac-server"]["volumes"].extend(volumes[:]) + docker_compose["services"]["dirac-client"]["volumes"].extend(volumes[:]) + docker_compose["services"]["dirac-pilot"]["volumes"].extend(volumes[:]) + docker_compose["services"]["diracx-wait-for-db"]["volumes"].extend(volumes[:]) + + module_configs = _load_module_configs(modules) + if diracx_dist_dir is not None: + for container_name in [ + "dirac-client", + "dirac-pilot", + "dirac-server", + "diracx-init-cs", + "diracx-wait-for-db", + "diracx-init-db", + "diracx", + ]: + docker_compose["services"][container_name].setdefault("volumes", []).append( + f"{diracx_dist_dir}:/diracx_sources" + ) + docker_compose["services"][container_name].setdefault("environment", []).append( + "DIRACX_CUSTOM_SOURCE_PREFIXES=/diracx_sources" + ) # Add any extension services - for module_name, module_configs in _load_module_configs(modules).items(): + for module_name, module_configs in module_configs.items(): for service_name, service_config in module_configs["extra-services"].items(): typer.secho(f"Adding service {service_name} for {module_name}", err=True, fg=c.GREEN) docker_compose["services"][service_name] = service_config.copy() @@ -514,16 +619,17 @@ def _gen_docker_compose(modules): def _check_containers_running(*, is_up=True): with _gen_docker_compose(DEFAULT_MODULES) as docker_compose_fn: running_containers = subprocess.run( - ["docker-compose", "-f", docker_compose_fn, "ps", "-q", "-a"], + ["docker", "compose", "-f", docker_compose_fn, "ps", "-q", "-a"], stdout=subprocess.PIPE, env=_make_env({}), - check=True, + # docker compose ps has a non-zero exit code when no containers are running + check=False, text=True, ).stdout.split("\n") if is_up: if not any(running_containers): typer.secho( - f"No running containers found, environment must be prepared first!", + "No running containers found, environment must be prepared first!", err=True, fg=c.RED, ) @@ -531,21 +637,21 @@ def _check_containers_running(*, is_up=True): else: if any(running_containers): typer.secho( - f"Running instance already found, it must be destroyed first!", + "Running instance already found, it must be destroyed first!", err=True, fg=c.RED, ) raise typer.Exit(code=1) -def _find_dirac_release_and_branch(): +def _find_dirac_release(): # Start by looking for the GitHub/GitLab environment variables - ref = os.environ.get("CI_COMMIT_REF_NAME", os.environ.get("GITHUB_REF")) - if ref == "refs/heads/integration": - return "integration", "" - ref = os.environ.get("CI_MERGE_REQUEST_TARGET_BRANCH_NAME", os.environ.get("GITHUB_BASE_REF")) - if ref == "integration": - return "integration", "" + if "GITHUB_BASE_REF" in os.environ: # this will be "rel-v8r0" + return os.environ["GITHUB_BASE_REF"] + if "CI_COMMIT_REF_NAME" in os.environ: + return os.environ["CI_COMMIT_REF_NAME"] + if "CI_MERGE_REQUEST_TARGET_BRANCH_NAME" in os.environ: + return os.environ["CI_MERGE_REQUEST_TARGET_BRANCH_NAME"] repo = git.Repo(os.getcwd()) # Try to make sure the upstream remote is up to date @@ -578,9 +684,9 @@ def _find_dirac_release_and_branch(): err=True, fg=c.YELLOW, ) - return "integration", "" + return "integration" else: - return "", f"v{version.major}r{version.minor}" + return version_branch def _make_env(flags): @@ -590,7 +696,15 @@ def _make_env(flags): env["HOST_OS"] = flags.pop("HOST_OS", DEFAULT_HOST_OS) env["CI_REGISTRY_IMAGE"] = flags.pop("CI_REGISTRY_IMAGE", "diracgrid") env["MYSQL_VER"] = flags.pop("MYSQL_VER", DEFAULT_MYSQL_VER) + if "mariadb" in env["MYSQL_VER"].lower(): + env["MYSQL_ADMIN_COMMAND"] = "mariadb-admin" + else: + env["MYSQL_ADMIN_COMMAND"] = "mysqladmin" env["ES_VER"] = flags.pop("ES_VER", DEFAULT_ES_VER) + env["IAM_VER"] = flags.pop("IAM_VER", DEFAULT_IAM_VER) + if "CVMFS_DIR" not in env or not Path(env["CVMFS_DIR"]).is_dir(): + typer.secho(f"CVMFS_DIR environment value: {env.get('CVMFS_DIR', 'NOT SET')}", fg=c.YELLOW) + env["CVMFS_DIR"] = "/tmp" return env @@ -611,9 +725,387 @@ def _dict_to_shell(variables): return "\n".join(lines) +def _prepare_iam_instance(): + """Prepare the IAM instance such as we have: + + * 2 clients: + * exchange-token-test: able to exchange token + * simple-token-test: for users + * 2 users: + * admin and jane doe (a user) + * 3 groups: + * dirac/admin + * dirac/prod + * dirac/user + """ + issuer = f"http://iam-login-service:{IAM_PORT}" + + typer.secho("Getting an IAM admin token", fg=c.GREEN) + + # It sometimes takes a while for IAM to be ready so wait for a while if needed + for _ in range(10): + try: + tokens = _get_iam_token(issuer, IAM_INIT_CLIENT_ID, IAM_INIT_CLIENT_SECRET) + break + except typer.Exit: + typer.secho("Failed to connect to IAM, will retry in 10 seconds", fg=c.YELLOW) + time.sleep(10) + else: + raise RuntimeError("All attempts to _get_iam_token failed") + + initial_admin_access_token = tokens.get("access_token") + + # Update the configuration of the initial IAM client adding the necessary scopes + _update_init_iam_client( + issuer, + initial_admin_access_token, + IAM_INIT_CLIENT_ID, + "Admin client (read-write)", + " ".join(["scim:read", "scim:write", "iam:admin.read", "iam:admin.write"]), + ["client_credentials"], + ) + # Fetch a new token with the updated client + tokens = _get_iam_token(issuer, IAM_INIT_CLIENT_ID, IAM_INIT_CLIENT_SECRET) + admin_access_token = tokens.get("access_token") + + typer.secho("Creating IAM clients", fg=c.GREEN) + _create_iam_client( + issuer, + admin_access_token, + IAM_SIMPLE_CLIENT_NAME, + grant_types=["password", "client_credentials"], + ) + _create_iam_client( + issuer, + admin_access_token, + IAM_ADMIN_CLIENT_NAME, + grant_types=["password", "urn:ietf:params:oauth:grant-type:token-exchange"], + ) + + typer.secho("Creating IAM users", fg=c.GREEN) + simple_user_config = _create_iam_user(issuer, admin_access_token, IAM_SIMPLE_USER, IAM_SIMPLE_PASSWORD) + + typer.secho("Creating IAM groups", fg=c.GREEN) + dirac_group_config = _create_iam_group(issuer, admin_access_token, "dirac") + dirac_group_id = dirac_group_config["id"] + _create_iam_subgroup(issuer, admin_access_token, "dirac", dirac_group_id, "admin") + dirac_prod_group_config = _create_iam_subgroup(issuer, admin_access_token, "dirac", dirac_group_id, "prod") + dirac_user_group_config = _create_iam_subgroup(issuer, admin_access_token, "dirac", dirac_group_id, "user") + + typer.secho("Adding IAM users to groups", fg=c.GREEN) + _create_iam_group_membership( + issuer, + admin_access_token, + simple_user_config["userName"], + simple_user_config["id"], + [dirac_group_id, dirac_prod_group_config["id"], dirac_user_group_config["id"]], + ) + + +def _iam_curl( + url: str, *, data: list[str] = [], verb: Optional[str] = None, user: Optional[str] = None, headers: list[str] = [] +) -> subprocess.CompletedProcess: + cmd = ["docker", "exec", "server", "curl", "-L", "-s"] + if verb: + cmd += ["-X", verb] + if user: + cmd += ["-u", user] + for arg in data: + cmd += ["-d", arg] + for header in headers: + cmd += ["-H", header] + cmd += [url] + + return subprocess.run(cmd, capture_output=True, check=False) + + +def _get_iam_token(issuer: str, client_id: str, client_secret: str) -> dict: + """Get a token using the password flow + + :param str issuer: url of the issuer + :param str user: username + :param str password: password + :param str client_id: client id + :param str client_secret: client secret + """ + # We use subprocess instead of requests to interact with IAM + # Otherwise, if executed from a docker container in a different network namespace, it would not work + url = os.path.join(issuer, "token") + ret = _iam_curl( + url, + user=f"{client_id}:{client_secret}", + data=["grant_type=client_credentials"], + ) + + if not ret.returncode == 0: + typer.secho(f"Failed to get an admin token: {ret.returncode} {ret.stdout} {ret.stderr}", err=True, fg=c.RED) + raise typer.Exit(code=1) + + return json.loads(ret.stdout) + + +def _update_init_iam_client( + issuer: str, admin_access_token: str, client_id: str, client_name: str, scope: str, grant_types: list[str] +) -> dict: + """Update the configuration of the initial IAM client adding the necessary scopes + + :param str issuer: url of the issuer + :param str admin_access_token: access token to register a client + :param str client_id: id of the client + """ + # Get the configuration of the client + url = os.path.join(issuer, "iam/api/clients", client_id) + ret = _iam_curl( + url, + headers=[f"Authorization: Bearer {admin_access_token}", "Content-Type: application/json"], + ) + + if not ret.returncode == 0: + typer.secho( + f"Failed to get config for client {client_id}: {ret.returncode} {ret.stdout} {ret.stderr}", + err=True, + fg=c.RED, + ) + raise typer.Exit(code=1) + + # Update the configuration with the provided values + client_config = json.loads(ret.stdout) + client_config["client_name"] = client_name + client_config["scope"] = scope + client_config["grant_types"] = grant_types + client_config["redirect_uris"] = [] + client_config["code_challenge_method"] = "S256" + + # Update the client + url = os.path.join(issuer, "iam/api/clients", client_id) + ret = _iam_curl( + url, + verb="PUT", + data=[json.dumps(client_config)], + headers=[f"Authorization: Bearer {admin_access_token}", "Content-Type: application/json"], + ) + + if not ret.returncode == 0: + typer.secho( + f"Failed to update config for client {client_id}: {ret.returncode} {ret.stdout} {ret.stderr}", + err=True, + fg=c.RED, + ) + raise typer.Exit(code=1) + + return json.loads(ret.stdout) + + +def _create_iam_client( + issuer: str, admin_access_token: str, client_name: str, scope: str = "", grant_types: list[str] = [] +) -> dict: + """Generate an IAM client + + :param str issuer: url of the issuer + :param str admin_access_token: access token to register a client + :param str client_name: name of the client + :param str scope: scope of the client + :param list grant_types: list of grant types + """ + scope = "openid profile offline_access " + scope + + default_grant_types = ["refresh_token"] + + # Some grant types are privileged and cannot be added at the creation of the client, but can be added later + privileged_grant_types = ["password", "urn:ietf:params:oauth:grant-type:token-exchange", "client_credentials"] + requested_privileged_grant_types = [] + for grant_type in privileged_grant_types: + if grant_type in grant_types: + grant_types.remove(grant_type) + requested_privileged_grant_types.append(grant_type) + + grant_types = list(set(default_grant_types + grant_types)) + + client_config = { + "client_name": client_name, + "token_endpoint_auth_method": "client_secret_basic", + "scope": scope, + "grant_types": grant_types, + "response_types": ["code"], + } + + url = os.path.join(issuer, "iam/api/client-registration") + ret = _iam_curl( + url, + verb="POST", + headers=[f"Authorization: Bearer {admin_access_token}", "Content-Type: application/json"], + data=[json.dumps(client_config)], + ) + + if not ret.returncode == 0: + typer.secho(f"Failed to create client {client_name}: {ret.returncode} {ret.stderr}", err=True, fg=c.RED) + raise typer.Exit(code=1) + + client_config = json.loads(ret.stdout) + client_id = client_config["client_id"] + ret = _update_init_iam_client( + issuer, + admin_access_token, + client_id, + client_name, + scope, + list(set(requested_privileged_grant_types + grant_types)), + ) + print(ret) + return ret + + +def _create_iam_user(issuer: str, admin_access_token: str, username: str, password: str) -> dict: + """Generate an IAM user + + :param str issuer: url of the issuer + :param str admin_access_token: access token to register a client + :param str given_name: name of user + :param str family_name: family name of the user + """ + given_name, family_name = username.split("_") + given_name = given_name.capitalize() + family_name = family_name.capitalize() + user_config = { + "active": True, + "userName": username, + "password": password, + "name": { + "givenName": given_name, + "familyName": family_name, + "formatted": f"{given_name} {family_name}", + }, + "emails": [ + { + "type": "work", + "value": f"{given_name}.{family_name}@donotexist.email", + "primary": True, + } + ], + } + + url = os.path.join(issuer, "scim/Users") + ret = _iam_curl( + url, + verb="POST", + headers=[f"Authorization: Bearer {admin_access_token}", "Content-Type: application/scim+json"], + data=[json.dumps(user_config)], + ) + + if not ret.returncode == 0: + typer.secho( + f"Failed to create user {given_name} {family_name}: {ret.returncode} {ret.stderr}", + err=True, + fg=c.RED, + ) + raise typer.Exit(code=1) + return json.loads(ret.stdout) + + +def _create_iam_group(issuer: str, admin_access_token: str, group_name: str) -> dict: + """Generate an IAM group + + :param str issuer: url of the issuer + :param str admin_access_token: access token to register a client + :param str group_name: name of the group + """ + group_config = {"schemas": ["urn:ietf:params:scim:schemas:core:2.0:Group"], "displayName": group_name} + + url = os.path.join(issuer, "scim/Groups") + ret = _iam_curl( + url, + verb="POST", + headers=[f"Authorization: Bearer {admin_access_token}", "Content-Type: application/scim+json"], + data=[json.dumps(group_config)], + ) + + if not ret.returncode == 0: + typer.secho(f"Failed to create group {group_name}: {ret.returncode} {ret.stderr}", err=True, fg=c.RED) + raise typer.Exit(code=1) + return json.loads(ret.stdout) + + +def _create_iam_subgroup( + issuer: str, admin_access_token: str, group_name: str, group_id: str, subgroup_name: str +) -> dict: + """Generate an IAM subgroup + + :param str issuer: url of the issuer + :param str admin_access_token: access token to register a client + :param str group_name: name of the group + :param str group_id: id of the group + :param str subgroup_name: name of the subgroup + """ + subgroup_config = { + "schemas": ["urn:ietf:params:scim:schemas:core:2.0:Group", "urn:indigo-dc:scim:schemas:IndigoGroup"], + "urn:indigo-dc:scim:schemas:IndigoGroup": { + "parentGroup": { + "display": group_name, + "value": group_id, + r"\$ref": os.path.join(issuer, "scim/Groups", group_id), + }, + }, + "displayName": subgroup_name, + } + + url = os.path.join(issuer, "scim/Groups") + ret = _iam_curl( + url, + verb="POST", + headers=[f"Authorization: Bearer {admin_access_token}", "Content-Type: application/scim+json"], + data=[json.dumps(subgroup_config)], + ) + + if not ret.returncode == 0: + typer.secho( + f"Failed to create subgroup {group_name}/{subgroup_name}: {ret.returncode} {ret.stderr}", + err=True, + fg=c.RED, + ) + raise typer.Exit(code=1) + return json.loads(ret.stdout) + + +def _create_iam_group_membership( + issuer: str, admin_access_token: str, username: str, user_id: str, group_ids: list[str] +): + """Bind a given user to some groups/subgroups + + :param str issuer: url of the issuer + :param str admin_access_token: access token to register a client + :param str username: username + :param str user_id:: id of the user + :param list group_ids: list of group/subgroup ids + """ + membership_config = { + "schemas": ["urn:ietf:params:scim:api:messages:2.0:PatchOp"], + "operations": [ + { + "op": "add", + "path": "members", + "value": [ + {"display": username, "value": user_id, r"\$ref": os.path.join(issuer, "scim/Users", user_id)} + ], + } + ], + } + + for group_id in group_ids: + url = os.path.join(issuer, "scim/Groups", group_id) + ret = _iam_curl( + url, + verb="PATCH", + headers=[f"Authorization: Bearer {admin_access_token}", "Content-Type: application/scim+json"], + data=[json.dumps(membership_config)], + ) + + if not ret.returncode == 0: + typer.secho(f"Failed to add {username} to {group_id}: {ret.returncode} {ret.stderr}", err=True, fg=c.RED) + raise typer.Exit(code=1) + + def _make_config(modules, flags, release_var, editable): config = { - "DEBUG": "True", # MYSQL Settings "DB_USER": DB_USER, "DB_PASSWORD": DB_PASSWORD, @@ -621,21 +1113,37 @@ def _make_config(modules, flags, release_var, editable): "DB_ROOTPWD": DB_ROOTPWD, "DB_HOST": DB_HOST, "DB_PORT": DB_PORT, - # ElasticSearch settings - "NoSQLDB_HOST": "elasticsearch", + # OpenSearch settings + "NoSQLDB_USER": "elastic", + "NoSQLDB_PASSWORD": "changeme", + "NoSQLDB_HOST": "opensearch", "NoSQLDB_PORT": "9200", + # IAM initial settings + "IAM_INIT_CLIENT_ID": IAM_INIT_CLIENT_ID, + "IAM_INIT_CLIENT_SECRET": IAM_INIT_CLIENT_SECRET, + "IAM_SIMPLE_CLIENT_NAME": IAM_SIMPLE_CLIENT_NAME, + "IAM_SIMPLE_USER": IAM_SIMPLE_USER, + "IAM_SIMPLE_PASSWORD": IAM_SIMPLE_PASSWORD, + "IAM_ADMIN_CLIENT_NAME": IAM_ADMIN_CLIENT_NAME, + "IAM_ADMIN_USER": IAM_ADMIN_USER, + "IAM_ADMIN_PASSWORD": IAM_ADMIN_PASSWORD, + "IAM_HOST": IAM_HOST, + "IAM_PORT": IAM_PORT, # Hostnames "SERVER_HOST": "server", "CLIENT_HOST": "client", + "PILOT_HOST": "pilot", # Test specific variables "WORKSPACE": "/home/dirac", + # DiracX variable + "DIRACX_URL": "http://diracx:8000", } if editable: config["PIP_INSTALL_EXTRA_ARGS"] = "-e" required_feature_flags = [] - for module_name, module_ci_config in _load_module_configs(modules).items(): + for _, module_ci_config in _load_module_configs(modules).items(): config |= module_ci_config["config"] required_feature_flags += module_ci_config.get("required-feature-flags", []) config["DIRAC_CI_SETUP_SCRIPT"] = "/home/dirac/LocalRepo/TestCode/" + config["DIRAC_CI_SETUP_SCRIPT"] @@ -644,7 +1152,7 @@ def _make_config(modules, flags, release_var, editable): if release_var: config |= dict([release_var.split("=", 1)]) else: - config["DIRAC_RELEASE"], config["DIRACBRANCH"] = _find_dirac_release_and_branch() + config["DIRAC_RELEASE"] = _find_dirac_release() for key, default_value in FEATURE_VARIABLES.items(): config[key] = flags.pop(key, default_value) @@ -654,6 +1162,12 @@ def _make_config(modules, flags, release_var, editable): except KeyError: typer.secho(f"Required feature variable {key!r} is missing", err=True, fg=c.RED) raise typer.Exit(code=1) + + # If we test DiracX, enable all the options + if config["TEST_DIRACX"].lower() in ("yes", "true"): + for key in DIRACX_OPTIONS: + config[key] = "Yes" + config["TESTREPO"] = [f"/home/dirac/LocalRepo/TestCode/{name}" for name in modules] config["ALTERNATIVE_MODULES"] = [f"/home/dirac/LocalRepo/ALTERNATIVE_MODULES/{name}" for name in modules] @@ -675,7 +1189,7 @@ def _load_module_configs(modules): return module_ci_configs -def _build_docker_cmd(container_name, *, use_root=False, cwd="/home/dirac", tty=True): +def _build_docker_cmd(container_name, *, use_root=False, cwd="/home/dirac", tty=True, daemon=False): if use_root or os.getuid() == 0: user = "root" else: @@ -690,6 +1204,8 @@ def _build_docker_cmd(container_name, *, use_root=False, cwd="/home/dirac", tty= err=True, fg=c.YELLOW, ) + if daemon: + cmd += ["-d"] cmd += [ "-e=TERM=xterm-color", "-e=INSTALLROOT=/home/dirac", diff --git a/pyproject.toml b/pyproject.toml index 695038f7fd6..113be701cff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,17 +10,92 @@ git_describe_command = "git describe --dirty --tags --long --match *[0-9].[0-9]* [tool.black] line-length = 120 -target-version = ['py39'] +target-version = ['py311'] -[tool.pylint.typecheck] -# List of decorators that change the signature of a decorated function. -signature-mutators = [] +[tool.isort] +profile = "black" [tool.mypy] allow_redefinition = true strict = true check_untyped_defs = true ignore_missing_imports = true +no_incremental=true +implicit_reexport=true +files = [ + 'src/DIRAC/Core/Utilities/ReturnValues.py', + 'src/DIRAC/Core/Security/Properties.py' +] exclude = [ '/tests/' ] + + +[tool.ruff] +select = ["E", "F", "B", "I", "PLE"] +ignore = ["B905", "B008", "B006"] +line-length = 120 + +[tool.pylint.basic] + +# We mostly have CamelCase, with a few differences. +# In tests we have quite some snake_case, mostly due to pytest +# We can instruct pylint to understand both, but the problem is that it +# will stick to one style per file (i.e if the first variable is snake, +# all the following must be snake) +# It's not quite the case yet... +# For the time being, I wrote the regex that matches best our code. +# (except for the services with their export_ and types_ ...) +# We will see about tests later... +# See https://pylint.readthedocs.io/en/latest/user_guide/messages/convention/invalid-name.html#multiple-naming-styles-for-custom-regular-expressions + +# Camel case with capital letter first +class-rgx = '([A-Z][a-z]*)+$' +module-rgx = '([A-Z][a-z]*)+$' + +# Attributes, variables, functions and methods +# are camelCase, but can start with one or two understcore +attr-rgx = '(?:_*[a-z]+([A-Z][a-z]*)*)$' +variable-rgx = '(?:_*[a-z]+([A-Z][a-z]*)*)$' +function-rgx = '(?:_*[a-z]+([A-Z][a-z]*)*)$' +method-rgx = '(?:_*[a-z]+([A-Z][a-z]*)*)$' + +argument-naming-style = "camelCase" + +[tool.pylint.main] +# Files or directories to be skipped. They should be base names, not paths. +ignore = [".svn", ".git", "integration_tests.py"] + +# List of module names for which member attributes should not be checked (useful +# for modules/projects where namespaces are manipulated during runtime and thus +# existing member attributes cannot be deduced by static analysis). It supports +# qualified module names, as well as Unix pattern matching. +ignored-modules = ["MySQLdb", "numpy"] + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use, and will cap the count on Windows to +# avoid hangs. +jobs = 0 + +# This allows pylint to not display the following false error : +# No name 'BaseModel' in module 'pydantic' (no-name-in-module) +# Related issue: https://github.com/pydantic/pydantic/issues/1961 +extension-pkg-whitelist = ["pydantic"] + +[tool.pylint.typecheck] +# List of decorators that change the signature of a decorated function. +signature-mutators = [] + +[tool.pylint.format] + +# Maximum number of characters on a single line. +max-line-length = 120 + +[tool.pylint."messages control"] +disable = ["R0903","I0011","c-extension-no-member"] + +[tool.pylint.reports] +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +output-format = "colorized" diff --git a/pytest.ini b/pytest.ini index af7b5e5d111..dc222a005e1 100644 --- a/pytest.ini +++ b/pytest.ini @@ -6,3 +6,5 @@ python_files=Test_*.py assert*.py # in order to make sure that M2Crypto and pyGSI do not step # on each other's feet addopts = --no-cov -rx -v --color=yes --showlocals --tb=long --ignore=tests +markers = + slow: marks tests as slow (deselect with '-m "not slow"') diff --git a/release.notes b/release.notes index c9829330815..de702ffefbc 100644 --- a/release.notes +++ b/release.notes @@ -1,3 +1,1802 @@ +[v9.0.0a53] + +*WorkloadManagement + +NEW: (#8159) cgroup2 limit support + +*FrameworkSystem + +CHANGE: (#8157) Improve performance of TheImpersonator +FIX: (#8154) Caching the proxy strength to avoid a DB call +NEW: (#8144) add a randomized connection pooling for diracx + +*WorkloadManagementSystem + +FIX: (#8156) we can kill a list of pilots instead of going one by one + +*Core + +NEW: (#8155) Add caches to asn1_utils for better performance + +*RequestManagementSystem + +FIX: (#8152) Printing DiracX ForwardDISET requests + +[v9.0.0a52] + +*RequestManagementSystem + +FIX: (#8150) RequestValidator sets correct Owner for v8 requests + +[v9.0.0a51] + +*FrameworkSystem + +FIX: (#8149) DiracX tokens should not be included in the proxies used to interact with CEs +FIX: (#8146) TypeError in TheImpersonator + +*TransformationSystem + +FIX: (#8147) bad escape in the updateTransformationParameter + +*WorkloadManagement + +FIX: (#8145) Running dirac-admin-update-pilot + +[v9.0.0a50] + +*Core + +FIX: (#8140) Make it possible to debug dirac-jobexec failures +NEW: (#8139) add a source parameter to the impersonator +NEW: (#8138) DiracX token from PEM is always stored in $TMP/dx_..... + +[v9.0.0a49] + +*Core + +FIX: (#8136) Proxy Pilots are sent with a token + +*WorkloadManagementSystem + +FIX: (#8135) the PoolXMLSlice should be created in the same directory where the job runs + +[v9.0.0a48] + +*WorkloadManagement + +FIX: (#8134) replace DIRACJOBID with JOBID in JobWrapper environment +FIX: (#8126) Write DiracX token in JobAgent +FIX: (#8126) Remove platform validation as extensions can redefine it +FIX: (#8125) add VO information to the pilotDict when killing pilots +FIX: (#8124) report the message of the Exception instead of the Exception itself in JobAgent.submitJob + +*Core + +CHANGE: (#8131) DIRAC.initialize(): ensure host credentials are not ignored in case passed as a list and not a tuple + +*ConfigurationSystem + +FIX: (#8127) dirac-admin-update-pilot can work without a specified VO + +*Resources + +FIX: (#8124) do not try to use a malformed StorageElement instance in SingularityCE + +[v9.0.0a47] + +*WorkloadManagement + +FIX: (#8123) PilotManager not using tokens to kill pilots +NEW: (#8119) JobStateUpdate legacy adapter + +*RequestManagementSystem + +CHANGE: (#8121) No longer directly use of the JobStateUpdateClient when processing requests +FIX: (#8121) Calling setJobParameter when processing requests + +*TransformationSystem + +CHANGE: (#8121) No longer directly use of the JobStateUpdateClient from the DataRecoveryAgent + +*DBs + +FIX: (#8120) Escape password when giving it to sqlalchemy + +*WorkloadManagementSystem + +FIX: (#8116) avoid repeating optimization when job goes to STAGING + +*Resources + +FIX: (#8115) Fix AREX CE pilot logs in alwaysIncludeProxy case + +[v9.0.0a46] + +*Subsystem + +NEW: (#8109) added setInputData to JobState + +*ConfigurationSystem + +FIX: (#8108) VOMS2CSAgent newDiracName might not be defined yet + +[v9.0.0a45] + +*WorkloadManagemnt + +FIX: (#8106) Clear any non-UTF encodable environment variables in pilots + +*ConfigurationSystem + +NEW: (#8104) backport the export of the Sub in DiracX + +*Core + +FIX: (#8102) Don't use string processing on X509 name objects +NEW: (#8099) -S option on dirac-configure is now ignored +FIX: (#8088) force M2Crypto to use the proxy instead of the host certificate if provided + +*WorkloadManagement + +FIX: (#8100) Move JobWrapperUtilities after import Script.parseCommandLine() + +*Resources + +FIX: (#8086) Catch ConnectionError when calling send on a MQ + +[v9.0.0a44] + +*Resources + +FIX: (#8084) htcondor x509 unsupported version +CHANGE: (#8075) Disable Bearer token for HTTPs unless upload/TPC +CHANGE: (#8074) conditionally reset the rlimit for xroot +NEW: (#8069) findFileByMetadata method for Rucio + +*Core + +FIX: (#8080) dirac-apptainer-exec should work also in the case of no proxy + +*WorkloadManagement + +FIX: (#8073) pass args to buildQueueDict() in the right order + +*Workload Management + +FIX: (#8067) Supress non-UTF8 variables from pilot environment + +*ConfigurationSystem + +CHANGE: (#8061) TTLCache for getProxyPrrovidersForDN + +*Test + +NEW: (#8005) use containerized certs creation for integration tests + +[v9.0.0a43] + +*Documentation + +CHANGE: (#8055) extend pilot documentation. + +*docs + +FIX: (#8054) update and correct the README to build the DIRAC documenation + +*Subsystem + +CHANGE: (#8044) default MySQL version from 8.0 to 8.4 + +[v9.0.0a42] + +*Core + +NEW: (#8041) new command dirac-apptainer-exec for running DIRAC commands inside apptainer +FIX: (#8034) Correct user mapping for DiracX from IAM + +*WorkloadManagement + +FIX: (#8040) get ElasticJobParametersDB index prefix from the configuration +CHANGE: (#8014) Use apptainer for SingularityComputingElement +CHANGE: (#8014) Drop support for SingularityComputingElement without user namespaces +CHANGE: (#8014) Enchance debugging output if SingularityComputingElement fails +CHANGE: (#8014) Drop support for using apptainer from outside of DIRACOS2 + +*ConfigurationSystem + +FIX: (#8040) evaluate useCRT flag as boolean in Utilities.getElasticDBParameters() + +[v9.0.0a41] + +*TransformationSystem + +FIX: (#8031) make the setting of inputDataBulk extendable +FIX: (#8022) make 2 methods of WorkflowTaskAgent extendable + +*Core + +FIX: (#8029) read at most 2^14 bytes at the same time +FIX: (#8002) one less flag for EnableSecurityLogging +FIX: (#7988) Add locks to AuthManager caches +CHANGE: (#7971) Optimise ASN1 decoding in X509Certificate +FIX: (#7969) Avoid locking in MessageQueueHandler + +*Resources + +FIX: (#8027) explicitly disconnect Stomp before reconnecting +FIX: (#8012) adapt AREX to ARC7 delegation output +FIX: (#7995) adapt HTCondorCE to latest htcondor version + +*WorkloadManagementSystem + +FIX: (#8020) StalledJobAgent: if StartExecTime is not set, use the last recorded heartbeat +FIX: (#7986) PilotBundle: compatibility with py2 and py3 + +*DataManagementSystem + +FIX: (#8003) exit with status 1 in case of error +FIX: (#7989) Apply a workaround for https://github.com/xrootd/xrootd/issues/2396 + +*CI + +FIX: (#7999) Add PilotAgentsDB definition to `tests/CI/docker-compose.yml` + +FIX: (#7997) pensearch configuration should consume the ca_certs parameter if it is there + +*ResourceStatusSystem + +CHANGE: (#7987) dirac-rss-sync: flip the default status to Active +FIX: (#7983) Docs: removed Setup from Operations +NEW: (#7972) Add a timeout for GocDB to avoid blocking the CacheFeederAgent + +*MonitoringSystem + +CHANGE: (#7978) removed ElasticSearch in favor or OpenSearch + +*RequestManagementSystem + +NEW: (#7975) Allow to disable DM transfer as an FTS failover + +*FrameworkSystem + +FIX: (#7970) Support https URLs with dirac-framework-ping-service + +*Integration tests + +FIX: (#7968) mount diracx in init-db container + +[v9.0.0a40] + +*Core + +CHANGE: (#7961) Introduce caches to AuthManager.getUsername + +[v9.0.0a39] + +*WorkloadManagement + +FIX: (#7954) correctly log the pilot job reference during the matching process + +[v9.0.0a38] + +*TransformationSystem + +FIX: (#7953) use updatemany in TransformationDB + +*WorkloadManagement + +FIX: (#7949) renew delegation prior to submitting pilots +FIX: (#7941) integrity check failure in RemoteRunner +FIX: (#7935) Run CE cleanup step at correct point + +*ConfigurationSystem + +FIX: (#7947) Make writing the CS atomic + +*RequestManagementSystem + +FIX: (#7934) call the correct executeRPCMethod depending on diset/diracx + +[v9.0.0a37] + +*TransformationSystem + +FIX: (#7927) Use parameterised query in addTransformation +FIX: (#7910) Use UTC to calculate older in export_getTasksToSubmit + +*WorkloadManagment + +CHANGE: (#7922) Better caching performance in the Matching Limiter + +*Resources + +CHANGE: (#7918) Add option to include proxy on AREX token submission + +*Integration tests + +CHANGE: (#7915) Upgrade Indigo IAM (1.10.2) + +[v9.0.0a36] + +*TransformationSystem + +CHANGE: (#7906) the TransformationCleaningAgent forces jobs to KILLED + +*WorkloadManagementSystem + +NEW: (#7905) for parametric jobs, added the possibility to bulk insert records in JobLoggingDB +CHANGE: (#7902) added ulimit -n 1048575 to pilotwrapper content + +*FrameworkSystem + +CHANGE: (#7901) Rename TokenManager service classes to follow standard convention for tornado and diset handlers + +Thank you for writing the text to appear in the release notes. It will show up +exactly as it appears between the two bold lines +Please follow the template: + +*CI + +NEW: (#7899) Add `PilotLogsDB` to `diracx` environment to allow CI testing. +For examples look into release.notes + +*Misc + +FIX: (#7893) Tidy up whitespace in cvmfs.yml + +*WorkloadManagement + +CHANGE: (#7892) Use plain proxy for the pilot bundle + +[v9.0.0a35] +[v9.0.0a34] + +*DataManagementSystem + +FIX: (#7879) Fix the condition for return type sanitation + +[v9.0.0a33] + +*Core + +CHANGE: (#7876) Remove lock in Logging._createLogRecord +FIX: (#7864) Prevent `-1` returned from `oSocket.write` being interpreted as number of bytes to avoid an infinite loop in `DISET BaseTransport.sendData`. +FIX: (#7856) ensure processProposal always returns "closeTransport" on error +FIX: (#7843) Returning DISET responses with >100000000 bytes + +*ResourceStatusSystem + +FIX: (#7871) some protocols or types are not in GOC + +*TransformationSystem + +FIX: (#7865) getTasksToSubmit consider tasks inserted by 30 seconds or more +NEW: (#7843) Use getTransformationFilesAsJsonString for faster getTransformationFiles + +*StompMQConnector + +FIX: (#7855) add a timeout for the StompConnector to handle nonresponsive sockets which can cause logging to be blocked + +*DataManagementSystem + +FIX: (#7850) Fix deadlock when FTS3Agent._treatOperation fails + +[v9.0.0a32] + +*Resources + +FIX: (#7839) hide private keys from the logs +FIX: (#7832) AREX interactions with tokens +FIX: (#7831) Drop CloudCE proxy handling +FIX: (#7829) CloudCE: truncate long messages returned from create_node + +*Core + +CHANGE: (#7837) minimum version of some packages + +*TransformationSystem + +NEW: (#7833) +CHANGE: (#7833) Improve getTransformationFiles performance + +[v9.0.0a31] + +*WorkloadManagement + +FIX: (#7827) SandboxStore not able to assign a sandbox to a job +FIX: (#7825) JobWrapper checks existence of executable in jobIDPath +FIX: (#7821) Getting pilot reference from job parameters + +*Resources + +FIX: (#7819) AREXCE returns an error if a queue is not found in the ARC instance configuration + +[v9.0.0a30] + +*DataManagementSystem + +NEW: (#7817) experimental token support for FTS transfers +FIX: (#7781) dirac_dms_find_lfns: Check if requested path exists and returns error if not. Prevents users hitting #7487 +FIX: (#7780) Return correct error for non-existing directory. Partial fix for #7487 +NEW: (#7764) added checksum comparison in the fc/se consistency tool +NEW: (#7756) FileCatalogHandler: add function export_getFileDetails to get the (user) metadata for a list of LFNs +FIX: (#7719) Remove `def findDirIDsByMetadata(self, metaDict, dPath, credDict):` method from `MultiVODirectoryMetadata` (derived) class which caused an extra VO suffix added when searching. The method is meant to be used _internally_ only on keys which are already expanded in a MultiVO case. Add a user-level def `findDirectoriesByMetadata(self, queryDict, path, credDict)` to the derived class thus adding a VO suffix for a directory search. Fixes #7687. +CHANGE: (#7694) dirac-dms-replicate-and-register-request: Make request chunk size configurable; default behaviour unchanged. + +*FrameworkSystem + +NEW: (#7817) allow to bypass cache when retrieving tokens + +FIX: (#7813) SSHCE, Try python3 before unversioned python +Fix a typo in the key Addler -> Adler of the return dict of putAndRegister + +*TransformationSystem + +FIX: (#7806) RequestTaskAgent only considers requests in final states, and consider files in intermediate state as problematic (https://github.com/DIRACGrid/DIRAC/issues/7116) +NEW: (#7806) RequestTaskAgent uses getBulkRequestStatus instead of getRequestStatus +RMS: (#7806) +NEW: (#7806) implement getRequestStatus +NEW: (#7697) InputDataAgent: new Option MultiVO, which makes the FileCatalog Query use the author of the DN, rather than the Host, to resolve MultiVO metadata correctly. Fixes #7681 + +*Resources + +FIX: (#7803) added a 30s gfal2 timeout for downloading the SRR +FIX: (#7790) SSHComputingElement fix: added check of result +FIX: (#7726) Update CloudCE cloudinit.template for EL8+ +CHANGE: (#7715) HTCondorCE: UseSSLSubmission: use the generated proxy file for everything, no longer need to have certificate of user present on the server. +FIX: (#7713) AREX submission issue not properly handled +NEW: (#7695) better error message when no matching protocol between 2 SE +CHANGE: (#7689) remove BOINCCE + +*ResourceStatusSystem + +FIX: (#7801) use always a from address (from Operations ResourceStatus/Config/FromAddress ) when sending email notifications, to avoid "spoofing" domains restrictions +NEW: (#7783) add a DIRAC to GOCDB service type conversion +CHANGE: (#7774) do not consider Endpoint for StorageOccupancy +FIX: (#7765) Do not use tinezone aware datetime +FIX: (#7755) delete the Occupancy cache only for older entries + +*WorkloadManagement + +FIX: (#7797) Allow jobs to be KILLED from more states +FIX: (#7797) KILLED is a final job state +FIX: (#7716) PilotWrapper - check for the presence of the -l pilot option + +*ConfigurationSystem + +CHANGE: (#7796) VOMS2CSAgent: if a nickname is set, this nickname will always be used and no new accounts are going to be created if a DN changes or a user is in multiple VOs +NEW: (#7796) VOMS2CSAgent: New option "ForceNickname", if this option is enabled no dirac user is created if no nickname attribute is set for a user +CHANGE: (#7796) IAMService: use logger and return errors for users so that the VOMS2CSAgent can notify admins about issues +NEW: (#7742) Configuration-system-shell: added reload and sort commands + +*WorkloadManagementSystem + +CHANGE: (#7792) SiteDirector will always bundle the proxy +CHANGE: (#7762) removed JobDB's SiteMask and Logging +FIX: (#7751) Proper killing of jobs when not matched, running or stalled +FIX: (#7707) JobDB: update LastUpdateTime when the job is matched +NEW: (#7699) added possibility to specify UserEnvVariable (pilot option) at CE level + +*Workflow + +FIX: (#7786) Avoid incorrect error strings in Workflow execute + +*Core + +NEW: (#7775) Support aggregating by date in MySQL.getCounters +FIX: (#7750) add more safeguard when processing result queue in ProcessPool +CHANGE: (#7720) register the DictCache destructor as an atexit handler +NEW: (#7700) fetch nickname attribute from IAM +CHANGE: (#7696) drop DIRAC_MYSQL_CONNECTION_GRACE_TIME and stalled connections reuse + +*Subsystem + +CHANGE: (#7733) For PilotWrapper tests, use the artifacts (created in Pilot repo) + +*Diracx + +NEW: (#7711) populate diracx section from Iam + +*Doc + +FIX: (#7688) Move the explanation how to enable tokens to a place that is shown in ReadTheDocs in the end. + +*Interfaces + +FIX: (#7684) Document BadJobParameterError + +[v9.0.0a29] + +*WorkloadManagementSystem + +FIX: (#7649) added log headers to InputDataResolution modules +CHANGE: (#7629) add jobGroup to job parameters +FIX: (#7584) ServerUtils: prevent getPilotAgentsDB from returning None +FIX: (#7576) Fix potential circular import in WorkflowReader. Mostly seen in the creation of the documentation. +FIX: (#7787) added a 30s gfal2 timeout for downloading the SRR + +*TransformationSystem + +FIX: (#7741) RequestTaskAgent only considers requests in final states, and consider files in intermediate state as problematic (https://github.com/DIRACGrid/DIRAC/issues/7116) +NEW: (#7741) RequestTaskAgent uses getBulkRequestStatus instead of getRequestStatus +RMS: (#7741) +NEW: (#7741) implement getRequestStatus + +[v8.0.52] + +*ResourceStatusSystem + +FIX: (#7800) use always a from address (from Operations ResourceStatus/Config/FromAddress ) when sending email notifications, to avoid "spoofing" domains restrictions + +*WorkloadManagement + +NEW: (#7643) Support Pydantic 2 +FIX: (#7621) remove random shuffle in PilotWrapper +CHANGE: (#7609) Perform bulk lookup of job parameters from elasticsearch +CHANGE: (#7608) Make RemoteRunner more resilient to CE issues +FIX: (#7594) JobMonitoring.getJobParameters should pass jobID as an int to ElasticJobParametersDB +FIX: (#7590) AREX "out" and "err" need to exist before file integrity check + +*Accounting + +FIX: (#7640) AccountingDB only generate condition if needed + +*Resources + +NEW: (#7638) HTCondorCE: Added UseSSLSubmission option. Allows one to use a configured DN at given Sites for job submission, instead of proxies or tokens. Only at participating CEs and conditions apply. + +*Core + +FIX: (#7634) Avoid printing out clear text password in SQLAlchemy +FIX: (#7591) File.secureOpenForWrite: fix exception when opening in binary mode, fixes #7581 + +*DataManagementSystem + +NEW: (#7633) Add a protocol parameter to the getReplicas method family +NEW: (#7619) prepare for FTS 3.13 release with breaking API +NEW: (#7617) DataManager.putAndRegister rejects too long filename + +*Deployment + +FIX: (#7628) fix the path of the CVMFS sync_packages.sh script + +*All + +FIX: (#7616) fix pylint 3.2.0 warnings + +*MonitoringSystem + +FIX: (#7584) ServerUtils: prevent getMonitoringDB from returning None + +[v9.0.0a28] + +*WorkloadManagementSystem + +FIX: (#7574) serverUtils: jit imports +FIX: (#7534) Pilots submitted by SiteDirector won't add the pythonVersion flag +FIX: (#7521) Fix memory reporting +FIX: (#7510) SandboxStore: add VO if needed + +*WorkloadManagement + +FIX: (#7571) support file:/... as a location for the pilot files +FIX: (#7564) make sure CVMFS_locations is a list +CHANGE: (#7553) Remove files from the RemoteRunner execution +FIX: (#7552) JobCleaningAgent: fix exception in deleteJobsByStatus caused by mismatching job ID types +NEW: (#7529) introduce JobWrapperOfflineTemplate for uses in systems without external connectivity +CHANGE: (#7460) introduce JobWrapper.preprocess, process and postprocess + +*test + +NEW: (#7570) added pilot workflow tests to integration_tests + +*Core + +FIX: (#7569) Support M2Crypto 0.40.0+ +CHANGE: (#7566) Replace the default PFN type ROOT_All with ROOT +FIX: (#7524) Depend on packaging + +*environment.yml + +NEW: (#7555) add cwltool + +*Resources + +FIX: (#7545) TimeLeft utility was unable to get values from the cfg +FIX: (#7532) support the case where HTCondor kills the jobs + +*Test + +FIX: (#7540) Redirect the output of popen in a file to fix #7473 +FIX: (#7539) extra_module default to empty list + +*FrameworkSystem + +CHANGE: (#7511) ProxyDB: removed tables ProxyDB_Proxies and ProxyDB_Tokens + +[v9.0.0a27] + +*Core + +FIX: (#7505) plotting TypeLoader works with editable installation +NEW: (#7453) Introduce an RPC stub equivalent for DiracX + +*Test + +CHANGE: (#7502) use pytest-rerun instead of flaky + +*WorkloadManagementSystem + +CHANGE: (#7498) removed GridEnv +FIX: (#7497) If the SoftwareDistModule is set in the Operations Section, add it to the Job JDL to restore previous behaviour +NEW: (#7453) FutureJobStateUpdate.setJobStatusBulk return a DiracX RPC stub + +*WorkloadManagement + +FIX: (#7493) check the VO from the task queues before submitting pilots +FIX: (#7488) JobAgent.setupProxy takes owner instead of ownerDN + +*RequestManagementSystem + +CHANGE: (#7453) adapt ForwardDISET to DiracX stub + +*FrameworkSystem + +CHANGE: (#7442) removed NotificationDN + +[v9.0.0a26] + +*FrameworkSystem + +FIX: (#7491) dirac-proxy-info without the dirac group + +*WorkloadManagement + +FIX: (#7490) missing result in return statement + +*tests + +NEW: (#7484) add the state key in the test environment to fix diracx execution + +[v9.0.0a25] + +*Accounting + +FIX: (#7486) Fix errors during insert into ac_in_* tables. + +*Core + +FIX: (#7483) Fix DISET calls with proxy to be used passed as an argument + +*RequestManagementSystem + +FIX: (#7482) RequestTask - download no-VOMS proxy if the owner group does not define VOMSRole + +[v9.0.0a24] + +*WorkloadManagement + +FIX: (#7480) JobCleaningAgent - select random jobs for deletion rather than head and tail jobs. +FIX: (#7475) JobID type in PushJobAgent + +*Interfaces + +CHANGE: (#7472) dfind - more explicit failure report + +*DataManagementSystem + +NEW: (#7471) add tools for consistency checks + +*tests + +FIX: (#7470) check running containers in integration tests script + +*WorkloadManagementSystem + +CHANGE: (#7464) PilotAgentsDB: removed OutputReady and Broker fields + +[v9.0.0a23] + +*WorkloadManagement + +FIX: (#7458) jobID type issue in JobAgent + +*ConfigurationSystem + +FIX: (#7454) getQueue() overriding the CE tags + +*FrameworkSystem + +FIX: (#7451) dirac-proxy-init printInfo without the dirac group + +[v9.0.0a22] + +*WorkloadManagement + +FIX: (#7448) get pilot logging info with a token from an AREXCE + +*WorkloadManagementSystem + +FIX: (#7447) Each job has its own JobReport in JobAgent +FIX: (#7447) JobAgent exits when all the jobs have been processed +FIX: (#7446) StatesAccountingAgent: skip the first iteration in order to avoid double commit after a restart +CHANGE: (#7439) SandboxStore: remove external SE feature +FIX: (#7436) JobDB: fix mismatch of string and integer jobIDs. In some cases API calls would fail because JObDB.getJobParameters and JobDB.getJobsAttributes return dictionaries with integer keys, while the function was called with a string jobID. This fixes for example the StalledJobAgent being unable to reschedule matched jobs. +NEW: (#7425) TornadoPilotLoggingHandler modify the handler to accept VO name sent by a pilot. Required in a case where the VO cannot be guessed from a proxy. This change requires https://github.com/DIRACGrid/Pilot/pull/230 +NEW: (#7421) SandboxDB: add VO field +CHANGE: (#7414) move the content of SubmissionPolicy in SiteDirector + +*FrameworkSystem + +FIX: (#7443) correctly set the duration of tokens in cache + +*RequestManagementSystem + +FIX: (#7441) make sure OwnerDN is defined before trying to access its value + +*MonitoringSystem + +CHANGE: (#7432) moved to weekly indices for agent and service monitoring + +[v9.0.0a21] + +*Test + +CHANGE: (#7417) write the diracx CsSync config in the CS instead of taking the yaml file from diracx repo + +*FrameworkSystem + +CHANGE: (#7413) removed Proxies persistency flag + +*Core + +FIX: (#7412) Adding VOMS extensions without having environment variables set +NEW: (#7412) Add DIRAC_DISABLE_GCONFIG_REFRESH environment variable to prevent gConfig being accidentally used +FIX: (#7409) Use proxy lifetime for tokens from legacy proxy exchange (https://github.com/DIRACGrid/diracx/issues/130) + +*WorkloadManagement + +FIX: (#7409) Add DiracX to payload proxies used by compute elements (#7402) +FIX: (#7406) SiteDirector should not interact with CEs if there is 0 pilot to submit + +*WorkloadManagementSystem + +CHANGE: (#7407) JobDB simplifications +CHANGE: (#7405) Removed Private Pilot functionality + +[v9.0.0a20] + +*WorkloadManagement + +FIX: (#7399) JobAgent rescheduling wrong jobs +FIX: (#7387) JobAgent interaction with JobMonitoringClient + +*WorkloadManagementSystem + +CHANGE: (#7396) PilotAgentsDB: move from OwnerGroup to VO +FIX: (#7379) The callback for the Stager was failing, because of a type mismatch in the jobID used to retrieve the status. Jobs never came out of Staging. +NEW: (#7375) pilotWrapper: using CVMFS_locations for discovering the pilot files + +*FrameworkSystem + +FIX: (#7391) send notifications for expiring proxies + +Documentation on how to deploy a third party tool (fluent-bit) to grab, format and send Dirac current logs to ElasticSearch and/or splitted logs files + +*Resources + +FIX: (#7376) AREXCE should break when a valid delegation ID is found + +*Core + +FIX: (#7374) Converting p12 files with filenames containing special characters + +[v9.0.0a19] +[v9.0.0a18] +[v9.0.0a17] +[v9.0.0a16] +[v9.0.0a14] +[v9.0.0a14] +[v9.0.0a15] +[v9.0.0a14] +[v9.0.0a13] +[v9.0.0a12] +[v9.0.0a11] +[v9.0.0a10] +[v9.0.0a9] +[v9.0.0a8] + + +[v9.0.0a7] + +*Resources + +FIX: (#7366) make sure the WLCG accounting file is json + +[v9.0.0a6] + +*WorkloadManagementSystem + +FIX: (#7364) added SiteDirector option for CVMFS_locations + +[v9.0.0a5] + +*FrameworkSystem + +FIX: (#7360) wrong service uptime calculation + +*Resources + +FIX: (#7357) get batch system details from local cfg instead of environment variables +FIX: (#7349) remove valid argument from setToken + +*docs + +NEW: (#7356) added doc for PreInstalledPilotEnv + +*Core + +NEW: (#7344) Introduce DIRAC_MYSQL_CONNECTION_GRACE_TIME to specify the grace time of the MySQL connection pool + +*TransformationSystem + +NEW: (#7337) add an index on Status and Type for the TransformationDB.Transformations table + +*WorkloadManagementSystem + +FIX: (#7332) basic JobMonitoring client fixes +FIX: (#7331) JobAgents will set jobStatus=Failed/Payload failed if and only if the job was previously Running + +*CORE + +FIX: (#7332) allow for proxyLocation to work with host certificate + +[v9.0.0a4] + +*docs + +FIX: (#7330) just use `pip install DIRAC` for DIRAC client install +FIX: (#7330) updated picture for py3 stack + +*Interfaces + +FIX: (#7328) Improve performance of job delete/kill/reschedule API +FIX: (#7322) Lower log level for printing version in submitJob + +*WorkloadManagementSystem + +FIX: (#7327) Allow "ANY" in the valid sites list +NEW: (#7318) initial FutureClient for JobStateUpdate +FIX: (#7316) ElasticJobParametersDB: do not configure the IndexPrefix name + +[v9.0.0a3] + +*WorkloadManagement + +FIX: (#7305) Use copy+remove in PilotSync agent to avoid SELinux problems +FIX: (#7288) WMSUtilities supports VO-specific token tags + +*WorkloadManagementSystem + +FIX: (#7301) correctly handle UTC in JobLoggingDB float timestamp + +*Core + +FIX: (#7297) convert exception object to string representation in ElasticSearchDB + +*DataManagementSystem + +CHANGE: (#7297) default proxy lifetime for FTS is 36h and force the redelegation when 12h are left + +*FrameworkSystem + +FIX: (#7284) disabledVOs not available when doing dirac-configure + +*DiracX + +NEW: (#7261) DiracX credentials are now included in proxy PEM files created by DIRAC +NEW: (#7261) DiracX is now mandatory 🎉 + +[v9.0.0a2] + +*WorkloadManagementSystem + +FIX: (#7282) JobStatus: allow WAITING -> KILLED +FIX: (#7263) SiteDirector acts for a VO + +*FrameworkSystem + +FIX: (#7280) show status SysAdminCLI KeyError + +*Resources + +FIX: (#7279) getting more details about failed/aborted pilots from HTCondor + +*Core + +FIX: (#7273) createClient: also look at HandlerMixin classes to find `export_`ed functions. Fixes the client Documentation creation, fixes #7265 + +*WorkloadManagement + +NEW: (#7271) SiteDirector - added preinstalledEnv and preinstalledEnvPrefix pilot options + +[v9.0.0a1] + +*Core + +NEW: (#7258) Support using DiracX as a backend for RPC calls + +*WorkloadManagementSystem + +CHANGE: (#7257) Refactor PilotLoggingAgent: download proxies at initialisation. Renew a proxy if it is about to expire. + +*Resources + +NEW: (#7256) Add support for loading external providers in CloudCE +NEW: (#7256) Add OpenNebula6 (XMLRPC) provider for CloudCE/libcloud + +*Subsystem + +FIX: (#7253) prevent AREXCE from creationg delegations each time a CE got a small and temporary issue + +*docs + +NEW: (#7252) add documentation about setting up DIRAC to submit pilots with tokens + +*WorkloadManagement + +CHANGE: (#7250) pilot submission with tokens in a multi-VO context + +[v8.1.0a23] + +*TransformationSystem + +NEW: (#7239) TransformationCleaningAgent: Use the RequestManagementSystem to asynchronously remove files, rather than serially in the agent run + +[v8.1.0a22] + +*WorkloadManagementSystem + +FIX: (#7238) Output sandbox are assigned when using DiracX +CHANGE: (#7237) Removed interaction with MJF +FIX: (#7226) SiteDirector adds wnVO Pilot options, if needed +CHANGE: (#7215) add VO info in the table Jobs of JobDB + +*WorkloadManagement + +FIX: (#7234) AREX does not return an error if delegation not found +FIX: (#7230) Group to vo conversion in TaskQueueDB + +*ConfigurationSystem + +CHANGE: (#7225) explicitly state the VO in each default group upon installation + +*DiracX + +CHANGE: (#7222) Get a access/refresh token pair when generating a proxy +NEW: (#7222) Support using diracx as backend for the SandboxStoreHandler + +[v8.1.0a21] + +*Core + +FIX: (#7216) replace ColorBar.draw_all() with figure.draw_without_rendering() + +*WorkloadManagement + +FIX: (#7213) emove OwnerDN from SiteDirector & PilotStatus + +*WorkloadManagementSystem + +CHANGE: (#7190) add VO info in TaskQueueDB +FIX: (#7187) code simplification for SandboxStoreClient + +[v8.1.0a20] + +*Test + +FIX: (#7206) manually start the extra services when running the integration test +NEW: (#7143) allow integration tests to run against DiracX + +*TransformationSystem + +CHANGE: (#7204) TransformationCleaningAgent removes files by chunk +FIX: (#7200) Drop TS permission check for all read-only functions + +*WorkloadManagement + +FIX: (#7201) JobMonitoringClient uses the new DiracClient class +CHANGE: (#7194) Job.setNumberOfProcessors raises an exception if the function arguments aren't valid. + +Please follow the template: + +*WorkloadManagementSystem + +FIX: (#7196) do not try to insert nan values in AccountingDB +NEW: (#7143) call the diracx Job Monitoring endpoint +NEW: (#7105) Enhance dirac-admin-get-pilot-output to download remote pilot logs. Currently only from a SE. + +*FrameworkSystem + +FIX: (#7192) replace create_access_token with create_token + +*ConfigurationSystem + +CHANGE: (#7143) do not look for the shifter under the Default or section + +*Core + +NEW: (#7143) get a token for a proxy + +[v8.1.0a19] + +*ConfigurationSystem + +CHANGE: (#7182) VOMS2CS agent only sends a mail in case of change in the CS + +*WorkloadManagementSystem + +FIX: (#7180) passing pilotDN instead of owner in getPilotProxyFromVOMSGroup() + +[v8.1.0a18] + +*Production + +CHANGE: (#7162) Move from DN to Username + +*WorkloadManagementSystem + +CHANGE: (#7157) PilotAgentsDB: move DN in favor of username + +[v8.1.0a17] + +*TransformationSystem + +NEW: (#7170) Implement a finer grained permissions model for Transformations. + +*DataManagementSystem + +FIX: (#7168) better error reporting when an existing LFN has no replicas (at least it doesn't crash) + +*WorkloadManagement + +CHANGE: (#7167) Stop using PID namespaces with SingularityCE + +*Resources + +FIX: (#7160) default providerType if not found in IdProviderFactory + +*WorkloadManagementSystem + +FIX: (#7155) Adding Owner as JDL requirement +CHANGE: (#7151) factorize some utilities of JobSubmit into functions + +[v8.1.0a16] + +*FrameworkSystem + +FIX: (#7149) Do not overwrite the tornado port if already set +FIX: (#7117) replaced deprecated importlib read_text function to fix failure in CI tests + +*Resources + +FIX: (#7147) MultiVO proxy renewal in AREX +FIX: (#7146) Propagate environment variables from host to container. +FIX: (#7134) Don't print client_secret in oauth2 debug +FEAT: (#7120) allow to pass credentials when downloading the SRR file +CHANGE: (#7103) ARC and ARC6 are now deprecated +CHANGE/FIX: (#7102) proxy renewal logic in AREX +FIX: (#7102) submitting tokens with AREX +FIX: (#7102) correctly report aborted pilots with AREX + +*TransformationSystem + +FIX: (#7144) dirac_production_runjoblocal: updated list of Pilot modules +CHANGE: (#7124) use userName instead of DN + +*Core + +FIX: (#7142) look also for DIRAC.rootPath/etc/dirac.cfg before printing a warning of not found cfg + +*docs + +change: Replace the use of sphinx-panels with sphinx-design, this unblocks the use of latest sphinx versions (panels was requiring sphinx < 5) +FIX: (#7133) formatting item list + +*WorkloadManagement + +NEW: (#7132) Introducing 'Scouting' status in job state transitions + +*ResourceStatusSystem + +FIX: (#7120) Add the Error status in the scripts + +*WorkloadManagementSystem + +FIX: (#7101) JobManagerHandler: get the OwnerDN only when needed. + +[v8.1.0a15] + +*Resources + +FIX: (#7100) concatenate tags instead of overriding them in ComputingElement.initializeParameters() +NEW: (#7079) audienceName attribute in CEs +NEW: (#7065) add a test for the PoolCE to highlight the fact that submission failures cannot be handled by the caller with the return value + +*WorkloadManagementSystem + +FIX: (#7098) Exceptions in StalledJobAgent +CHANGE: (#7097) modifying access specifiers of JobPolicy from private to protected +FIX: (#7081) disable activityMonitoring for JobAgent +NEW: (#7079) audience support in SiteDirector +CHANGE: (#6973) validating the JDL job format using a pydantic model + +*FrameworkSystem + +FIX: (#7094) Support Authlib v1.2.1 + +*FrameworkSyste + +FIX: (#7079) fix TokenManager(Client/Handler/Utilities) + +*WorkloadManagent + +NEW: (#7068) New TornadoSandboxStoreHandler implementation + +*Core + +NEW: (#7068) TornadoClient - added sendFile() method + +FIX: (#7067) Close DISET selector when we no longer need it + +*WorkloadManagement + +FIX: (#7065) management of the job status in JobAgent +NEW: (#7065) add a test for the JobAgent to make sure the status of the submissions are correctly handled + +*docs + +NEW: (#7065) documentation about ComputingElement + +*tests + +NEW: (#7024) added script for comdirac hackathon certification tests + +[v8.1.0a14] + +*WorkloadManagementSystem + +NEW: (#7059) new script dirac-admin-add-pilot + +*dashboard + +NEW: (#7039) add missing dashboards of issue #6742 also to the LHCb grafana organisation. The dashboards can be checked/reviewed at https://monit-grafana.cern.ch/dashboards/f/qwdBd3KVk/dev-dirac-certification + +Please follow the template: + +*docs + +FIX: (#7023) added remaining info from COMDIRAC Wiki. Addresses part of #6746 + +[v8.1.0a13] + +*Core + +FIX: (#7058) print a warning message when no CFG file is found +FIX: (#6976) retry upon DNS lookup error +FIX: (#6955) HandlerManager - create separate class for each service handler +FIX: (#6955) BaseRequestHandler - fullComponentName can be only defined in the configuration + +*docs + +FIX: (#7057) added a note about removal of elastic job parameters +CHANGE: (#6969) more details in the "supercomputers" section + +*Resources + +CHANGE: (#7055) use double slash in Echo xroot URLs +FIX: (#7042) reduce memory consumption of AREXCE.getJobOutput() +CHANGE: (#7036) remove CREAM CE +FIX: (#7028) SSHCE getPilotOutput remove stamp +FIX: (#7000) ARCLogLevel +FIX: (#6999) remove executables definition from AREXCE.submitJob() +NEW: (#6969) cleanJob() in AREX +CHANGE: (#6969) executables from inputs are passed as executables in the XRSL in ARC and AREX + +*test + +FIX: (#7053) Workflow tests need pilot.cfg to work + +*WorkloadManagementSystem + +FIX: (#7052) Implementation of DefaultLocalCEType option +FIX: (#7046) fix(StalledJobAgent): after a job was Killed, we have to force the Failed. +CHANGE: (#7033) TokenManagerHandler provides client access token if username is not provided. +CHANGE: (#7031) Using JobManager instead of JobDB for rescheduling stalled jobs +CHANGE: (#7019) removed old format of ElasticJobParametersDB +NEW: (#7017) add the possibility of adding generic options for running the pilots +FIX: (#6980) JobStatus: add staging to checking transition, needed by StorageManagementSystem (Stager) +NEW: (#6969) tests for RemoteRunner +CHANGE: (#6969) RemoteRunner can deal with MP applications, clean job outputs and process more applications +FIX: (#6969) Modify the DIRACVersion in the PushJobAgent + +*WorkloadManagement + +NEW: (#7042) md5 checksum comparison in RemoteRunner +NEW: (#7041) Add /Resources/Computing/DefaultLocalCEType option +FIX: (#7007) return specific exit code in RemoteRunner in case of failure +FIX: (#7006) Getting the value of /LocalSite/RemoteExecution + +*RequestManagementSystem + +FIX: (#7013) fix the monitoring of Operations not yet entered into the DB + +*FrameworkSystem + +CHANGE: (#6967) enable/disableLogsFromExternalLibs only used from LocalConfiguration +FIX: (#6967) remove a deprecated method +CHANGE: (#6967) gLogger date format +NEW: (#6967) type hinting in gLogger + +*Interfaces + +FIX: (#6961) Removed dinit, sessions are now instantiated automatically. Also removed DCOMMANDS_PPID. + +PR for adding Dirac side code related to https://github.com/DIRACGrid/WebAppDIRAC/pull/729/commits +CHANGE: (#6956) Add getTokensByUserID method and make it compliant with ProxyManager code + +*Integration tests + +FIX: (#6954) Make error nicer when the IAM docker instance is not working properly + +*dashboards + +NEW: (#6911) added json export of Grafana demo dashboard + +[v8.1.0a12] + +*WorkloadManagementSystem + +FIX: (#6953) Watchdog fix: always report correct types +FIX: (#6931) make singletons of VirtualMachineDB and PilotAgentsDB + +PR for fixing dead link in doc (monitoring part) +FIX: (#6949) change urls for WMS and Logs dashboards + +*tests + +CHANGE: (#6948) Integration tests: upped Opensearch version to 2.1.0 +NEW: (#6937) integration tests for IdProviders + +*Core + +CHANGE: (#6947) remove support for MySQL 5.7 +FIX: (#6944) protection for opensearch_dsl being merged in opensearchpy +FIX: (#6924) cast method names to string before sending monitoring + +*Resources + +NEW: (#6937) unit tests for IdProviders +FIX: (#6927) HTCondorCEComputingElement - set _CONDOR_AUTH_SSL_CLIENT_CADIR before executing condor commands + +*Tornado + +FIX: (#6933) do not use activityMonitor in TornadoREST + +*FrameworkSystem + +FIX: (#6922) stdoutJson logger prints all the necessary fields +FIX: (#6922) MicrosecondJsonFormatter does not dump DIRAC specific fields (`timeStampIsShown`, `colorIsShown`, etc) + +[v8.1.0a11] + +*Resources + +FIX: (#6913) better handle exceptions in AREXCE +CHANGE: (#6902) (AREX)ComputingElement - use proxy only if it is set up +FIX: (#6883) add workaround for https timeout when dowloading file + +*WorkloadManagementSystem + +FIX: (#6913) config helper error message +FIX: (#6907) SiteDirector: failure in submitting Accounting or Monitoring report only prints an error + +*MonitoringSystem + +FIX: (#6904) while updating ES documents, wait and retry on ConflictError +CHANGE: (#6875) introduce gMonitoringDB global instance + +*Interfaces + +FIX: (#6903) make -F (file) option work for drm command. This fixes #6896 + +*WorkloadManagement + +NEW: (#6902) Enable PilotManager to manage pilots with tokens + +*Core + +FIX: (#6901) Tornado PeriodicCallbacks are actually called +FEAT: (#6901) Tornado Handlers report to ActivityMonitor +FEAT: (#6901) Activity Monitoring reports useful information +FIX: (#6874) TornadoREST - fix the error in resolution of exposed methods + +Thank you for writing the text to appear in the release notes. It will show up +exactly as it appears between the two bold lines +Please follow the template: + +*Production System + +CHANGE: (#6898) change column types in ProductionDB from VARCHAR(255) to LONGTEXT + +*FrameworkSystem + +CHANGE: (#6897) Default log level is INFO +NEW: (#6897) Define global configuration options for log level + +*ConfigurationSystem + +FIX: (#6888) Fix the connection to ES database when using certificates + +*DataManagementSystem + +CHANGE: (#6884) refactor (DFC): rewrite the file/replica deletion stored procedure as they were in 5.6 +CHANGE: (#6880) FTS3DB uses `in` instead of `not in` for some query +CHANGE: (#6880) FTS3DB removal of operations uses python datetime instead of mysql builtin + +[v8.1.0a11] + +*Resources + +FIX: (#6913) better handle exceptions in AREXCE +CHANGE: (#6902) (AREX)ComputingElement - use proxy only if it is set up +FIX: (#6883) add workaround for https timeout when dowloading file + +*WorkloadManagementSystem + +FIX: (#6913) config helper error message +FIX: (#6907) SiteDirector: failure in submitting Accounting or Monitoring report only prints an error + +*MonitoringSystem + +FIX: (#6904) while updating ES documents, wait and retry on ConflictError +CHANGE: (#6875) introduce gMonitoringDB global instance + +*Interfaces + +FIX: (#6903) make -F (file) option work for drm command. This fixes #6896 + +*WorkloadManagement + +NEW: (#6902) Enable PilotManager to manage pilots with tokens + +*Core + +FIX: (#6901) Tornado PeriodicCallbacks are actually called +FEAT: (#6901) Tornado Handlers report to ActivityMonitor +FEAT: (#6901) Activity Monitoring reports useful information +FIX: (#6874) TornadoREST - fix the error in resolution of exposed methods + +Thank you for writing the text to appear in the release notes. It will show up +exactly as it appears between the two bold lines +Please follow the template: + +*Production System + +CHANGE: (#6898) change column types in ProductionDB from VARCHAR(255) to LONGTEXT + +*FrameworkSystem + +CHANGE: (#6897) Default log level is INFO +NEW: (#6897) Define global configuration options for log level + +*ConfigurationSystem + +FIX: (#6888) Fix the connection to ES database when using certificates + +*DataManagementSystem + +CHANGE: (#6884) refactor (DFC): rewrite the file/replica deletion stored procedure as they were in 5.6 +CHANGE: (#6880) FTS3DB uses `in` instead of `not in` for some query +CHANGE: (#6880) FTS3DB removal of operations uses python datetime instead of mysql builtin + +[v8.1.0a10] + +*Tests + +CHANGE: (#6861) factorize FTS3 tests + +[v8.1.0a9] + +*ConfigurationSystem + +FIX: (#6868) VOMS2CSAgent: strip whitespaces from DN entried and nickname +NEW: (#6865) Resources helper - getQueues() - can select queues by Tag values also + +*FrameworkSystem + +FIX: (#6865) TokenManager - use refreshToken flow to generate access tokens from the stored refresh tokens + +*WorkloadManagement + +NEW: (#6865) SiteDirector enabled to select queues by tags +NEW: (#6865) SiteDirector sets up tokens for ComputingElements configured with the Token tag + +*Resources + +NEW: (#6865) HTCondorComputingElement, ARC(6)ComputingElement and AREXComputingElement enabled to for job operations with tokens +FIX: (#6842) Complete AREX renewal + +*DataManagementSystem + +NEW: (#6863) ReplicateAndRegister FTS mode also checks the SE status +NEW: (#6863) add a delay in monitoring in the FTS3Agent +NEW: (#6860) dirac-dms-filecatalog-cli - use the default VO FileCatalog if not explicitly specified +CHANGE: (#6835) Speed up ReplicateAndRegister operation + +*dashboards/PilotSubmissions + +FIX: (#6858) efficiency plots now use heatmaps +FIX: (#6858) index patterns are now environment agnostic + +*dashboards/ServiceMonitoring + +FIX: (#6858) index patterns are now environment agnostic + +*dashboards/AgentMonitoring + +FIX: (#6858) index patterns are now environment agnostic + +*dashboards/PilotsHistory + +FIX: (#6858) style of plots +FIX: (#6858) index patterns are now environment agnostic + +*Interfaces + +NEW: (#6853) Add setPriority to Job API +FIX: (#6849) removed print statements and replaced with gLogger as appropriate as requested in #6746 + +*RequestManagementSystem + +CHANGE: (#6848) ReqProxy does not attempt a direct forward to central ReqManager + +[v8.1.0a8] + +*Resources + +FIX: (#6832) Update AREX delegation renewal +FIX: (#6830) Fix AREX ARCRESTTimeout +FIX: (#6817) fix account parameter in SSHBatchCE + +*TransformationSystem + +FIX: (#6829) keep transID an int in utilities + +*DataManagementSystem + +FIX: (#6819) specify the same column in order_by and distinct in FTS3DB.getNonFinishedOperations + +*Tests + +NEW: (#6812) Integration test for StorageElementHandler and FileHelper + +[v8.1.0a7] + +*Core + +FIX: (#6811) don't use matplotlib deprecated methods +FIX: (#6793) Fix exception in FileHelper.networkToString on Python3 +FIX: (#6784) Tornado components are initialized using the Tornado system section +FIX: (#6784) each Tornado handler reads its own log level +New: dirac-configure warns you about existing config file +FIX: (#6731) Fix in FileHelper module writing to file, fixed exception "TypeError: write() argument must be str, not bytes, because file was opened in the wrong mode. +CHANGE: (#6706) debug log level instead of info in a few places + +*Resources + +NEW: (#6806) [ALL]ComputingElements - pass pilot stamp to the pilot environment +FIX: (#6768) do not handle ECOMM when removing file +FIX: (#6753) Add a note about XRSL times in ARC(6) vs AREX +NEW: (#6729) SensitiveDataFilter in LogFilters +FIX: (#6719) remove stupid CS sync when getting MQ config +CHANGE: (#6690) drop the WholeNode CE option from Condor +CHANGE: (#6683) https comes before gsiftp in the preference list +NEW: (#6683) Singularity detects locally mounted file system on the worker node to bind mount +FIX: (#6681) fix upload from relative path + +*Docs + +FIX: (#6801) Removed Google forum from docs +NEW: (#6786) Improve documentation on installing HTTPs services +FIX: (#6732) nginx takes aliases into account +NEW: (#6683) document how to access locally mounted file system + +*DataManagementSystem + +CHANGE: (#6787) take into account all possible sources for intermediate copy when replicating files +CHANGE: (#6763) Removed deprecated StorageElementProxy and FileCatalogProxy +NEW: (#6759) Provide `diskOnly` option in getFile. Defaul is `False` +CHANGE: (#6752) adapt FTS3DB to sqlalchemy 2.0 +NEW: (#6697) Introduce getDirectoryDump method in the DFC +CHANGE: (#6697) dirac-dms-user-lfns uses getDirectoryDump method +CHANGE: (#6697) DataManager getReplicasFromDirectory and __cleanDirectory use getDirectoryDump +FIX: (#6696) fix a non initialized variable when doing a TPC with gfal2 +FIX: (#6696) propagate error number when calling getTURL in SRM2 +FIX : (#6696) ConsistencyInspector default value should be a dict and not a list + +*ConfigurationSystem + +FIX: (#6779) VOMS2CSSynchronizer: strip leading and trailing whitespaces from DN entries +FIX: (#6706) getMonitoringBackends returns a list + +*WorkloadManagement + +FIX: (#6756) Repect paramList argument in ElasticJobParametersDB.getJobParameters +FIX: (#6749) Reorder pilot downloads to minimise race condition +FIX: (#6704) Calculating index for JobParametersDB for string job ids + +*Interfaces + +NEW: (#6755) Merge COMDIRAC + +*RequestManagementSystem + +CHANGE: (#6752) adapt ReqDB to sqlalchemy 2.0 + +*FrameworkSystem + +CHANGE: (#6743) Increased length of MySQL field UserProfileDB.up_Users to 64 (was 32) +CHANGE: (#6729) Refactor access to log filters and Logging.__init__() +FIX: (#6718) ComponentSupervisionAgent: Fix problem if two instances in different systems but with the same name are present on the same machine (e.g., Framework/Monitoring and Monitoring/Monitoring) +CHANGE: (#6686) Removed old RabbitMQ specific interface + +*WorkloadManagementSystem + +FIX: (#6706) Matcher handles cases of empty Tag in resource description +FIX: (#6684) InputData optimizer: keep the tape LFNs in + +*Accounting + +FIX: (#6695) JobsPerPilot plot when no data available + +[v8.1.0a6] + +*Resources + +FIX: (#6620) convert ARC to DIRAC JobID in AREXCE +FIX: (#6618) SSHBatchComputingElement needs self.account +FIX: (#6608) HTCondorCE: use CS location Resources/Computing/CEDefaults/HTCondorCE instead of Resources/Computing/HTCondorCE +FIX: (#6595) fix delegation process in the AREXCE + +*WorkloadManagementSystem + +FIX: (#6618) encoding proxy in PilotWrapper.py +FIX: (#6609) ExtraPilotOptions can be a comma-separated list + +*StorageManagamentSystem + +FIX: (#6617) fix py3 incompatibility in wakeupOldRequests + +*DataManagementSystem + +FIX: (#6591) Faster query to list directory for DFC LHCb managers + +[v8.1.0a5] + +*Resources + +FIX: (#6577) multi-node allocations with SLURM +FIX: (#6558) AREXCE._getDelegation() returns an error if the process cannot be completed + +*ConfigurationSystem + +NEW: (#6576) BDII2CSAgent: allow using AREX or ARC6 Computing Elements instead of ARC, fixes #6541 + +*tests + +NEW: (#6569) added system/rms_script.sh test for basic RMS test + +*WorkloadManagementSystem + +CHANGE: (#6568) improve killing of process with psutil +CHANGE: (#6556) completely removed old PilotsLogging machinery + +*DataManagementSystem + +NEW: (#6567) TornadoS3GatewayHandler service + +*ProductionSystem + +NEW: (#6567) TornadoProductionManagerHandler service + +*StorageManagementSystem + +NEW: (#6567) TornadoStorageManagerHandler service + +*MonitoringSystem + +CHANGE: (#6563) remove setup + +*Subsystem + +CHANGE: (#6561) Swapped the default of EnableSecurityLogging flag +FIX: (#6553) simplified NotificationHandler + +*Core + +CHANGE: (#6466) removed DIRAC_USE_JSON_DECODE, flipped DIRAC_USE_JSON_ENCODE + +[v8.1.0a4] + +*Resources + +NEW: (#6530) add the ARCLogLevel option in ARC +NEW: (#6530) add the ComputingInfoEndpoint in ARC6 +FIX: (#6530) restore a commented line in AREX._reset() +FIX: (#6524) Support using ARC that was compiled against SWIG 4.1+ +NEW: (#6506) Added network selection to CloudCE. +FIX: (#6505) simplifying `if` statements for setting bools + +*WorkloadManagement + +FIX: (#6525) Ensure cpuNormalizationFactor is a float +FIX: (#6489) pass CE parameters to dirac-jobexec instead of writing them in the configuration + +*WorkloadManagementSystem + +FIX: (#6521) JobDB.getJobAttributes() returns an empty dict if jobID does not exist +FIX: (#6479) reduce the PollingTime of JobAgent to 20s + +*ResourceStatusSystem + +FIX: (#6513) GOCDB parser to understand also real URLs +FIX: (#6501) replace pre-Python 2.5 ternary syntax + +*docs + +FIX: (#6512) Added CloudComputingElement to AdminGuide/Resources + +*ConfigurationSystem + +FIX: (#6505) simplifying `if` statements for returning bools + +*DataManagementSystem + +FIX: (#6505) simplifying `if` statements for returning bools +FIX: (#6505) use `not in` instead of `not ... in` +FIX: (#6505) simplify loop with `any` call + +*FrameworkSystem + +FIX: (#6505) simplifying `if` statements for setting bools +FIX: (#6505) simplified if statement for checking list/string len +FIX: (#6501) replace pre-Python 2.5 ternary syntax +NEW: (#6450) added TornadoComponentMonitoringHandler +NEW: (#6450) added TornadoNotificationHandler +NEW: (#6450) added TornadoUserProfileManagerHandler + +*TransformationSystem + +FIX: (#6505) simplifying `if` statements [unneeded `not`] +FIX: (#6505) simplified if statement for checking list/string len + +*Core + +FIX: (#6480) quoting the DB passwords for special characters +FIX: (#6462) Scripts: adapt to proper way of getting console_scripts entry_points, mandatory for importlib_metadata v5 +FIX: (#6459) Setting timeout when adding VOMS extension + +*tests + +NEW: (#6463) added Glasgow to sites that should work(TM) in multiVO test +NEW: (#6454) launching the Optimizers in integration-tests + +*Docs + +FIX: (#6462) Fail if one the commands cannot be documented + +[v8.1.0a3] + +*Docs + +FIX: (#6457) Fix the title for the Command Reference Page, fixes #6455 + +*Interfaces + +CHANGE: (#6452) remove JobRepository functionality + +*FrameworkSystem + +CHANGE: (#6451) remove support for myProxy + +*Core + +FIX: (#6448) Only use X- headers if /WebApp/Balancer is defined +FIX: (#6428) MySQL tracer bugs + +*ConfigurationSystem + +FIX: (#6444) remove warning logs when initializing ElasticDB parameters + +*WorkloadManagementSystem + +FIX: (#6441) fix the timestamp of initial logging records +FIX: (#6436) WMSUtilities.killPilotsInQueues have a consistent return value +FIX: (#6432) removed unused SubmissionMode CE parameter + +NEW: (#6429) Information about VirtualOrganization parameter added. + +*Test + +FIX: (#6428) server installation do not install JobAgent + +*MonitoringSystem + +CHANGE: (#6426) Remove PilotSubmissionPlotter + +*WorkloadManagement + +CHANGE: (#6380) Remove VMDIRAC components + +[v8.1.0a2] + +*DataManagementSystem + +FIX: (#6414) Proper casting for data sent to the monitoring +NEW: (#6410) backport getDestinationSEList from LHCbDIRAC +FIX: (#6392) DirectoryLevelTree: fix missing keyword argument name causing failures to upload files for example + +*Tests + +NEW: (#6410) introduce generateDIRACConfig utility to generate DIRAC config during tests + +*Docs + +CHANGE: (#6410) move site documentation from ConfigRef to Resources and dirac.cfg +NEW: (#6410) document Countries + +*WorkloadManagementSystem + +CHANGE: (#6410) JobWrapper uses getDestinationSEList rather that resolveSEGroup +CHANGE: (#6408) ElasticJobParametersDB: create a new index every 1M jobs +FIX: (#6397) JobScheduling: select the correct RescheduleDelay instead of 1 higher + +*MonitoringSystem + +CHANGE: (#6409) Remove DataOperationPlotter + +*Production + +NEW: (#6406) add dirac-prod-complete command + +*Core + +FIX: (#6398) Use safer mode for grid-security directories +FIX: (#6397) TimeUtilities.fromString: when given a datetime.datetime, return the same object, instead of None. Also Fixes JobScheduling issue + +[v8.1.0a1] + +*WorkloadManagementSystem + +FIX: (#6388) replace opPath by opChain in JobPath + +*DataManagementSystem + +FIX: (#6385) Monitoring does not update the dict using by the accounting, avoiding a crash +NEW: (#6385) FailedDataOperation monitoring now keeps track of interactive/job failures +NEW: (#6383) cancel the FTS transfers when an RMS request is found to be canceled + +*Core + +FIX: (#6379) Explicitly depend on db12 +CHANGE: (#6337) Use Python's default SSL ciphers by default + +*Resources + +CHANGE: (#6378) negative free space value is transformed to 0 + [v8.0.0] *Resources diff --git a/releases.cfg b/releases.cfg index 2e347783fe2..ada62dafc8e 100644 --- a/releases.cfg +++ b/releases.cfg @@ -23,9 +23,49 @@ Releases Modules = DIRAC DIRACOS = master } - v8r0-pre4 + v7r3p38 { - Modules = DIRAC, RESTDIRAC:v0r5, COMDIRAC:v0r20, WebAppDIRAC:v4r4-pre1 + Modules = DIRAC, RESTDIRAC:v0r7, COMDIRAC:v0r20, WebAppDIRAC:v4r3p11 + DIRACOS = v1r27 + } + v7r3p37 + { + Modules = DIRAC, RESTDIRAC:v0r7, COMDIRAC:v0r20, WebAppDIRAC:v4r3p11 + DIRACOS = v1r27 + } + v7r3p36 + { + Modules = DIRAC, RESTDIRAC:v0r7, COMDIRAC:v0r20, WebAppDIRAC:v4r3p11 + DIRACOS = v1r27 + } + v7r3p35 + { + Modules = DIRAC, RESTDIRAC:v0r7, COMDIRAC:v0r20, WebAppDIRAC:v4r3p11 + DIRACOS = v1r27 + } + v7r3p34 + { + Modules = DIRAC, RESTDIRAC:v0r7, COMDIRAC:v0r20, WebAppDIRAC:v4r3p11 + DIRACOS = v1r27 + } + v7r3p33 + { + Modules = DIRAC, RESTDIRAC:v0r7, COMDIRAC:v0r20, WebAppDIRAC:v4r3p11 + DIRACOS = v1r27 + } + v7r3p32 + { + Modules = DIRAC, RESTDIRAC:v0r7, COMDIRAC:v0r20, WebAppDIRAC:v4r3p11 + DIRACOS = v1r27 + } + v7r3p31 + { + Modules = DIRAC, RESTDIRAC:v0r7, COMDIRAC:v0r20, WebAppDIRAC:v4r3p11 + DIRACOS = v1r27 + } + v7r3p30 + { + Modules = DIRAC, RESTDIRAC:v0r7, COMDIRAC:v0r20, WebAppDIRAC:v4r3p11 DIRACOS = v1r27 } v7r3p29 @@ -163,6 +203,16 @@ Releases Modules = DIRAC, RESTDIRAC:v0r5, COMDIRAC:v0r19, WebAppDIRAC:v4r3 DIRACOS = v1r26 } + v7r2p52 + { + Modules = DIRAC, VMDIRAC:v2r4p12, RESTDIRAC:v0r5, COMDIRAC:v0r20, WebAppDIRAC:v4r2p12 + DIRACOS = v1r27 + } + v7r2p51 + { + Modules = DIRAC, VMDIRAC:v2r4p12, RESTDIRAC:v0r5, COMDIRAC:v0r20, WebAppDIRAC:v4r2p12 + DIRACOS = v1r27 + } v7r2p50 { Modules = DIRAC, VMDIRAC:v2r4p12, RESTDIRAC:v0r5, COMDIRAC:v0r20, WebAppDIRAC:v4r2p12 diff --git a/setup.cfg b/setup.cfg index 49470ac465f..ae6f999aef9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,40 +14,42 @@ classifiers = Intended Audience :: Science/Research License :: OSI Approved :: GNU General Public License v3 (GPLv3) Programming Language :: Python :: 3 - Programming Language :: Python :: 3.9 - Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 Topic :: Scientific/Engineering Topic :: System :: Distributed Computing [options] -python_requires = >=3.9 +python_requires = >=3.11 package_dir= =src packages = find: -# TODO: This should be treated as a legacy workaround and eventually removed -scripts = - src/DIRAC/Core/scripts/install_site.sh install_requires = - boto3 - botocore + boto3>=1.35 + botocore>=1.35 cachetools certifi + cwltool diraccfg + diracx-client >=v0.0.1a18 + diracx-core >=v0.0.1a18 + db12 fts3 - future gfal2-python importlib_metadata >=4.4 + importlib_resources M2Crypto >=0.36 + packaging pexpect prompt-toolkit >=3 psutil pyasn1 pyasn1-modules + pydantic >=2.4 pyparsing python-dateutil pytz requests - rucio-clients + rucio-clients >=34.4.2 setuptools sqlalchemy typing_extensions >=4.3.0 @@ -69,8 +71,7 @@ server = # (it just installs into site-packages) # arc CMRESHandler - elasticsearch <7.14 - elasticsearch_dsl + opensearch-py GitPython ldap3 apache-libcloud @@ -86,13 +87,13 @@ server = tornado-m2crypto importlib_resources testing = - flaky hypothesis mock parameterized pytest pytest-cov pytest-mock + pytest-rerunfailures pycodestyle [options.entry_points] @@ -116,6 +117,7 @@ console_scripts = dirac-configuration-shell = DIRAC.ConfigurationSystem.scripts.dirac_configuration_shell:main [admin] # Core dirac-agent = DIRAC.Core.scripts.dirac_agent:main [server,pilot] + dirac-apptainer-exec = DIRAC.Core.scripts.dirac_apptainer_exec:main [server,pilot] dirac-configure = DIRAC.Core.scripts.dirac_configure:main dirac-executor = DIRAC.Core.scripts.dirac_executor:main [server] dirac-info = DIRAC.Core.scripts.dirac_info:main @@ -163,6 +165,7 @@ console_scripts = # FrameworkSystem dirac-login = DIRAC.FrameworkSystem.scripts.dirac_login:main dirac-logout = DIRAC.FrameworkSystem.scripts.dirac_logout:main + dirac-diracx-whoami = DIRAC.FrameworkSystem.scripts.dirac_diracx_whoami:main dirac-admin-get-CAs = DIRAC.FrameworkSystem.scripts.dirac_admin_get_CAs:main [server] dirac-admin-get-proxy = DIRAC.FrameworkSystem.scripts.dirac_admin_get_proxy:main [admin] dirac-admin-proxy-upload = DIRAC.FrameworkSystem.scripts.dirac_admin_proxy_upload:main [admin] @@ -171,9 +174,6 @@ console_scripts = dirac-admin-update-pilot = DIRAC.FrameworkSystem.scripts.dirac_admin_update_pilot:main [admin] dirac-admin-users-with-proxy = DIRAC.FrameworkSystem.scripts.dirac_admin_users_with_proxy:main [admin] dirac-install-component = DIRAC.FrameworkSystem.scripts.dirac_install_component:main [server] - dirac-install-tornado-service = DIRAC.FrameworkSystem.scripts.dirac_install_tornado_service:main [server] - dirac-myproxy-upload = DIRAC.FrameworkSystem.scripts.dirac_myproxy_upload:main - dirac-populate-component-db = DIRAC.FrameworkSystem.scripts.dirac_populate_component_db:main [admin] dirac-proxy-destroy = DIRAC.FrameworkSystem.scripts.dirac_proxy_destroy:main dirac-proxy-get-uploaded-info = DIRAC.FrameworkSystem.scripts.dirac_proxy_get_uploaded_info:main dirac-proxy-info = DIRAC.FrameworkSystem.scripts.dirac_proxy_info:main @@ -204,7 +204,6 @@ console_scripts = dirac-admin-modify-user = DIRAC.Interfaces.scripts.dirac_admin_modify_user:main [admin] dirac-admin-pilot-summary = DIRAC.Interfaces.scripts.dirac_admin_pilot_summary:main [admin] dirac-admin-reset-job = DIRAC.Interfaces.scripts.dirac_admin_reset_job:main [admin] - dirac-admin-service-ports = DIRAC.Interfaces.scripts.dirac_admin_service_ports:main [admin] dirac-admin-set-site-protocols = DIRAC.Interfaces.scripts.dirac_admin_set_site_protocols:main [admin] dirac-admin-site-info = DIRAC.Interfaces.scripts.dirac_admin_site_info:main dirac-admin-site-mask-logging = DIRAC.Interfaces.scripts.dirac_admin_site_mask_logging:main [admin] @@ -218,11 +217,8 @@ console_scripts = dirac-dms-replicate-lfn = DIRAC.Interfaces.scripts.dirac_dms_replicate_lfn:main dirac-framework-ping-service = DIRAC.Interfaces.scripts.dirac_framework_ping_service:main [admin] dirac-framework-self-ping = DIRAC.Interfaces.scripts.dirac_framework_self_ping:main [server] - dirac-repo-monitor = DIRAC.Interfaces.scripts.dirac_repo_monitor:main [admin] dirac-utils-file-adler = DIRAC.Interfaces.scripts.dirac_utils_file_adler:main dirac-utils-file-md5 = DIRAC.Interfaces.scripts.dirac_utils_file_md5:main - dirac-wms-get-normalized-queue-length = DIRAC.Interfaces.scripts.dirac_wms_get_normalized_queue_length:main [admin] - dirac-wms-get-queue-normalization = DIRAC.Interfaces.scripts.dirac_wms_get_queue_normalization:main [pilot] dirac-wms-job-attributes = DIRAC.Interfaces.scripts.dirac_wms_job_attributes:main dirac-wms-job-delete = DIRAC.Interfaces.scripts.dirac_wms_job_delete:main dirac-wms-job-get-input = DIRAC.Interfaces.scripts.dirac_wms_job_get_input:main @@ -238,8 +234,32 @@ console_scripts = dirac-wms-job-submit = DIRAC.Interfaces.scripts.dirac_wms_job_submit:main dirac-wms-jobs-select-output-search = DIRAC.Interfaces.scripts.dirac_wms_jobs_select_output_search:main dirac-wms-select-jobs = DIRAC.Interfaces.scripts.dirac_wms_select_jobs:main + dcd = DIRAC.Interfaces.scripts.dcd:main + dchgrp = DIRAC.Interfaces.scripts.dchgrp:main + dchmod = DIRAC.Interfaces.scripts.dchmod:main + dchown = DIRAC.Interfaces.scripts.dchown:main + dconfig = DIRAC.Interfaces.scripts.dconfig:main + dfind = DIRAC.Interfaces.scripts.dfind:main + dget = DIRAC.Interfaces.scripts.dget:main + dgetenv = DIRAC.Interfaces.scripts.dgetenv:main + dkill = DIRAC.Interfaces.scripts.dkill:main + dlogging = DIRAC.Interfaces.scripts.dlogging:main + dls = DIRAC.Interfaces.scripts.dls:main + dmeta = DIRAC.Interfaces.scripts.dmeta:main + dmkdir = DIRAC.Interfaces.scripts.dmkdir:main + doutput = DIRAC.Interfaces.scripts.doutput:main + dput = DIRAC.Interfaces.scripts.dput:main + dpwd = DIRAC.Interfaces.scripts.dpwd:main + drepl = DIRAC.Interfaces.scripts.drepl:main + dreplicas = DIRAC.Interfaces.scripts.dreplicas:main + drm = DIRAC.Interfaces.scripts.drm:main + drmdir = DIRAC.Interfaces.scripts.drmdir:main + dsize = DIRAC.Interfaces.scripts.dsize:main + dstat = DIRAC.Interfaces.scripts.dstat:main + dsub = DIRAC.Interfaces.scripts.dsub:main # ProductionSystem dirac-prod-add-trans = DIRAC.ProductionSystem.scripts.dirac_prod_add_trans:main [admin] + dirac-prod-complete = DIRAC.ProductionSystem.scripts.dirac_prod_complete:main [admin] dirac-prod-clean = DIRAC.ProductionSystem.scripts.dirac_prod_clean:main [admin] dirac-prod-delete = DIRAC.ProductionSystem.scripts.dirac_prod_delete:main [admin] dirac-prod-get = DIRAC.ProductionSystem.scripts.dirac_prod_get:main [admin] @@ -276,20 +296,18 @@ console_scripts = dirac-transformation-clean = DIRAC.TransformationSystem.scripts.dirac_transformation_clean:main [admin] dirac-transformation-cli = DIRAC.TransformationSystem.scripts.dirac_transformation_cli:main [admin] dirac-transformation-get-files = DIRAC.TransformationSystem.scripts.dirac_transformation_get_files:main [admin] + dirac-transformation-information = DIRAC.TransformationSystem.scripts.dirac_transformation_information:main [admin] dirac-transformation-recover-data = DIRAC.TransformationSystem.scripts.dirac_transformation_recover_data:main [admin] dirac-transformation-remove-output = DIRAC.TransformationSystem.scripts.dirac_transformation_remove_output:main [admin] dirac-transformation-replication = DIRAC.TransformationSystem.scripts.dirac_transformation_replication:main [admin] dirac-transformation-verify-outputdata = DIRAC.TransformationSystem.scripts.dirac_transformation_verify_outputdata:main [admin] + dirac-transformation-update-derived = DIRAC.TransformationSystem.scripts.dirac_transformation_update_derived:main [admin] # WorkloadManagementSystem + dirac-admin-add-pilot = DIRAC.WorkloadManagementSystem.scripts.dirac_admin_add_pilot:main [pilot] dirac-admin-kill-pilot = DIRAC.WorkloadManagementSystem.scripts.dirac_admin_kill_pilot:main [admin] - dirac-admin-pilot-logging-info = DIRAC.WorkloadManagementSystem.scripts.dirac_admin_pilot_logging_info:main [admin] dirac-admin-show-task-queues = DIRAC.WorkloadManagementSystem.scripts.dirac_admin_show_task_queues:main [admin] dirac-admin-sync-pilot = DIRAC.WorkloadManagementSystem.scripts.dirac_admin_sync_pilot:main [admin] dirac-jobexec = DIRAC.WorkloadManagementSystem.scripts.dirac_jobexec:main [pilot] - dirac-vm-cli = DIRAC.WorkloadManagementSystem.scripts.dirac_vm_cli:main [admin] - dirac-vm-endpoint-status = DIRAC.WorkloadManagementSystem.scripts.dirac_vm_endpoint_status:main [admin] - dirac-vm-get-pilot-output = DIRAC.WorkloadManagementSystem.scripts.dirac_vm_get_pilot_output:main [admin] - dirac-vm-instance-stop = DIRAC.WorkloadManagementSystem.scripts.dirac_vm_instance_stop:main [admin] dirac-wms-cpu-normalization = DIRAC.WorkloadManagementSystem.scripts.dirac_wms_cpu_normalization:main [pilot] dirac-wms-get-queue-cpu-time = DIRAC.WorkloadManagementSystem.scripts.dirac_wms_get_queue_cpu_time:main [pilot] dirac-wms-get-wn = DIRAC.WorkloadManagementSystem.scripts.dirac_wms_get_wn:main [admin] diff --git a/src/DIRAC/AccountingSystem/Agent/NetworkAgent.py b/src/DIRAC/AccountingSystem/Agent/NetworkAgent.py index e6ac8508776..f8b6f2bfc67 100644 --- a/src/DIRAC/AccountingSystem/Agent/NetworkAgent.py +++ b/src/DIRAC/AccountingSystem/Agent/NetworkAgent.py @@ -29,7 +29,6 @@ class NetworkAgent(AgentModule): BUFFER_TIMEOUT = 3600 def initialize(self): - self.log = gLogger.getSubLogger(self.__class__.__name__) # API initialization is required to get an up-to-date configuration from the CS @@ -92,7 +91,7 @@ def updateNameDictionary(self): result = gConfig.getConfigurationTree("/Resources/Sites", "Network/", "/Enabled") if not result["OK"]: - self.log.error("getConfigurationTree() failed with message: %s" % result["Message"]) + self.log.error(f"getConfigurationTree() failed with message: {result['Message']}") return S_ERROR("Unable to fetch perfSONAR endpoints from CS.") tmpDict = {} @@ -113,17 +112,16 @@ def checkConsumers(self): # recreate consumers if there are any problems if not self.consumers or self.messagesCount == self.messagesCountOld: - for consumer in self.consumers: consumer.close() for uri in self.am_getOption("MessageQueueURI", "").replace(" ", "").split(","): result = createConsumer(uri, self.processMessage) if not result["OK"]: - self.log.error("Failed to create a consumer from URI: %s" % uri) + self.log.error(f"Failed to create a consumer from URI: {uri}") continue else: - self.log.info("Successfully created a consumer from URI: %s" % uri) + self.log.info(f"Successfully created a consumer from URI: {uri}") self.consumers.append(result["Value"]) @@ -157,7 +155,7 @@ def processMessage(self, headers, body): except KeyError as error: # messages with unsupported source or destination host name can be safely skipped self.skippedMessagesCount += 1 - self.log.debug('Host "%s" does not exist in the host-to-dirac name dictionary (message skipped)' % error) + self.log.debug(f'Host "{error}" does not exist in the host-to-dirac name dictionary (message skipped)') return S_OK() metadataKey = "" @@ -180,7 +178,7 @@ def processMessage(self, headers, body): timeDifference = datetime.now() - self.buffer[networkAccountingObjectKey]["addTime"] if timeDifference.total_seconds() > 60: - self.log.warn("Object was taken from buffer after %s" % (timeDifference)) + self.log.warn(f"Object was taken from buffer after {timeDifference}") else: net = Network() net.setStartTime(date) @@ -194,7 +192,7 @@ def processMessage(self, headers, body): if headers["event-type"] == "packet-loss-rate": self.PLRMetricCount += 1 if metricData < 0 or metricData > 1: - raise Exception("Invalid PLR metric (%s)" % (metricData)) + raise Exception(f"Invalid PLR metric ({metricData})") net.setValueByKey("PacketLossRate", metricData * 100) elif headers["event-type"] == "histogram-owdelay": @@ -230,7 +228,7 @@ def processMessage(self, headers, body): # suppress all exceptions to protect the listener thread except Exception as e: self.skippedMetricCount += 1 - self.log.warn("Metric skipped because of an exception: %s" % e) + self.log.warn(f"Metric skipped because of an exception: {e}") return S_OK() @@ -266,14 +264,14 @@ def commitData(self): def showStatistics(self): """Display different statistics as info messages in the log file.""" - self.log.info("\tReceived messages: %s" % self.messagesCount) - self.log.info("\tSkipped messages: %s" % self.skippedMessagesCount) - self.log.info("\tPacket-Loss-Rate metrics: %s" % self.PLRMetricCount) - self.log.info("\tOne-Way-Delay metrics: %s" % self.OWDMetricCount) - self.log.info("\tSkipped metrics: %s" % self.skippedMetricCount) + self.log.info(f"\tReceived messages: {self.messagesCount}") + self.log.info(f"\tSkipped messages: {self.skippedMessagesCount}") + self.log.info(f"\tPacket-Loss-Rate metrics: {self.PLRMetricCount}") + self.log.info(f"\tOne-Way-Delay metrics: {self.OWDMetricCount}") + self.log.info(f"\tSkipped metrics: {self.skippedMetricCount}") self.log.info("") - self.log.info("\tObjects in the buffer: %s" % len(self.buffer)) - self.log.info("\tObjects inserted to DB: %s" % self.insertedCount) - self.log.info("\tPermanently removed objects: %s" % self.removedCount) + self.log.info(f"\tObjects in the buffer: {len(self.buffer)}") + self.log.info(f"\tObjects inserted to DB: {self.insertedCount}") + self.log.info(f"\tPermanently removed objects: {self.removedCount}") return S_OK() diff --git a/src/DIRAC/AccountingSystem/Agent/test/Test_NetworkAgent.py b/src/DIRAC/AccountingSystem/Agent/test/Test_NetworkAgent.py index 95140f3f2ff..bcc16830b3d 100644 --- a/src/DIRAC/AccountingSystem/Agent/test/Test_NetworkAgent.py +++ b/src/DIRAC/AccountingSystem/Agent/test/Test_NetworkAgent.py @@ -36,7 +36,6 @@ class NetworkAgentSuccessTestCase(unittest.TestCase): """Test class to check success scenarios.""" def setUp(self): - # external dependencies module.datetime = MagicMock() @@ -62,7 +61,6 @@ def tearDownClass(cls): sys.modules.pop("DIRAC.AccountingSystem.Agent.NetworkAgent") def test_updateNameDictionary(self): - module.gConfig.getConfigurationTree.side_effect = [ {"OK": True, "Value": INITIAL_CONFIG}, {"OK": True, "Value": UPDATED_CONFIG}, @@ -85,7 +83,6 @@ def test_updateNameDictionary(self): self.assertRaises(KeyError, lambda: self.agent.nameDictionary[SITE2_HOST1]) def test_agentExecute(self): - module.NetworkAgent.am_getOption.return_value = f"{MQURI1}, {MQURI2}" module.gConfig.getConfigurationTree.return_value = {"OK": True, "Value": INITIAL_CONFIG} diff --git a/src/DIRAC/AccountingSystem/Client/AccountingCLI.py b/src/DIRAC/AccountingSystem/Client/AccountingCLI.py index e67f9036f53..ad11d660a60 100644 --- a/src/DIRAC/AccountingSystem/Client/AccountingCLI.py +++ b/src/DIRAC/AccountingSystem/Client/AccountingCLI.py @@ -8,12 +8,14 @@ from DIRAC import gLogger from DIRAC.Core.Base.CLI import CLI, colorize from DIRAC.AccountingSystem.Client.DataStoreClient import DataStoreClient +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader class AccountingCLI(CLI): def __init__(self): CLI.__init__(self) self.do_connect(None) + self.objectLoader = ObjectLoader() def start(self): """ @@ -34,17 +36,17 @@ def do_connect(self, args): """ gLogger.info("Trying to connect to server") self.connected = False - self.prompt = "(%s)> " % colorize("Not connected", "red") + self.prompt = f"({colorize('Not connected', 'red')})> " acClient = DataStoreClient() retVal = acClient.ping() if retVal["OK"]: - self.prompt = "(%s)> " % colorize("Connected", "green") + self.prompt = f"({colorize('Connected', 'green')})> " self.connected = True def printComment(self, comment): commentList = comment.split("\n") for commentLine in commentList[:-1]: - print("# %s" % commentLine.strip()) + print(f"# {commentLine.strip()}") def showTraceback(self): import traceback @@ -70,22 +72,19 @@ def do_registerType(self, args): gLogger.error("No type name specified") return # Try to import the type - try: - typeModule = __import__( - "DIRAC.AccountingSystem.Client.Types.%s" % typeName, globals(), locals(), typeName - ) - typeClass = getattr(typeModule, typeName) - except Exception as e: - gLogger.error(f"Can't load type {typeName}: {str(e)}") - return - gLogger.info("Loaded type %s" % typeClass.__name__) + result = self.objectLoader.loadObject(f"DIRAC.AccountingSystem.Client.Types.{typeName}") + if not result["OK"]: + return result + typeClass = result["Value"] + + gLogger.info(f"Loaded type {typeClass.__name__}") typeDef = typeClass().getDefinition() acClient = DataStoreClient() retVal = acClient.registerType(*typeDef) if retVal["OK"]: gLogger.info("Type registered successfully") else: - gLogger.error("Error: %s" % retVal["Message"]) + gLogger.error(f"Error: {retVal['Message']}") except Exception: self.showTraceback() @@ -103,23 +102,20 @@ def do_resetBucketLength(self, args): else: gLogger.error("No type name specified") return + # Try to import the type - try: - typeModule = __import__( - "DIRAC.AccountingSystem.Client.Types.%s" % typeName, globals(), locals(), typeName - ) - typeClass = getattr(typeModule, typeName) - except Exception as e: - gLogger.error(f"Can't load type {typeName}: {str(e)}") - return - gLogger.info("Loaded type %s" % typeClass.__name__) + result = self.objectLoader.loadObject(f"DIRAC.AccountingSystem.Client.Types.{typeName}") + if not result["OK"]: + return result + typeClass = result["Value"] + gLogger.info(f"Loaded type {typeClass.__name__}") typeDef = typeClass().getDefinition() acClient = DataStoreClient() retVal = acClient.setBucketsLength(typeDef[0], typeDef[3]) if retVal["OK"]: gLogger.info("Type registered successfully") else: - gLogger.error("Error: %s" % retVal["Message"]) + gLogger.error(f"Error: {retVal['Message']}") except Exception: self.showTraceback() @@ -137,23 +133,20 @@ def do_regenerateBuckets(self, args): else: gLogger.error("No type name specified") return + # Try to import the type - try: - typeModule = __import__( - "DIRAC.AccountingSystem.Client.Types.%s" % typeName, globals(), locals(), typeName - ) - typeClass = getattr(typeModule, typeName) - except Exception as e: - gLogger.error(f"Can't load type {typeName}: {str(e)}") - return - gLogger.info("Loaded type %s" % typeClass.__name__) + result = self.objectLoader.loadObject(f"DIRAC.AccountingSystem.Client.Types.{typeName}") + if not result["OK"]: + return result + typeClass = result["Value"] + gLogger.info(f"Loaded type {typeClass.__name__}") typeDef = typeClass().getDefinition() acClient = DataStoreClient() retVal = acClient.regenerateBuckets(typeDef[0]) if retVal["OK"]: gLogger.info("Buckets recalculated!") else: - gLogger.error("Error: %s" % retVal["Message"]) + gLogger.error(f"Error: {retVal['Message']}") except Exception: self.showTraceback() @@ -169,7 +162,7 @@ def do_showRegisteredTypes(self, args): print(retVal) if not retVal["OK"]: - gLogger.error("Error: %s" % retVal["Message"]) + gLogger.error(f"Error: {retVal['Message']}") return for typeList in retVal["Value"]: print(typeList[0]) @@ -192,20 +185,19 @@ def do_deleteType(self, args): else: gLogger.error("No type name specified") return - while True: - choice = input( - "Are you completely sure you want to delete type %s and all it's data? yes/no [no]: " % typeName - ) - choice = choice.lower() - if choice in ("yes", "y"): - break - else: - print("Delete aborted") - return + + choice = input( + f"Are you completely sure you want to delete type {typeName} and all it's data? yes/no [no]: " + ) + choice = choice.lower() + if choice not in ("yes", "y"): + print("Delete aborted") + return + acClient = DataStoreClient() retVal = acClient.deleteType(typeName) if not retVal["OK"]: - gLogger.error("Error: %s" % retVal["Message"]) + gLogger.error(f"Error: {retVal['Message']}") return print("Hope you meant it, because it's done") except Exception: @@ -220,7 +212,7 @@ def do_compactBuckets(self, args): acClient = DataStoreClient() retVal = acClient.compactDB() if not retVal["OK"]: - gLogger.error("Error: %s" % retVal["Message"]) + gLogger.error(f"Error: {retVal['Message']}") return gLogger.info("Done") except Exception: diff --git a/src/DIRAC/AccountingSystem/Client/DataStoreClient.py b/src/DIRAC/AccountingSystem/Client/DataStoreClient.py index d41611526ce..6e796dc7d9c 100644 --- a/src/DIRAC/AccountingSystem/Client/DataStoreClient.py +++ b/src/DIRAC/AccountingSystem/Client/DataStoreClient.py @@ -103,7 +103,7 @@ def commit(self): del registersList[: self.__maxRecordsInABundle] except Exception as e: # pylint: disable=broad-except gLogger.exception("Error committing", lException=e) - return S_ERROR("Error committing %s" % repr(e).replace(",)", ")")) + return S_ERROR(f"Error committing {repr(e).replace(',)', ')')}") finally: # if something is left because of an error return it to the main list self.__registersList.extend(registersList) @@ -150,7 +150,7 @@ def _sendToFailover(rpcStub): # We catch all the exceptions, because it should never crash except Exception as e: # pylint: disable=broad-except - return S_ERROR(ERMSUKN, "Exception sending accounting failover request: %s" % repr(e)) + return S_ERROR(ERMSUKN, f"Exception sending accounting failover request: {repr(e)}") gDataStoreClient = DataStoreClient() diff --git a/src/DIRAC/AccountingSystem/Client/ReportCLI.py b/src/DIRAC/AccountingSystem/Client/ReportCLI.py index 78a6dbe82b9..5eed280970d 100644 --- a/src/DIRAC/AccountingSystem/Client/ReportCLI.py +++ b/src/DIRAC/AccountingSystem/Client/ReportCLI.py @@ -48,16 +48,16 @@ def do_connect(self, args): """ gLogger.info("Trying to connect to server") self.connected = False - self.prompt = "(%s)> " % colorize("Not connected", "red") + self.prompt = f"({colorize('Not connected', 'red')})> " retVal = ReportsClient().ping() if retVal["OK"]: - self.prompt = "(%s)> " % colorize("Connected", "green") + self.prompt = f"({colorize('Connected', 'green')})> " self.connected = True def printComment(self, comment): commentList = comment.split("\n") for commentLine in commentList[:-1]: - print("# %s" % commentLine.strip()) + print(f"# {commentLine.strip()}") def showTraceback(self): import traceback diff --git a/src/DIRAC/AccountingSystem/Client/ReportsClient.py b/src/DIRAC/AccountingSystem/Client/ReportsClient.py index f090da6885f..e57fc5aa70a 100644 --- a/src/DIRAC/AccountingSystem/Client/ReportsClient.py +++ b/src/DIRAC/AccountingSystem/Client/ReportsClient.py @@ -20,8 +20,7 @@ def __init__(self, transferClient=None, **kwargs): def __getTransferClient(self): if not self.transferClient: return TransferClient("Accounting/ReportGenerator") - else: - return self.transferClient + return self.transferClient def listReports(self, typeName): result = self._getRPC().listReports(typeName) @@ -30,7 +29,6 @@ def listReports(self, typeName): return result def getReport(self, typeName, reportName, startTime, endTime, condDict, grouping, extraArgs=None): - if not isinstance(extraArgs, dict): extraArgs = {} plotRequest = { diff --git a/src/DIRAC/AccountingSystem/Client/Types/BaseAccountingType.py b/src/DIRAC/AccountingSystem/Client/Types/BaseAccountingType.py index 0add7585918..b68a1902455 100644 --- a/src/DIRAC/AccountingSystem/Client/Types/BaseAccountingType.py +++ b/src/DIRAC/AccountingSystem/Client/Types/BaseAccountingType.py @@ -81,7 +81,7 @@ def setValueByKey(self, key, value): Add value for key """ if key not in self.fieldsList: - return S_ERROR("Key %s is not defined" % key) + return S_ERROR(f"Key {key} is not defined") keyPos = self.fieldsList.index(key) self.valuesList[keyPos] = value return S_OK() @@ -95,7 +95,7 @@ def setValuesFromDict(self, dataDict): if key not in self.fieldsList: errKeys.append(key) if errKeys: - return S_ERROR("Key(s) %s are not valid" % ", ".join(errKeys)) + return S_ERROR(f"Key(s) {', '.join(errKeys)} are not valid") for key in dataDict: self.setValueByKey(key, dataDict[key]) return S_OK() @@ -104,9 +104,9 @@ def getValue(self, key): try: return S_OK(self.valuesList[self.fieldsList.index(key)]) except IndexError: - return S_ERROR("%s does not have a value" % key) + return S_ERROR(f"{key} does not have a value") except ValueError: - return S_ERROR("%s is not a valid key" % key) + return S_ERROR(f"{key} is not a valid key") def checkValues(self): """ @@ -116,11 +116,11 @@ def checkValues(self): for i in range(len(self.valuesList)): key = self.fieldsList[i] if self.valuesList[i] is None: - errorList.append("no value for %s" % key) + errorList.append(f"no value for {key}") if key in self.valueFieldsList and not isinstance(self.valuesList[i], (int, float)): - errorList.append("value for key %s is not numerical type" % key) + errorList.append(f"value for key {key} is not numerical type") if errorList: - return S_ERROR("Invalid values: %s" % ", ".join(errorList)) + return S_ERROR(f"Invalid values: {', '.join(errorList)}") if not self.startTime: return S_ERROR("Start time has not been defined") if not isinstance(self.startTime, datetime.datetime): diff --git a/src/DIRAC/AccountingSystem/Client/Types/WMSHistory.py b/src/DIRAC/AccountingSystem/Client/Types/WMSHistory.py index 4277b9d6b54..8e3bd5264e0 100644 --- a/src/DIRAC/AccountingSystem/Client/Types/WMSHistory.py +++ b/src/DIRAC/AccountingSystem/Client/Types/WMSHistory.py @@ -1,5 +1,5 @@ """ MySQL based WMSHistory accounting. - It's suggested to replace this with the ElasticSearch based WMSHistory monitoring. + It's suggested to replace this with the OpenSearch based WMSHistory monitoring. Filled by the agent "WorkloadManagement/StatesAccountingAgent" """ diff --git a/src/DIRAC/AccountingSystem/DB/AccountingDB.py b/src/DIRAC/AccountingSystem/DB/AccountingDB.py index 7f18b257d17..e66686ce00a 100644 --- a/src/DIRAC/AccountingSystem/DB/AccountingDB.py +++ b/src/DIRAC/AccountingSystem/DB/AccountingDB.py @@ -1,13 +1,13 @@ """ Frontend to MySQL DB AccountingDB """ import datetime -import time -import threading import random +import threading +import time +from DIRAC import S_ERROR, S_OK from DIRAC.Core.Base.DB import DB -from DIRAC import S_OK, S_ERROR, gConfig -from DIRAC.Core.Utilities import List, ThreadSafe, DEncode, TimeUtilities +from DIRAC.Core.Utilities import DEncode, List, ThreadSafe, TimeUtilities from DIRAC.Core.Utilities.Plotting.TypeLoader import TypeLoader from DIRAC.Core.Utilities.ThreadPool import ThreadPool @@ -21,11 +21,8 @@ def __init__(self, name="Accounting/AccountingDB", readOnly=False, parentLogger= self.autoCompact = False self.__readOnly = readOnly self.__doingCompaction = False - self.__oldBucketMethod = False self.__doingPendingLockTime = 0 self.__deadLockRetries = 2 - self.__queuedRecordsLock = ThreadSafe.Synchronizer() - self.__queuedRecordsToInsert = [] self.dbCatalog = {} self.dbBucketsLength = {} self.__keysCache = {} @@ -52,7 +49,6 @@ def __init__(self, name="Accounting/AccountingDB", readOnly=False, parentLogger= lcd = datetime.datetime.utcnow() lcd.replace(hour=self.__compactTime.hour + 1, minute=0, second=0) self.__lastCompactionEpoch = TimeUtilities.toEpoch(lcd) - self.__registerTypes() def __loadTablesCreated(self): @@ -75,7 +71,7 @@ def __periodicAutoCompactDB(self): nct = nct.replace( hour=self.__compactTime.hour, minute=self.__compactTime.minute, second=self.__compactTime.second ) - self.log.info("Next db compaction", "will be at %s" % nct) + self.log.info("Next db compaction", f"will be at {nct}") sleepTime = TimeUtilities.toEpoch(nct) - TimeUtilities.toEpoch() time.sleep(sleepTime) self.compactBuckets() @@ -84,52 +80,45 @@ def __registerTypes(self): """ Register all types """ - retVal = gConfig.getSections("/DIRAC/Setups") - if not retVal["OK"]: - return S_ERROR("Can't get a list of setups: %s" % retVal["Message"]) - setupsList = retVal["Value"] objectsLoaded = TypeLoader().getTypes() # Load the files - for pythonClassName in sorted(objectsLoaded): - typeClass = objectsLoaded[pythonClassName] - for setup in setupsList: - typeName = f"{setup}_{pythonClassName}" - - typeDef = typeClass().getDefinition() - # dbTypeName = "%s_%s" % ( setup, typeName ) - definitionKeyFields, definitionAccountingFields, bucketsLength = typeDef[1:] - # If already defined check the similarities - if typeName in self.dbCatalog: - bucketsLength.sort() - if bucketsLength != self.dbBucketsLength[typeName]: - bucketsLength = self.dbBucketsLength[typeName] - self.log.warn("Bucket length has changed", "for type %s" % typeName) - keyFields = [f[0] for f in definitionKeyFields] - if keyFields != self.dbCatalog[typeName]["keys"]: - keyFields = self.dbCatalog[typeName]["keys"] - self.log.error("Definition fields have changed", "Type %s" % typeName) - valueFields = [f[0] for f in definitionAccountingFields] - if valueFields != self.dbCatalog[typeName]["values"]: - valueFields = self.dbCatalog[typeName]["values"] - self.log.error("Accountable fields have changed", "Type %s" % typeName) - # Try to re register to check all the tables are there - retVal = self.registerType(typeName, definitionKeyFields, definitionAccountingFields, bucketsLength) - if not retVal["OK"]: - self.log.error("Can't register type", "{}: {}".format(typeName, retVal["Message"])) - # If it has been properly registered, update info - elif retVal["Value"]: - # Set the timespan - self.dbCatalog[typeName]["dataTimespan"] = typeClass().getDataTimespan() - self.dbCatalog[typeName]["definition"] = { - "keys": definitionKeyFields, - "values": definitionAccountingFields, - } + for typeName in sorted(objectsLoaded): + typeClass = objectsLoaded[typeName] + + typeDef = typeClass().getDefinition() + definitionKeyFields, definitionAccountingFields, bucketsLength = typeDef[1:] + # If already defined check the similarities + if typeName in self.dbCatalog: + bucketsLength.sort() + if bucketsLength != self.dbBucketsLength[typeName]: + bucketsLength = self.dbBucketsLength[typeName] + self.log.warn("Bucket length has changed", f"for type {typeName}") + keyFields = [f[0] for f in definitionKeyFields] + if keyFields != self.dbCatalog[typeName]["keys"]: + keyFields = self.dbCatalog[typeName]["keys"] + self.log.error("Definition fields have changed", f"Type {typeName}") + valueFields = [f[0] for f in definitionAccountingFields] + if valueFields != self.dbCatalog[typeName]["values"]: + valueFields = self.dbCatalog[typeName]["values"] + self.log.error("Accountable fields have changed", f"Type {typeName}") + # Try to re register to check all the tables are there + retVal = self.registerType(typeName, definitionKeyFields, definitionAccountingFields, bucketsLength) + if not retVal["OK"]: + self.log.error("Can't register type", f"{typeName}: {retVal['Message']}") + # If it has been properly registered, update info + elif retVal["Value"]: + # Set the timespan + self.dbCatalog[typeName]["dataTimespan"] = typeClass().getDataTimespan() + self.dbCatalog[typeName]["definition"] = { + "keys": definitionKeyFields, + "values": definitionAccountingFields, + } return S_OK() def __loadCatalogFromDB(self): retVal = self._query( - "SELECT `name`, `keyFields`, `valueFields`, `bucketsLength` FROM `%s`" % self.catalogTableName + f"SELECT `name`, `keyFields`, `valueFields`, `bucketsLength` FROM `{self.catalogTableName}`" ) if not retVal["OK"]: raise Exception(retVal["Message"]) @@ -154,7 +143,7 @@ def markAllPendingRecordsAsNotTaken(self): self.log.always("Marking all records to be processed as not taken") for typeName in self.dbCatalog: sqlTableName = _getTableName("in", typeName) - result = self._update("UPDATE `%s` SET taken=0" % sqlTableName) + result = self._update(f"UPDATE `{sqlTableName}` SET taken=0") if not result["OK"]: return result return S_OK() @@ -175,8 +164,8 @@ def loadPendingRecords(self): pending = 0 now = TimeUtilities.toEpoch() recordsPerSlot = self.getCSOption("RecordsPerSlot", 100) - for typeName in self.dbCatalog: - self.log.info("[PENDING] Checking %s" % typeName) + for typeName, typeDef in self.dbCatalog.items(): + self.log.info(f"[PENDING] Checking {typeName}") pendingInQueue = self.__threadPool.pendingJobs() emptySlots = max(0, 3000 - pendingInQueue) self.log.info("[PENDING] %s in the queue, %d empty slots" % (pendingInQueue, emptySlots)) @@ -184,7 +173,7 @@ def loadPendingRecords(self): continue emptySlots = min(100, emptySlots) sqlTableName = _getTableName("in", typeName) - sqlFields = ["id"] + self.dbCatalog[typeName]["typeFields"] + sqlFields = ["id"] + typeDef["typeFields"] sqlCond = ( "WHERE taken = 0 or TIMESTAMPDIFF( SECOND, takenSince, UTC_TIMESTAMP() ) > %s" % self.getWaitingRecordsLifeTime() @@ -196,10 +185,10 @@ def loadPendingRecords(self): if not result["OK"]: self.log.error( "[PENDING] Error when trying to get pending records", - "for {} : {}".format(typeName, result["Message"]), + f"for {typeName} : {result['Message']}", ) return result - self.log.info("[PENDING] Got {} pending records for type {}".format(len(result["Value"]), typeName)) + self.log.info(f"[PENDING] Got {len(result['Value'])} pending records for type {typeName}") dbData = result["Value"] idList = [str(r[0]) for r in dbData] # If nothing to do, continue @@ -212,7 +201,7 @@ def loadPendingRecords(self): if not result["OK"]: self.log.error( "[PENDING] Error when trying set state to waiting records", - "for {} : {}".format(typeName, result["Message"]), + f"for {typeName} : {result['Message']}", ) self.__doingPendingLockTime = 0 return result @@ -230,7 +219,7 @@ def loadPendingRecords(self): recordsToProcess = [] if recordsToProcess: self.__threadPool.generateJobAndQueueIt(self.__insertFromINTable, args=(recordsToProcess,)) - self.log.info("[PENDING] Got %s records requests for all types" % pending) + self.log.info(f"[PENDING] Got {pending} records requests for all types") self.__doingPendingLockTime = 0 return S_OK() @@ -238,7 +227,7 @@ def __addToCatalog(self, typeName, keyFields, valueFields, bucketsLength): """ Add type to catalog """ - self.log.verbose("Adding to catalog type %s" % typeName, "with length %s" % str(bucketsLength)) + self.log.verbose(f"Adding to catalog type {typeName}", f"with length {str(bucketsLength)}") self.dbCatalog[typeName] = { "keys": keyFields, "values": valueFields, @@ -252,14 +241,13 @@ def __addToCatalog(self, typeName, keyFields, valueFields, bucketsLength): self.dbCatalog[typeName]["typeFields"].extend(["startTime", "endTime"]) self.dbCatalog[typeName]["bucketFields"].extend(["entriesInBucket", "startTime", "bucketLength"]) self.dbBucketsLength[typeName] = bucketsLength - # ADRI: TEST COMPACT BUCKETS - # self.dbBucketsLength[ typeName ] = [ ( 31104000, 3600 ) ] def changeBucketsLength(self, typeName, bucketsLength): gSynchro.lock() + try: if typeName not in self.dbCatalog: - return S_ERROR("%s is not a valid type name" % typeName) + return S_ERROR(f"{typeName} is not a valid type name") bucketsLength.sort() bucketsEncoding = DEncode.encode(bucketsLength) retVal = self._update( @@ -291,10 +279,10 @@ def registerType(self, name, definitionKeyFields, definitionAccountingFields, bu valueFieldsList.append(value[0]) for field in definitionKeyFields: if field in valueFieldsList: - return S_ERROR("Key field %s is also in the list of value fields" % field) + return S_ERROR(f"Key field {field} is also in the list of value fields") for field in definitionAccountingFields: if field in keyFieldsList: - return S_ERROR("Value field %s is also in the list of key fields" % field) + return S_ERROR(f"Value field {field} is also in the list of key fields") for bucket in bucketsLength: if not isinstance(bucket, tuple): return S_ERROR("Length of buckets should be a list of tuples") @@ -307,9 +295,9 @@ def registerType(self, name, definitionKeyFields, definitionAccountingFields, bu for key in definitionKeyFields: keyTableName = _getTableName("key", name, key[0]) if keyTableName not in tablesInThere: - self.log.info("Table for key %s has to be created" % key[0]) + self.log.info(f"Table for key {key[0]} has to be created") tables[keyTableName] = { - "Fields": {"id": "INTEGER NOT NULL AUTO_INCREMENT", "value": "%s NOT NULL" % key[1]}, + "Fields": {"id": "INTEGER NOT NULL AUTO_INCREMENT", "value": f"{key[1]} NOT NULL"}, "UniqueIndexes": {"valueindex": ["value"]}, "PrimaryKey": "id", } @@ -320,7 +308,7 @@ def registerType(self, name, definitionKeyFields, definitionAccountingFields, bu bucketIndexes = {"startTimeIndex": ["startTime"], "bucketLengthIndex": ["bucketLength"]} uniqueIndexFields = ["startTime"] for field in definitionKeyFields: - bucketIndexes["%sIndex" % field[0]] = [field[0]] + bucketIndexes[f"{field[0]}Index"] = [field[0]] uniqueIndexFields.append(field[0]) fieldsDict[field[0]] = "INTEGER NOT NULL" bucketFieldsDict[field[0]] = "INTEGER NOT NULL" @@ -353,21 +341,21 @@ def registerType(self, name, definitionKeyFields, definitionAccountingFields, bu tables[inTableName] = {"Fields": inbufferDict, "PrimaryKey": "id"} if self.__readOnly: if tables: - self.log.notice("ReadOnly mode: Skipping create of tables for %s. Removing from memory catalog" % name) - self.log.verbose("Skipping creation of tables %s" % ", ".join([tn for tn in tables])) + self.log.notice(f"ReadOnly mode: Skipping create of tables for {name}. Removing from memory catalog") + self.log.verbose(f"Skipping creation of tables {', '.join([tn for tn in tables])}") try: self.dbCatalog.pop(name) except KeyError: pass else: - self.log.notice("ReadOnly mode: %s is OK" % name) + self.log.notice(f"ReadOnly mode: {name} is OK") return S_OK(not updateDBCatalog) if tables: retVal = self._createTables(tables) if not retVal["OK"]: - self.log.error("Can't create type", "{}: {}".format(name, retVal["Message"])) - return S_ERROR("Can't create type {}: {}".format(name, retVal["Message"])) + self.log.error("Can't create type", f"{name}: {retVal['Message']}") + return S_ERROR(f"Can't create type {name}: {retVal['Message']}") if updateDBCatalog: bucketsLength.sort() bucketsEncoding = DEncode.encode(bucketsLength) @@ -377,7 +365,7 @@ def registerType(self, name, definitionKeyFields, definitionAccountingFields, bu [name, ",".join(keyFieldsList), ",".join(valueFieldsList), bucketsEncoding], ) self.__addToCatalog(name, keyFieldsList, valueFieldsList, bucketsLength) - self.log.info("Registered type %s" % name) + self.log.info(f"Registered type {name}") return S_OK(True) def getRegisteredTypes(self): @@ -385,7 +373,7 @@ def getRegisteredTypes(self): Get list of registered types """ retVal = self._query( - "SELECT `name`, `keyFields`, `valueFields`, `bucketsLength` FROM `%s`" % self.catalogTableName + f"SELECT `name`, `keyFields`, `valueFields`, `bucketsLength` FROM `{self.catalogTableName}`" ) if not retVal["OK"]: return retVal @@ -405,7 +393,7 @@ def getKeyValues(self, typeName, condDict, connObj=False): keyTables = [] sqlCond = [] - mainTable = "`%s`" % _getTableName("bucket", typeName) + mainTable = f"`{_getTableName('bucket', typeName)}`" try: typeKeysList = self.dbCatalog[typeName]["keys"] except KeyError: @@ -413,23 +401,23 @@ def getKeyValues(self, typeName, condDict, connObj=False): for keyName in condDict: if keyName in typeKeysList: - keyTable = "`%s`" % _getTableName("key", typeName, keyName) + keyTable = f"`{_getTableName('key', typeName, keyName)}`" if keyTable not in keyTables: keyTables.append(keyTable) sqlCond.append(f"{keyTable}.id = {mainTable}.`{keyName}`") for value in condDict[keyName]: - sqlCond.append("{}.value = {}".format(keyTable, self._escapeString(value)["Value"])) + sqlCond.append(f"{keyTable}.value = {self._escapeString(value)['Value']}") for keyName in typeKeysList: - keyTable = "`%s`" % _getTableName("key", typeName, keyName) + keyTable = f"`{_getTableName('key', typeName, keyName)}`" allKeyTables = keyTables if keyTable not in allKeyTables: allKeyTables = list(keyTables) allKeyTables.append(keyTable) - cmd = "SELECT DISTINCT {}.value FROM {}".format(keyTable, ", ".join(allKeyTables)) + cmd = f"SELECT DISTINCT {keyTable}.value FROM {', '.join(allKeyTables)}" if sqlCond: sqlValueLink = f"{keyTable}.id = {mainTable}.`{keyName}`" - cmd += ", {} WHERE {} AND {}".format(mainTable, sqlValueLink, " AND ".join(sqlCond)) + cmd += f", {mainTable} WHERE {sqlValueLink} AND {' AND '.join(sqlCond)}" retVal = self._query(cmd, conn=connObj) if not retVal["OK"]: return retVal @@ -445,18 +433,18 @@ def deleteType(self, typeName): if self.__readOnly: return S_ERROR("ReadOnly mode enabled. No modification allowed") if typeName not in self.dbCatalog: - return S_ERROR("Type %s does not exist" % typeName) + return S_ERROR(f"Type {typeName} does not exist") self.log.info("Deleting type", typeName) tablesToDelete = [] for keyField in self.dbCatalog[typeName]["keys"]: - tablesToDelete.append("`%s`" % _getTableName("key", typeName, keyField)) - tablesToDelete.insert(0, "`%s`" % _getTableName("type", typeName)) - tablesToDelete.insert(0, "`%s`" % _getTableName("bucket", typeName)) - tablesToDelete.insert(0, "`%s`" % _getTableName("in", typeName)) - retVal = self._query("DROP TABLE %s" % ", ".join(tablesToDelete)) + tablesToDelete.append(f"`{_getTableName('key', typeName, keyField)}`") + tablesToDelete.insert(0, f"`{_getTableName('type', typeName)}`") + tablesToDelete.insert(0, f"`{_getTableName('bucket', typeName)}`") + tablesToDelete.insert(0, f"`{_getTableName('in', typeName)}`") + retVal = self._query(f"DROP TABLE {', '.join(tablesToDelete)}") if not retVal["OK"]: return retVal - retVal = self._update("DELETE FROM `{}` WHERE name='{}'".format(_getTableName("catalog", "Types"), typeName)) + retVal = self._update(f"DELETE FROM `{_getTableName('catalog', 'Types')}` WHERE name='{typeName}'") del self.dbCatalog[typeName] return S_OK() @@ -469,7 +457,7 @@ def __getIdForKeyValue(self, typeName, keyName, keyValue, conn=False): return retVal keyValue = retVal["Value"] retVal = self._query( - "SELECT `id` FROM `{}` WHERE `value`={}".format(_getTableName("key", typeName, keyName), keyValue), + f"SELECT `id` FROM `{_getTableName('key', typeName, keyName)}` WHERE `value`={keyValue}", conn=conn, ) if not retVal["OK"]: @@ -510,7 +498,7 @@ def __addKeyValue(self, typeName, keyName, keyValue): return retVal connection = retVal["Value"] self.log.info(f"Value {keyValue} for key {keyName} didn't exist, inserting") - retVal = self.insertFields(keyTable, ["id", "value"], [0, keyValue], conn=connection) + retVal = self.insertFields(keyTable, ["value"], [keyValue], conn=connection) if not retVal["OK"] and retVal["Message"].find("Duplicate key") == -1: return retVal result = self.__getIdForKeyValue(typeName, keyName, keyValue, conn=connection) @@ -553,8 +541,8 @@ def calculateBuckets(self, typeName, startTime, endTime, nowEpoch=False): return buckets def __insertInQueueTable(self, typeName, startTime, endTime, valuesList): - sqlFields = ["id", "taken", "takenSince"] + self.dbCatalog[typeName]["typeFields"] - sqlValues = ["0", "0", "UTC_TIMESTAMP()"] + valuesList + [startTime, endTime] + sqlFields = ["taken", "takenSince"] + self.dbCatalog[typeName]["typeFields"] + sqlValues = ["0", "UTC_TIMESTAMP()"] + valuesList + [startTime, endTime] if len(sqlFields) != len(sqlValues): numRcv = len(valuesList) + 2 numExp = len(self.dbCatalog[typeName]["typeFields"]) @@ -591,7 +579,7 @@ def insertRecordThroughQueue(self, typeName, startTime, endTime, valuesList): % (typeName, TimeUtilities.fromEpoch(startTime), TimeUtilities.fromEpoch(endTime)), ) if typeName not in self.dbCatalog: - return S_ERROR("Type %s has not been defined in the db" % typeName) + return S_ERROR(f"Type {typeName} has not been defined in the db") result = self.__insertInQueueTable(typeName, startTime, endTime, valuesList) if not result["OK"]: return result @@ -602,15 +590,15 @@ def __insertFromINTable(self, recordTuples): """ Do the real insert and delete from the in buffer table """ - self.log.verbose("Received bundle to process", "of %s elements" % len(recordTuples)) + self.log.verbose("Received bundle to process", f"of {len(recordTuples)} elements") for record in recordTuples: iD, typeName, startTime, endTime, valuesList, insertionEpoch = record result = self.insertRecordDirectly(typeName, startTime, endTime, valuesList) if not result["OK"]: - self._update("UPDATE `{}` SET taken=0 WHERE id={}".format(_getTableName("in", typeName), iD)) + self._update(f"UPDATE `{_getTableName('in', typeName)}` SET taken=0 WHERE id={iD}") self.log.error("Can't insert row", result["Message"]) continue - result = self._update("DELETE FROM `{}` WHERE id={}".format(_getTableName("in", typeName), iD)) + result = self._update(f"DELETE FROM `{_getTableName('in', typeName)}` WHERE id={iD}") if not result["OK"]: self.log.error("Can't delete row from the IN table", result["Message"]) @@ -626,14 +614,14 @@ def insertRecordDirectly(self, typeName, startTime, endTime, valuesList): % (typeName, TimeUtilities.fromEpoch(startTime), TimeUtilities.fromEpoch(endTime)), ) if typeName not in self.dbCatalog: - return S_ERROR("Type %s has not been defined in the db" % typeName) + return S_ERROR(f"Type {typeName} has not been defined in the db") # Discover key indexes for keyPos, keyName in enumerate(self.dbCatalog[typeName]["keys"]): keyValue = valuesList[keyPos] retVal = self.__addKeyValue(typeName, keyName, keyValue) if not retVal["OK"]: return retVal - self.log.debug("Value {} for key {} has id {}".format(keyValue, keyName, retVal["Value"])) + self.log.debug(f"Value {keyValue} for key {keyName} has id {retVal['Value']}") valuesList[keyPos] = retVal["Value"] insertList = list(valuesList) insertList.append(startTime) @@ -668,7 +656,7 @@ def deleteRecord(self, typeName, startTime, endTime, valuesList): if self.__readOnly: return S_ERROR("ReadOnly mode enabled. No modification allowed") if typeName not in self.dbCatalog: - return S_ERROR("Type %s has not been defined in the db" % typeName) + return S_ERROR(f"Type {typeName} has not been defined in the db") self.log.info( "Deleting record", @@ -683,7 +671,7 @@ def deleteRecord(self, typeName, startTime, endTime, valuesList): retVal = self.__addKeyValue(typeName, keyName, keyValue) if not retVal["OK"]: return retVal - self.log.verbose("Value {} for key {} has id {}".format(keyValue, keyName, retVal["Value"])) + self.log.verbose(f"Value {keyValue} for key {keyName} has id {retVal['Value']}") sqlValues[keyPos] = retVal["Value"] sqlCond = [] mainTable = _getTableName("type", typeName) @@ -697,11 +685,11 @@ def deleteRecord(self, typeName, startTime, endTime, valuesList): if self.dbCatalog[typeName]["definition"]["values"][vIndex][1].find("FLOAT") > -1: needToRound = True if needToRound: - compVal = ["`{}`.`{}`".format(mainTable, self.dbCatalog[typeName]["typeFields"][i]), "%f" % value] - compVal = ["CEIL( %s * 1000 )" % v for v in compVal] - compVal = "ABS( %s ) <= 1 " % " - ".join(compVal) + compVal = [f"`{mainTable}`.`{self.dbCatalog[typeName]['typeFields'][i]}`", f"{value:f}"] + compVal = [f"CEIL( {v} * 1000 )" for v in compVal] + compVal = f"ABS( {' - '.join(compVal)} ) <= 1 " else: - sqlCond.append("`{}`.`{}`={}".format(mainTable, self.dbCatalog[typeName]["typeFields"][i], value)) + sqlCond.append(f"`{mainTable}`.`{self.dbCatalog[typeName]['typeFields'][i]}`={value}") retVal = self._getConnection() if not retVal["OK"]: return retVal @@ -709,7 +697,7 @@ def deleteRecord(self, typeName, startTime, endTime, valuesList): retVal = self.__startTransaction(connObj) if not retVal["OK"]: return retVal - retVal = self._update("DELETE FROM `{}` WHERE {}".format(mainTable, " AND ".join(sqlCond)), conn=connObj) + retVal = self._update(f"DELETE FROM `{mainTable}` WHERE {' AND '.join(sqlCond)}", conn=connObj) if not retVal["OK"]: return retVal numInsertions = retVal["Value"] @@ -740,7 +728,7 @@ def __splitInBuckets(self, typeName, startTime, endTime, valuesList, connObj=Fal numKeys = len(self.dbCatalog[typeName]["keys"]) keyValues = valuesList[:numKeys] valuesList = valuesList[numKeys:] - self.log.debug("Splitting entry", " in %s buckets" % len(buckets)) + self.log.debug("Splitting entry", f" in {len(buckets)} buckets") return self.__writeBuckets(typeName, buckets, keyValues, valuesList, connObj=connObj) def __deleteFromBuckets(self, typeName, startTime, endTime, valuesList, numInsertions, connObj=False): @@ -753,7 +741,7 @@ def __deleteFromBuckets(self, typeName, startTime, endTime, valuesList, numInser numKeys = len(self.dbCatalog[typeName]["keys"]) keyValues = valuesList[:numKeys] valuesList = valuesList[numKeys:] - self.log.verbose("Deleting bucketed entry", "from %s buckets" % len(buckets)) + self.log.verbose("Deleting bucketed entry", f"from {len(buckets)} buckets") for bucketInfo in buckets: bucketStartTime = bucketInfo[0] bucketProportion = bucketInfo[1] @@ -792,28 +780,9 @@ def __generateSQLConditionForKeys(self, typeName, keyValues): if not retVal["OK"]: return retVal keyValue = retVal["Value"] - realCondList.append("`{}`.`{}` = {}".format(_getTableName("bucket", typeName), keyField, keyValue)) + realCondList.append(f"`{_getTableName('bucket', typeName)}`.`{keyField}` = {keyValue}") return " AND ".join(realCondList) - def __getBucketFromDB(self, typeName, startTime, bucketLength, keyValues, connObj=False): - """ - Get a bucket from the DB - """ - tableName = _getTableName("bucket", typeName) - sqlFields = [] - for valueField in self.dbCatalog[typeName]["values"]: - sqlFields.append(f"`{tableName}`.`{valueField}`") - sqlFields.append("`%s`.`entriesInBucket`" % (tableName)) - cmd = "SELECT {} FROM `{}`".format(", ".join(sqlFields), _getTableName("bucket", typeName)) - cmd += " WHERE `{}`.`startTime`='{}' AND `{}`.`bucketLength`='{}' AND ".format( - tableName, - startTime, - tableName, - bucketLength, - ) - cmd += self.__generateSQLConditionForKeys(typeName, keyValues) - return self._query(cmd, conn=connObj) - def __extractFromBucket( self, typeName, startTime, bucketLength, keyValues, bucketValues, proportion, connObj=False ): @@ -821,7 +790,7 @@ def __extractFromBucket( Update a bucket when coming from the raw insert """ tableName = _getTableName("bucket", typeName) - cmd = "UPDATE `%s` SET " % tableName + cmd = f"UPDATE `{tableName}` SET " sqlValList = [] for pos, valueField in enumerate(self.dbCatalog[typeName]["values"]): value = bucketValues[pos] @@ -847,10 +816,10 @@ def __writeBuckets(self, typeName, buckets, keyValues, valuesList, connObj=False # INSERT PART OF THE QUERY sqlFields = ["`startTime`", "`bucketLength`", "`entriesInBucket`"] for keyPos in range(len(self.dbCatalog[typeName]["keys"])): - sqlFields.append("`%s`" % self.dbCatalog[typeName]["keys"][keyPos]) + sqlFields.append(f"`{self.dbCatalog[typeName]['keys'][keyPos]}`") sqlUpData = ["`entriesInBucket`=`entriesInBucket`+VALUES(`entriesInBucket`)"] for valPos in range(len(self.dbCatalog[typeName]["values"])): - valueField = "`%s`" % self.dbCatalog[typeName]["values"][valPos] + valueField = f"`{self.dbCatalog[typeName]['values'][valPos]}`" sqlFields.append(valueField) sqlUpData.append(f"{valueField}={valueField}+VALUES({valueField})") valuesGroups = [] @@ -864,11 +833,11 @@ def __writeBuckets(self, typeName, buckets, keyValues, valuesList, connObj=False for valPos in range(len(self.dbCatalog[typeName]["values"])): # value = valuesList[ valPos ] sqlValues.append(f"({valuesList[valPos]}*{bProportion})") - valuesGroups.append("( %s )" % ",".join(str(val) for val in sqlValues)) + valuesGroups.append(f"( {','.join(str(val) for val in sqlValues)} )") - cmd = "INSERT INTO `{}` ( {} ) ".format(_getTableName("bucket", typeName), ", ".join(sqlFields)) - cmd += "VALUES %s " % ", ".join(valuesGroups) - cmd += "ON DUPLICATE KEY UPDATE %s" % ", ".join(sqlUpData) + cmd = f"INSERT INTO `{_getTableName('bucket', typeName)}` ( {', '.join(sqlFields)} ) " + cmd += f"VALUES {', '.join(valuesGroups)} " + cmd += f"ON DUPLICATE KEY UPDATE {', '.join(sqlUpData)}" for _i in range(max(1, self.__deadLockRetries)): result = self._update(cmd, conn=connObj) @@ -881,14 +850,14 @@ def __writeBuckets(self, typeName, buckets, keyValues, valuesList, connObj=False if result["OK"]: return result - return S_ERROR("Cannot update bucket: %s" % result["Message"]) + return S_ERROR(f"Cannot update bucket: {result['Message']}") def __checkFieldsExistsInType(self, typeName, fields, tableType): """ Check wether a list of fields exist for a given typeName """ missing = [] - tableFields = self.dbCatalog[typeName]["%sFields" % tableType] + tableFields = self.dbCatalog[typeName][f"{tableType}Fields"] for key in fields: if key not in tableFields: missing.append(key) @@ -897,26 +866,26 @@ def __checkFieldsExistsInType(self, typeName, fields, tableType): def __checkIncomingFieldsForQuery(self, typeName, selectFields, condDict, groupFields, orderFields, tableType): missing = self.__checkFieldsExistsInType(typeName, selectFields[1], tableType) if missing: - return S_ERROR("Value keys %s are not defined" % ", ".join(missing)) + return S_ERROR(f"Value keys {', '.join(missing)} are not defined") missing = self.__checkFieldsExistsInType(typeName, condDict, tableType) if missing: - return S_ERROR("Condition keys %s are not defined" % ", ".join(missing)) + return S_ERROR(f"Condition keys {', '.join(missing)} are not defined") if groupFields: missing = self.__checkFieldsExistsInType(typeName, groupFields[1], tableType) if missing: - return S_ERROR("Group fields %s are not defined" % ", ".join(missing)) + return S_ERROR(f"Group fields {', '.join(missing)} are not defined") if orderFields: missing = self.__checkFieldsExistsInType(typeName, orderFields[1], tableType) if missing: - return S_ERROR("Order fields %s are not defined" % ", ".join(missing)) + return S_ERROR(f"Order fields {', '.join(missing)} are not defined") return S_OK() - def retrieveRawRecords(self, typeName, startTime, endTime, condDict, orderFields, connObj=False): + def retrieveRawRecords(self, typeName, startTime, endTime, condDict, orderFields): """ Get RAW data from the DB """ if typeName not in self.dbCatalog: - return S_ERROR("Type %s not defined" % typeName) + return S_ERROR(f"Type {typeName} not defined") selectFields = [["%s", "%s"], ["startTime", "endTime"]] for tK in ("keys", "values"): for key in self.dbCatalog[typeName][tK]: @@ -947,8 +916,7 @@ def retrieveBucketedData( """ if typeName not in self.dbCatalog: - return S_ERROR("Type %s is not defined" % typeName) - startQueryEpoch = time.time() + return S_ERROR(f"Type {typeName} is not defined") if len(selectFields) < 2: return S_ERROR("selectFields has to be a list containing a string and a list of fields") retVal = self.__checkIncomingFieldsForQuery( @@ -990,32 +958,32 @@ def __queryType( if "bucketLength" in selectFields[1]: groupFields = list(groupFields) - groupFields[0] = "{}, {}".format(groupFields[0], "%s") + groupFields[0] = f"{groupFields[0]}, %s" groupFields[1].append("bucketlength") groupFields = tuple(groupFields) except TypeError as e: - return S_ERROR("Cannot format properly group string: %s" % repr(e)) + return S_ERROR(f"Cannot format properly group string: {repr(e)}") if orderFields: try: orderFields[0] % tuple(orderFields[1]) except TypeError as e: - return S_ERROR("Cannot format properly order string: %s" % repr(e)) + return S_ERROR(f"Cannot format properly order string: {repr(e)}") # Calculate fields to retrieve realFieldList = [] for rawFieldName in selectFields[1]: keyTable = _getTableName("key", typeName, rawFieldName) if rawFieldName in self.dbCatalog[typeName]["keys"]: - realFieldList.append("`%s`.`value`" % keyTable) + realFieldList.append(f"`{keyTable}`.`value`") List.appendUnique(sqlLinkList, f"`{tableName}`.`{rawFieldName}` = `{keyTable}`.`id`") else: realFieldList.append(f"`{tableName}`.`{rawFieldName}`") try: - cmd += " %s" % selectFields[0] % tuple(realFieldList) + cmd += f" {selectFields[0]}" % tuple(realFieldList) except TypeError as e: - return S_ERROR("Error generating select fields string: %s" % repr(e)) + return S_ERROR(f"Error generating select fields string: {repr(e)}") # Calculate tables needed - sqlFromList = ["`%s`" % tableName] + sqlFromList = [f"`{tableName}`"] for key in self.dbCatalog[typeName]["keys"]: if ( key in condDict @@ -1023,8 +991,8 @@ def __queryType( or (groupFields and key in groupFields[1]) or (orderFields and key in orderFields[1]) ): - sqlFromList.append("`%s`" % _getTableName("key", typeName, key)) - cmd += " FROM %s" % ", ".join(sqlFromList) + sqlFromList.append(f"`{_getTableName('key', typeName, key)}`") + cmd += f" FROM {', '.join(sqlFromList)}" # Calculate time conditions sqlTimeCond = [] if startTime: @@ -1041,7 +1009,7 @@ def __queryType( else: endTimeSQLVar = "endTime" sqlTimeCond.append(f"`{tableName}`.`{endTimeSQLVar}` <= {endTime}") - cmd += " WHERE %s" % " AND ".join(sqlTimeCond) + cmd += f" WHERE {' AND '.join(sqlTimeCond)}" # Calculate conditions sqlCondList = [] for keyName in condDict: @@ -1049,7 +1017,7 @@ def __queryType( if keyName in self.dbCatalog[typeName]["keys"]: List.appendUnique( sqlLinkList, - "`{}`.`{}` = `{}`.`id`".format(tableName, keyName, _getTableName("key", typeName, keyName)), + f"`{tableName}`.`{keyName}` = `{_getTableName('key', typeName, keyName)}`.`id`", ) if not isinstance(condDict[keyName], (list, tuple)): condDict[keyName] = [condDict[keyName]] @@ -1059,12 +1027,13 @@ def __queryType( return retVal keyValue = retVal["Value"] if keyName in self.dbCatalog[typeName]["keys"]: - sqlORList.append("`{}`.`value` = {}".format(_getTableName("key", typeName, keyName), keyValue)) + sqlORList.append(f"`{_getTableName('key', typeName, keyName)}`.`value` = {keyValue}") else: sqlORList.append(f"`{tableName}`.`{keyName}` = {keyValue}") - sqlCondList.append("( %s )" % " OR ".join(sqlORList)) + if sqlORList: + sqlCondList.append(f"( {' OR '.join(sqlORList)} )") if sqlCondList: - cmd += " AND %s" % " AND ".join(sqlCondList) + cmd += f" AND {' AND '.join(sqlCondList)}" # Calculate grouping and sorting for preGenFields in (groupFields, orderFields): if preGenFields: @@ -1072,11 +1041,11 @@ def __queryType( if field in self.dbCatalog[typeName]["keys"]: List.appendUnique( sqlLinkList, - "`{}`.`{}` = `{}`.`id`".format(tableName, field, _getTableName("key", typeName, field)), + f"`{tableName}`.`{field}` = `{_getTableName('key', typeName, field)}`.`id`", ) if preGenFields[0] != "%s": # The default grouping was changed - preGenFields[1][i] = "`%s`.Value" % _getTableName("key", typeName, field) + preGenFields[1][i] = f"`{_getTableName('key', typeName, field)}`.Value" else: # The default grouping is maintained preGenFields[1][i] = f"`{tableName}`.`{field}`" @@ -1084,10 +1053,10 @@ def __queryType( preGenFields[1][i] = f"`{tableName}`.`{field}`" if sqlLinkList: - cmd += " AND %s" % " AND ".join(sqlLinkList) + cmd += f" AND {' AND '.join(sqlLinkList)}" if groupFields: if len(groupFields[1]) == 1: - testGroupFields = " %s" % selectFields[0] % tuple(realFieldList) + testGroupFields = f" {selectFields[0]}" % tuple(realFieldList) testGroupFieldsList = testGroupFields.split(",") realGroupFields = () for testGroupFields in testGroupFieldsList: @@ -1095,10 +1064,9 @@ def __queryType( realGroupFields += (testGroupFields.strip(),) cmd += " GROUP BY " + ",".join(realGroupFields) else: - cmd += " GROUP BY %s" % (groupFields[0] % tuple(groupFields[1])) + cmd += f" GROUP BY {groupFields[0] % tuple(groupFields[1])}" if orderFields: - cmd += " ORDER BY %s" % (orderFields[0] % tuple(orderFields[1])) - self.log.verbose(cmd) + cmd += f" ORDER BY {orderFields[0] % tuple(orderFields[1])}" return self._query(cmd, conn=connObj) def compactBuckets(self, typeFilter=False): @@ -1117,12 +1085,12 @@ def compactBuckets(self, typeFilter=False): slow = True for typeName in self.dbCatalog: if typeFilter and typeName.find(typeFilter) == -1: - self.log.info("[COMPACT] Skipping %s" % typeName) + self.log.info(f"[COMPACT] Skipping {typeName}") continue if self.dbCatalog[typeName]["dataTimespan"] > 0: - self.log.info("[COMPACT] Deleting records older that timespan for type %s" % typeName) + self.log.info(f"[COMPACT] Deleting records older that timespan for type {typeName}") self.__deleteRecordsOlderThanDataTimespan(typeName) - self.log.info("[COMPACT] Compacting %s" % typeName) + self.log.info(f"[COMPACT] Compacting {typeName}") if slow: self.__slowCompactBucketsForType(typeName) else: @@ -1148,18 +1116,18 @@ def __selectForCompactBuckets(self, typeName, timeLimit, bucketLength, nextBucke sqlSelectList.append(f"`{tableName}`.`{field}`") for field in self.dbCatalog[typeName]["values"]: sqlSelectList.append(f"SUM( `{tableName}`.`{field}` )") - sqlSelectList.append("SUM( `%s`.`entriesInBucket` )" % (tableName)) - sqlSelectList.append("MIN( `%s`.`startTime` )" % tableName) - sqlSelectList.append("MAX( `%s`.`startTime` )" % tableName) + sqlSelectList.append(f"SUM( `{tableName}`.`entriesInBucket` )") + sqlSelectList.append(f"MIN( `{tableName}`.`startTime` )") + sqlSelectList.append(f"MAX( `{tableName}`.`startTime` )") selectSQL += ", ".join(sqlSelectList) - selectSQL += " FROM `%s`" % tableName + selectSQL += f" FROM `{tableName}`" selectSQL += f" WHERE `{tableName}`.`startTime` < '{timeLimit}' AND" selectSQL += f" `{tableName}`.`bucketLength` = {bucketLength}" # MAGIC bucketing - sqlGroupList = [_bucketizeDataField("`%s`.`startTime`" % tableName, nextBucketLength)] + sqlGroupList = [_bucketizeDataField(f"`{tableName}`.`startTime`", nextBucketLength)] for field in self.dbCatalog[typeName]["keys"]: sqlGroupList.append(f"`{tableName}`.`{field}`") - selectSQL += " GROUP BY %s" % ", ".join(sqlGroupList) + selectSQL += f" GROUP BY {', '.join(sqlGroupList)}" return self._query(selectSQL, conn=connObj) def __deleteForCompactBuckets(self, typeName, timeLimit, bucketLength, connObj=False): @@ -1167,7 +1135,7 @@ def __deleteForCompactBuckets(self, typeName, timeLimit, bucketLength, connObj=F Delete compacted buckets """ tableName = _getTableName("bucket", typeName) - deleteSQL = "DELETE FROM `%s` WHERE " % tableName + deleteSQL = f"DELETE FROM `{tableName}` WHERE " deleteSQL += f"`{tableName}`.`startTime` < '{timeLimit}' AND " deleteSQL += f"`{tableName}`.`bucketLength` = {bucketLength}" return self._update(deleteSQL, conn=connObj) @@ -1196,7 +1164,7 @@ def __compactBucketsForType(self, typeName): # self.__rollbackTransaction(connObj) return retVal bucketsData = retVal["Value"] - self.log.info("[COMPACT] Got %d records to compact" % len(bucketsData)) + self.log.info(f"[COMPACT] Got {len(bucketsData)} records to compact") if len(bucketsData) == 0: continue retVal = self.__deleteForCompactBuckets(typeName, timeLimit, bucketLength) @@ -1212,9 +1180,7 @@ def __compactBucketsForType(self, typeName): retVal = self.__splitInBuckets(typeName, startTime, endTime, valuesList) if not retVal["OK"]: # self.__rollbackTransaction( connObj ) - self.log.error( - "[COMPACT] Error while compacting data for record", "{}: {}".format(typeName, retVal["Value"]) - ) + self.log.error("[COMPACT] Error while compacting data for record", f"{typeName}: {retVal['Value']}") self.log.info("[COMPACT] Finished compaction %d of %d" % (bPos, len(self.dbBucketsLength[typeName]) - 1)) # return self.__commitTransaction( connObj ) return S_OK() @@ -1276,7 +1242,7 @@ def __slowCompactBucketsForType(self, typeName): if not retVal["OK"]: self.log.error( "[COMPACT] Error while compacting data for buckets", - "{}: {}".format(typeName, retVal["Value"]), + f"{typeName}: {retVal['Value']}", ) totalCompacted += len(bucketsData) insertElapsedTime = time.time() - deleteEndTime @@ -1299,11 +1265,11 @@ def __selectIndividualForCompactBuckets(self, typeName, timeLimit, bucketLength, sqlSelectList.append(f"`{tableName}`.`{field}`") for field in self.dbCatalog[typeName]["values"]: sqlSelectList.append(f"`{tableName}`.`{field}`") - sqlSelectList.append("`%s`.`entriesInBucket`" % (tableName)) - sqlSelectList.append("`%s`.`startTime`" % tableName) - sqlSelectList.append("`%s`.bucketLength" % (tableName)) + sqlSelectList.append(f"`{tableName}`.`entriesInBucket`") + sqlSelectList.append(f"`{tableName}`.`startTime`") + sqlSelectList.append(f"`{tableName}`.bucketLength") selectSQL += ", ".join(sqlSelectList) - selectSQL += " FROM `%s`" % tableName + selectSQL += f" FROM `{tableName}`" selectSQL += f" WHERE `{tableName}`.`startTime` < '{timeLimit}' AND" selectSQL += f" `{tableName}`.`bucketLength` = {bucketLength}" # MAGIC bucketing @@ -1326,8 +1292,8 @@ def __deleteIndividualForCompactBuckets(self, typeName, bucketsData, connObj=Fal condSQL.append(f"`{tableName}`.`{field}` = {record[iPos]}") condSQL.append("`%s`.`startTime` = %d" % (tableName, record[-2])) condSQL.append("`%s`.`bucketLength` = %d" % (tableName, record[-1])) - delCondsSQL.append("(%s)" % " AND ".join(condSQL)) - delSQL = "DELETE FROM `{}` WHERE {}".format(tableName, " OR ".join(delCondsSQL)) + delCondsSQL.append(f"({' AND '.join(condSQL)})") + delSQL = f"DELETE FROM `{tableName}` WHERE {' OR '.join(delCondsSQL)}" result = self._update(delSQL, conn=connObj) if not result["OK"]: self.log.error("Cannot delete individual records for compaction", result["Message"]) @@ -1347,7 +1313,7 @@ def __deleteRecordsOlderThanDataTimespan(self, typeName): (_getTableName("type", typeName), "endTime"), (_getTableName("bucket", typeName), "startTime"), ): - self.log.info("[COMPACT] Deleting old records for table %s" % table) + self.log.info(f"[COMPACT] Deleting old records for table {table}") deleteLimit = 100000 deleted = deleteLimit while deleted >= deleteLimit: @@ -1361,7 +1327,7 @@ def __deleteRecordsOlderThanDataTimespan(self, typeName): if not result["OK"]: self.log.error( "[COMPACT] Cannot delete old records", - "Table: {} Timespan: {} Error: {}".format(table, dataTimespan, result["Message"]), + f"Table: {table} Timespan: {dataTimespan} Error: {result['Message']}", ) break self.log.info("[COMPACT] Deleted %d records for %s table" % (result["Value"], table)) @@ -1373,21 +1339,21 @@ def regenerateBuckets(self, typeName): return S_ERROR("ReadOnly mode enabled. No modification allowed") # Delete old entries if any if self.dbCatalog[typeName]["dataTimespan"] > 0: - self.log.info("[REBUCKET] Deleting records older that timespan for type %s" % typeName) + self.log.info(f"[REBUCKET] Deleting records older that timespan for type {typeName}") self.__deleteRecordsOlderThanDataTimespan(typeName) self.log.info("[REBUCKET] Done deleting old records") rawTableName = _getTableName("type", typeName) # retVal = self.__startTransaction(connObj) # if not retVal[ 'OK' ]: # return retVal - self.log.info("[REBUCKET] Deleting buckets for %s" % typeName) - retVal = self._update("DELETE FROM `%s`" % _getTableName("bucket", typeName)) + self.log.info(f"[REBUCKET] Deleting buckets for {typeName}") + retVal = self._update(f"DELETE FROM `{_getTableName('bucket', typeName)}`") if not retVal["OK"]: return retVal # Generate the common part of the query # SELECT fields - startTimeTableField = "`%s`.startTime" % rawTableName - endTimeTableField = "`%s`.endTime" % rawTableName + startTimeTableField = f"`{rawTableName}`.startTime" + endTimeTableField = f"`{rawTableName}`.endTime" # Select strings and sum select strings sqlSUMSelectList = [] sqlSelectList = [] @@ -1407,7 +1373,7 @@ def regenerateBuckets(self, typeName): # List to contain all queries sqlQueries = [] dateInclusiveConditions = [] - countedField = "`{}`.`{}`".format(rawTableName, self.dbCatalog[typeName]["keys"][0]) + countedField = f"`{rawTableName}`.`{self.dbCatalog[typeName]['keys'][0]}`" lastTime = TimeUtilities.toEpoch() # Iterate for all ranges for iRange, iValue in enumerate(self.dbBucketsLength[typeName]): @@ -1453,7 +1419,7 @@ def regenerateBuckets(self, typeName): sameBucketCondition, ) sqlQueries.append(sqlQuery) - dateInclusiveConditions.append("( %s )" % whereString) + dateInclusiveConditions.append(f"( {whereString} )") # Query for records that are in between two ranges sqlQuery = "SELECT {}, {}, {}, 1 FROM `{}` WHERE NOT {}".format( startTimeTableField, @@ -1463,10 +1429,10 @@ def regenerateBuckets(self, typeName): " AND NOT ".join(dateInclusiveConditions), ) sqlQueries.append(sqlQuery) - self.log.info("[REBUCKET] Retrieving data for rebuilding buckets for type %s..." % (typeName)) + self.log.info(f"[REBUCKET] Retrieving data for rebuilding buckets for type {typeName}...") queryNum = 0 for sqlQuery in sqlQueries: - self.log.info("[REBUCKET] Executing query #%s..." % queryNum) + self.log.info(f"[REBUCKET] Executing query #{queryNum}...") queryNum += 1 retVal = self._query(sqlQuery) if not retVal["OK"]: @@ -1474,7 +1440,7 @@ def regenerateBuckets(self, typeName): # self.__rollbackTransaction(connObj) return retVal rawData = retVal["Value"] - self.log.info("[REBUCKET] Retrieved %s records" % len(rawData)) + self.log.info(f"[REBUCKET] Retrieved {len(rawData)} records") rebucketedRecords = 0 startQuery = time.time() startBlock = time.time() diff --git a/src/DIRAC/AccountingSystem/DB/MultiAccountingDB.py b/src/DIRAC/AccountingSystem/DB/MultiAccountingDB.py index 8b94ad8c72d..17636916703 100644 --- a/src/DIRAC/AccountingSystem/DB/MultiAccountingDB.py +++ b/src/DIRAC/AccountingSystem/DB/MultiAccountingDB.py @@ -1,8 +1,8 @@ """ Module for handling AccountingDB tables on multiple DBs (e.g. 2 MySQL servers) """ from DIRAC import gConfig, S_OK, gLogger -from DIRAC.Core.Utilities.Plotting.TypeLoader import TypeLoader from DIRAC.AccountingSystem.DB.AccountingDB import AccountingDB +from DIRAC.Core.Utilities.Plotting.TypeLoader import TypeLoader class MultiAccountingDB: @@ -18,10 +18,9 @@ def __init__(self, csPath, readOnly=False): def __generateDBs(self): self.__log.notice("Creating default AccountingDB...") self.__allDBs = {self.__defaultDB: AccountingDB(readOnly=self.__readOnly)} - types = self.__allDBs[self.__defaultDB].getRegisteredTypes() result = gConfig.getOptionsDict(self.__csPath) if not result["OK"]: - gLogger.verbose("No extra databases defined", "in %s" % self.__csPath) + gLogger.verbose("No extra databases defined", f"in {self.__csPath}") return validTypes = TypeLoader().getTypes() opts = result["Value"] @@ -35,8 +34,8 @@ def __generateDBs(self): if dbName not in self.__allDBs: fields = dbName.split("/") if len(fields) == 1: - dbName = "Accounting/%s" % dbName - gLogger.notice("Creating DB", "%s" % dbName) + dbName = f"Accounting/{dbName}" + gLogger.notice("Creating DB", dbName) self.__allDBs[dbName] = AccountingDB(dbName, readOnly=self.__readOnly) self.__dbByType[acType] = dbName @@ -69,13 +68,13 @@ def __registerMethods(self): ): (lambda closure: setattr(self, closure, lambda *x: self.__mimeMethod(closure, *x)))(methodName) - def __mimeTypeMethod(self, methodName, setup, acType, *args): - return getattr(self.__db(acType), methodName)(f"{setup}_{acType}", *args) + def __mimeTypeMethod(self, methodName, acType, *args): + return getattr(self.__db(acType), methodName)(acType, *args) def __mimeMethod(self, methodName, *args): end = S_OK() - for dbName in self.__allDBs: - res = getattr(self.__allDBs[dbName], methodName)(*args) + for DB in self.__allDBs.values(): + res = getattr(DB, methodName)(*args) if res and not res["OK"]: end = res return end @@ -89,10 +88,10 @@ def insertRecordBundleThroughQueue(self, records): acType = record[1] if acType not in recByType: recByType[acType] = [] - recByType[acType].append((f"{record[0]}_{record[1]}", record[2], record[3], record[4])) + recByType[acType].append((f"{record[0]}", record[1], record[2], record[3])) end = S_OK() - for acType in recByType: - res = self.__db(acType).insertRecordBundleThroughQueue(recByType[acType]) + for acType, records in recByType.items(): + res = self.__db(acType).insertRecordBundleThroughQueue(records) if not res["OK"]: end = res return end diff --git a/src/DIRAC/AccountingSystem/DB/test/Test_AccountingDB.py b/src/DIRAC/AccountingSystem/DB/test/Test_AccountingDB.py index 023ad56de59..ee3831e46e2 100644 --- a/src/DIRAC/AccountingSystem/DB/test/Test_AccountingDB.py +++ b/src/DIRAC/AccountingSystem/DB/test/Test_AccountingDB.py @@ -13,7 +13,6 @@ class TestCase(unittest.TestCase): """Base class for the AccountingDB test cases""" def setUp(self): - self.moduleTested = moduleTested self.testClass = self.moduleTested.AccountingDB @@ -70,7 +69,7 @@ def test_calculateBuckets(self): module = self.testClass() module.dbCatalog = { - "LHCb-Certification_DataOperation": { + "DataOperation": { "definition": { "keys": [ ("OperationType", "VARCHAR(32)"), @@ -142,23 +141,23 @@ def test_calculateBuckets(self): } } - module.dbBucketsLength["LHCb-Certification_DataOperation"] = [ + module.dbBucketsLength["DataOperation"] = [ (259200, 900), (691200, 3600), (15552000, 86400), (31104000, 604800), ] - retVal = module.calculateBuckets("LHCb-Certification_DataOperation", 1495328400, 1495328400)[0][0] + retVal = module.calculateBuckets("DataOperation", 1495328400, 1495328400)[0][0] self.assertTrue(retVal) - retVal = module.calculateBuckets("LHCb-Certification_DataOperation", 1497964315, 1497964315)[0][0] + retVal = module.calculateBuckets("DataOperation", 1497964315, 1497964315)[0][0] self.assertTrue(retVal) self.assertTrue(retVal) - retVal = module.calculateBuckets("LHCb-Certification_DataOperation", 1495414800, 1495414800)[0][0] + retVal = module.calculateBuckets("DataOperation", 1495414800, 1495414800)[0][0] self.assertTrue(retVal) - retVal = module.calculateBuckets("LHCb-Certification_DataOperation", 1498047038, 1498047038)[0][0] + retVal = module.calculateBuckets("DataOperation", 1498047038, 1498047038)[0][0] self.assertTrue(retVal) def test__queryType1(self): @@ -167,7 +166,7 @@ def test__queryType1(self): """ module = self.testClass() module.dbCatalog = { - "LHCb-Certification_DataOperation": { + "DataOperation": { "definition": { "keys": [ ("OperationType", "VARCHAR(32)"), @@ -239,7 +238,7 @@ def test__queryType1(self): } } - module.dbBucketsLength["LHCb-Certification_DataOperation"] = [ + module.dbBucketsLength["DataOperation"] = [ (259200, 900), (691200, 3600), (15552000, 86400), @@ -248,32 +247,32 @@ def test__queryType1(self): module._query = self.query startTime = 1495324800 endTime = 1497960715 - retVal = module.calculateBuckets("LHCb-Certification_DataOperation", startTime + 3600, startTime + 3600)[0][0] + retVal = module.calculateBuckets("DataOperation", startTime + 3600, startTime + 3600)[0][0] self.assertTrue(retVal) expectedStartTime = retVal - retVal = module.calculateBuckets("LHCb-Certification_DataOperation", endTime + 3600, endTime + 3600)[0][0] + retVal = module.calculateBuckets("DataOperation", endTime + 3600, endTime + 3600)[0][0] self.assertTrue(retVal) expectedEndTime = retVal expectedQuery = ( - "SELECT `ac_key_LHCb-Certification_DataOperation_Source`.`value`, \ -`ac_bucket_LHCb-Certification_DataOperation`.`startTime`, \ -`ac_bucket_LHCb-Certification_DataOperation`.`bucketLength`, \ -SUM(`ac_bucket_LHCb-Certification_DataOperation`.`TransferOK`), \ -SUM(`ac_bucket_LHCb-Certification_DataOperation`.`TransferTotal`)-\ -SUM(`ac_bucket_LHCb-Certification_DataOperation`.`TransferOK`) \ -FROM `ac_bucket_LHCb-Certification_DataOperation`, \ -`ac_key_LHCb-Certification_DataOperation_Source` \ -WHERE `ac_bucket_LHCb-Certification_DataOperation`.`startTime` >= %s \ -AND `ac_bucket_LHCb-Certification_DataOperation`.`startTime` <= %s \ -AND `ac_bucket_LHCb-Certification_DataOperation`.`Source` = `ac_key_LHCb-Certification_DataOperation_Source`.`id` \ -GROUP BY startTime, `ac_key_LHCb-Certification_DataOperation_Source`.Value, bucketlength \ + "SELECT `ac_key_DataOperation_Source`.`value`, \ +`ac_bucket_DataOperation`.`startTime`, \ +`ac_bucket_DataOperation`.`bucketLength`, \ +SUM(`ac_bucket_DataOperation`.`TransferOK`), \ +SUM(`ac_bucket_DataOperation`.`TransferTotal`)-\ +SUM(`ac_bucket_DataOperation`.`TransferOK`) \ +FROM `ac_bucket_DataOperation`, \ +`ac_key_DataOperation_Source` \ +WHERE `ac_bucket_DataOperation`.`startTime` >= %s \ +AND `ac_bucket_DataOperation`.`startTime` <= %s \ +AND `ac_bucket_DataOperation`.`Source` = `ac_key_DataOperation_Source`.`id` \ +GROUP BY startTime, `ac_key_DataOperation_Source`.Value, bucketlength \ ORDER BY startTime" % (expectedStartTime, expectedEndTime) ) retVal = module._AccountingDB__queryType( - "LHCb-Certification_DataOperation", # pylint: disable=no-member + "DataOperation", # pylint: disable=no-member startTime, endTime, ( @@ -293,7 +292,7 @@ def test_queryType2(self): """Test the query creation for a given condition""" module = self.testClass() module.dbCatalog = { - "LHCb-Certification_DataOperation": { + "DataOperation": { "definition": { "keys": [ ("OperationType", "VARCHAR(32)"), @@ -365,7 +364,7 @@ def test_queryType2(self): } } - module.dbBucketsLength["LHCb-Certification_DataOperation"] = [ + module.dbBucketsLength["DataOperation"] = [ (259200, 900), (691200, 3600), (15552000, 86400), @@ -374,33 +373,33 @@ def test_queryType2(self): module._query = self.query startTime = 1495411200 endTime = 1498043438 - retVal = module.calculateBuckets("LHCb-Certification_DataOperation", startTime + 3600, startTime + 3600)[0][0] + retVal = module.calculateBuckets("DataOperation", startTime + 3600, startTime + 3600)[0][0] self.assertTrue(retVal) expectedStartTime = retVal - retVal = module.calculateBuckets("LHCb-Certification_DataOperation", endTime + 3600, endTime + 3600)[0][0] + retVal = module.calculateBuckets("DataOperation", endTime + 3600, endTime + 3600)[0][0] self.assertTrue(retVal) expectedEndTime = retVal expectedQuery = ( - "SELECT `ac_key_LHCb-Certification_DataOperation_Source`.`value`, \ -`ac_bucket_LHCb-Certification_DataOperation`.`startTime`, \ -`ac_bucket_LHCb-Certification_DataOperation`.`bucketLength`, \ -SUM(`ac_bucket_LHCb-Certification_DataOperation`.`TransferOK`), \ -SUM(`ac_bucket_LHCb-Certification_DataOperation`.`TransferTotal`)-\ -SUM(`ac_bucket_LHCb-Certification_DataOperation`.`TransferOK`) \ -FROM `ac_bucket_LHCb-Certification_DataOperation`, \ -`ac_key_LHCb-Certification_DataOperation_Source` \ -WHERE `ac_bucket_LHCb-Certification_DataOperation`.`startTime` >= %s \ -AND `ac_bucket_LHCb-Certification_DataOperation`.`startTime` <= %s \ -AND `ac_bucket_LHCb-Certification_DataOperation`.`Source` = `ac_key_LHCb-Certification_DataOperation_Source`.`id` \ -GROUP BY startTime, `ac_key_LHCb-Certification_DataOperation_Source`.Value, bucketlength \ + "SELECT `ac_key_DataOperation_Source`.`value`, \ +`ac_bucket_DataOperation`.`startTime`, \ +`ac_bucket_DataOperation`.`bucketLength`, \ +SUM(`ac_bucket_DataOperation`.`TransferOK`), \ +SUM(`ac_bucket_DataOperation`.`TransferTotal`)-\ +SUM(`ac_bucket_DataOperation`.`TransferOK`) \ +FROM `ac_bucket_DataOperation`, \ +`ac_key_DataOperation_Source` \ +WHERE `ac_bucket_DataOperation`.`startTime` >= %s \ +AND `ac_bucket_DataOperation`.`startTime` <= %s \ +AND `ac_bucket_DataOperation`.`Source` = `ac_key_DataOperation_Source`.`id` \ +GROUP BY startTime, `ac_key_DataOperation_Source`.Value, bucketlength \ ORDER BY startTime" % (expectedStartTime, expectedEndTime) ) retVal = module._AccountingDB__queryType( - "LHCb-Certification_DataOperation", # pylint: disable=no-member + "DataOperation", # pylint: disable=no-member startTime, endTime, ( diff --git a/src/DIRAC/AccountingSystem/Service/DataStoreHandler.py b/src/DIRAC/AccountingSystem/Service/DataStoreHandler.py index 67864dc3d1b..cb468ab25c4 100644 --- a/src/DIRAC/AccountingSystem/Service/DataStoreHandler.py +++ b/src/DIRAC/AccountingSystem/Service/DataStoreHandler.py @@ -1,6 +1,6 @@ """ DataStore is the service for inserting accounting reports (rows) in the Accounting DB - This service CAN be duplicated iff the first is a "master" and all the others are slaves. + This service CAN be duplicated iff the first is a "controller" and all the others are workers. See the information about :ref:`datastorehelpers`. .. literalinclude:: ../ConfigTemplate.cfg @@ -11,13 +11,13 @@ """ import datetime -from DIRAC import S_OK, S_ERROR, gConfig +from DIRAC import S_ERROR, S_OK from DIRAC.AccountingSystem.DB.MultiAccountingDB import MultiAccountingDB from DIRAC.ConfigurationSystem.Client import PathFinder +from DIRAC.Core.Base.Client import Client from DIRAC.Core.DISET.RequestHandler import RequestHandler, getServiceOption from DIRAC.Core.Utilities import TimeUtilities from DIRAC.Core.Utilities.ThreadScheduler import gThreadScheduler -from DIRAC.Core.Base.Client import Client class DataStoreHandler(RequestHandler): @@ -32,11 +32,11 @@ def initializeHandler(cls, svcInfoDict): # we can run multiple services in read only mode. In that case we do not bucket cls.runBucketing = getServiceOption(svcInfoDict, "RunBucketing", True) if cls.runBucketing: - cls.__acDB.autoCompactDB() # pylint: disable=no-member - result = cls.__acDB.markAllPendingRecordsAsNotTaken() # pylint: disable=no-member + cls.__acDB.autoCompactDB() + result = cls.__acDB.markAllPendingRecordsAsNotTaken() if not result["OK"]: return result - gThreadScheduler.addPeriodicTask(60, cls.__acDB.loadPendingRecords) # pylint: disable=no-member + gThreadScheduler.addPeriodicTask(60, cls.__acDB.loadPendingRecords) return S_OK() types_registerType = [str, list, list, list] @@ -44,87 +44,32 @@ def initializeHandler(cls, svcInfoDict): def export_registerType(self, typeName, definitionKeyFields, definitionAccountingFields, bucketsLength): """ Register a new type. (Only for all powerful admins) - (Bow before me for I am admin! :) - """ - retVal = gConfig.getSections("/DIRAC/Setups") - if not retVal["OK"]: - return retVal - errorsList = [] - for setup in retVal["Value"]: - retVal = self.__acDB.registerType( # pylint: disable=no-member - setup, typeName, definitionKeyFields, definitionAccountingFields, bucketsLength - ) - if not retVal["OK"]: - errorsList.append(retVal["Message"]) - if errorsList: - return S_ERROR("Error while registering type:\n %s" % "\n ".join(errorsList)) - return S_OK() + """ + return self.__acDB.registerType(typeName, definitionKeyFields, definitionAccountingFields, bucketsLength) types_setBucketsLength = [str, list] def export_setBucketsLength(self, typeName, bucketsLength): """ Change the buckets Length. (Only for all powerful admins) - (Bow before me for I am admin! :) - """ - retVal = gConfig.getSections("/DIRAC/Setups") - if not retVal["OK"]: - return retVal - errorsList = [] - for setup in retVal["Value"]: - retVal = self.__acDB.changeBucketsLength(setup, typeName, bucketsLength) # pylint: disable=no-member - if not retVal["OK"]: - errorsList.append(retVal["Message"]) - if errorsList: - return S_ERROR("Error while changing bucketsLength type:\n %s" % "\n ".join(errorsList)) - return S_OK() + """ + return self.__acDB.changeBucketsLength(typeName, bucketsLength) types_regenerateBuckets = [str] def export_regenerateBuckets(self, typeName): """ Recalculate buckets. (Only for all powerful admins) - (Bow before me for I am admin! :) - """ - retVal = gConfig.getSections("/DIRAC/Setups") - if not retVal["OK"]: - return retVal - errorsList = [] - for setup in retVal["Value"]: - retVal = self.__acDB.regenerateBuckets(setup, typeName) # pylint: disable=no-member - if not retVal["OK"]: - errorsList.append(retVal["Message"]) - if errorsList: - return S_ERROR("Error while recalculating buckets for type:\n %s" % "\n ".join(errorsList)) - return S_OK() + """ + return self.__acDB.regenerateBuckets(typeName) types_getRegisteredTypes = [] def export_getRegisteredTypes(self): """ Get a list of registered types (Only for all powerful admins) - (Bow before me for I am admin! :) """ - return self.__acDB.getRegisteredTypes() # pylint: disable=no-member - - types_deleteType = [str] - - def export_deleteType(self, typeName): - """ - Delete accounting type and ALL its contents. VERY DANGEROUS! (Only for all powerful admins) - (Bow before me for I am admin! :) - """ - retVal = gConfig.getSections("/DIRAC/Setups") - if not retVal["OK"]: - return retVal - errorsList = [] - for setup in retVal["Value"]: - retVal = self.__acDB.deleteType(setup, typeName) # pylint: disable=too-many-function-args,no-member - if not retVal["OK"]: - errorsList.append(retVal["Message"]) - if errorsList: - return S_ERROR("Error while deleting type:\n %s" % "\n ".join(errorsList)) - return S_OK() + return self.__acDB.getRegisteredTypes() types_commit = [str, datetime.datetime, datetime.datetime, list] @@ -132,12 +77,9 @@ def export_commit(self, typeName, startTime, endTime, valuesList): """ Add a record for a type """ - setup = self.serviceInfoDict["clientSetup"] startTime = int(TimeUtilities.toEpoch(startTime)) endTime = int(TimeUtilities.toEpoch(endTime)) - return self.__acDB.insertRecordThroughQueue( # pylint: disable=no-member - setup, typeName, startTime, endTime, valuesList - ) + return self.__acDB.insertRecordThroughQueue(typeName, startTime, endTime, valuesList) types_commitRegisters = [list] @@ -145,7 +87,6 @@ def export_commitRegisters(self, entriesList): """ Add a record for a type """ - setup = self.serviceInfoDict["clientSetup"] expectedTypes = [str, datetime.datetime, datetime.datetime, list] for entry in entriesList: if len(entry) != 4: @@ -162,7 +103,7 @@ def export_commitRegisters(self, entriesList): startTime = int(TimeUtilities.toEpoch(entry[1])) endTime = int(TimeUtilities.toEpoch(entry[2])) self.log.debug("inserting", entry) - records.append((setup, entry[0], startTime, endTime, entry[3])) + records.append((entry[0], startTime, endTime, entry[3])) return self.__acDB.insertRecordBundleThroughQueue(records) types_compactDB = [] @@ -171,11 +112,11 @@ def export_compactDB(self): """ Compact the db by grouping buckets """ - # if we are running slaves (not only one service) we can redirect the request to the master + # if we are running workers (not only one service) we can redirect the request to the master # For more information please read the Administrative guide Accounting part! # ADVICE: If you want to trigger the bucketing, please make sure the bucketing is not running!!!! if self.runBucketing: - return self.__acDB.compactBuckets() # pylint: disable=no-member + return self.__acDB.compactBuckets() return Client(url="Accounting/DataStoreMaster").compactDB() @@ -185,10 +126,9 @@ def export_remove(self, typeName, startTime, endTime, valuesList): """ Remove a record for a type """ - setup = self.serviceInfoDict["clientSetup"] startTime = int(TimeUtilities.toEpoch(startTime)) endTime = int(TimeUtilities.toEpoch(endTime)) - return self.__acDB.deleteRecord(setup, typeName, startTime, endTime, valuesList) # pylint: disable=no-member + return self.__acDB.deleteRecord(typeName, startTime, endTime, valuesList) types_removeRegisters = [list] @@ -196,20 +136,19 @@ def export_removeRegisters(self, entriesList): """ Remove a record for a type """ - setup = self.serviceInfoDict["clientSetup"] expectedTypes = [str, datetime.datetime, datetime.datetime, list] for entry in entriesList: if len(entry) != 4: return S_ERROR("Invalid records") - for i in range(len(entry)): - if not isinstance(entry[i], expectedTypes[i]): + for i, en in enumerate(entry): + if not isinstance(en, expectedTypes[i]): return S_ERROR(f"{i} field in the records should be {expectedTypes[i]}") ok = 0 for entry in entriesList: startTime = int(TimeUtilities.toEpoch(entry[1])) endTime = int(TimeUtilities.toEpoch(entry[2])) record = entry[3] - result = self.__acDB.deleteRecord(setup, entry[0], startTime, endTime, record) # pylint: disable=no-member + result = self.__acDB.deleteRecord(entry[0], startTime, endTime, record) if not result["OK"]: return S_OK(ok) ok += 1 diff --git a/src/DIRAC/AccountingSystem/Service/ReportGeneratorHandler.py b/src/DIRAC/AccountingSystem/Service/ReportGeneratorHandler.py index a543cd3fd72..0101fbc52ba 100644 --- a/src/DIRAC/AccountingSystem/Service/ReportGeneratorHandler.py +++ b/src/DIRAC/AccountingSystem/Service/ReportGeneratorHandler.py @@ -9,7 +9,7 @@ import os import datetime -from DIRAC import S_OK, S_ERROR, rootPath, gConfig, gLogger +from DIRAC import S_OK, S_ERROR, rootPath, gConfig from DIRAC.Core.Utilities.File import mkDir from DIRAC.Core.Utilities import TimeUtilities from DIRAC.AccountingSystem.DB.MultiAccountingDB import MultiAccountingDB @@ -43,19 +43,19 @@ def initializeHandler(cls, serviceInfo): cls.__acDB = MultiAccountingDB(multiPath, readOnly=True) # Get data location reportSection = serviceInfo["serviceSectionPath"] - dataPath = gConfig.getValue("%s/DataLocation" % reportSection, "data/accountingGraphs") + dataPath = gConfig.getValue(f"{reportSection}/DataLocation", "data/accountingGraphs") dataPath = dataPath.strip() if "/" != dataPath[0]: dataPath = os.path.realpath(f"{rootPath}/{dataPath}") - gLogger.info(f"Data will be written into {dataPath}") + cls.log.info(f"Data will be written into {dataPath}") mkDir(dataPath) try: - testFile = "%s/acc.jarl.test" % dataPath + testFile = f"{dataPath}/acc.jarl.test" with open(testFile, "w"): pass os.unlink(testFile) except OSError: - gLogger.fatal("Can't write to %s" % dataPath) + cls.log.fatal("Can't write to", dataPath) return S_ERROR("Data location is not writable") gDataCache.setGraphsLocation(dataPath) return S_OK() @@ -65,14 +65,14 @@ def __checkPlotRequest(self, reportRequest): if "extraArgs" not in reportRequest: reportRequest["extraArgs"] = {} if not isinstance(reportRequest["extraArgs"], self.__reportRequestDict["extraArgs"]): - return S_ERROR("Extra args has to be of type %s" % self.__reportRequestDict["extraArgs"]) + return S_ERROR(f"Extra args has to be of type {self.__reportRequestDict['extraArgs']}") reportRequestExtra = reportRequest["extraArgs"] # Check sliding plots if "lastSeconds" in reportRequestExtra: try: lastSeconds = int(reportRequestExtra["lastSeconds"]) except ValueError: - gLogger.error("lastSeconds key must be a number") + self.log.error("lastSeconds key must be a number") return S_ERROR("Value Error") if lastSeconds < 3600: return S_ERROR("lastSeconds must be more than 3600") @@ -84,15 +84,12 @@ def __checkPlotRequest(self, reportRequest): if not reportRequest.get("endTime", False): reportRequest["endTime"] = datetime.datetime.utcnow() # Check keys - for key in self.__reportRequestDict: + for key, keyType in self.__reportRequestDict.items(): if key not in reportRequest: - return S_ERROR("Missing mandatory field %s in plot reques" % key) + return S_ERROR(f"Missing mandatory field {key} in plot reques") - if not isinstance(reportRequest[key], self.__reportRequestDict[key]): - return S_ERROR( - "Type mismatch for field %s (%s), required one of %s" - % (key, str(type(reportRequest[key])), str(self.__reportRequestDict[key])) - ) + if not isinstance(reportRequest[key], keyType): + return S_ERROR(f"Type mismatch for field {key} ({type(reportRequest[key])}), required one of {keyType}") if key in ("startTime", "endTime"): reportRequest[key] = int(TimeUtilities.toEpoch(reportRequest[key])) @@ -102,20 +99,20 @@ def __checkPlotRequest(self, reportRequest): def export_generatePlot(self, reportRequest): """ - Plot a accounting + Generate an accounting plot - Arguments: - - viewName : Name of view (easy!) + :param dict reportRequest: dictionary with arguments: + - viewName - startTime - endTime - - argsDict : Arguments to the view. + - argsDict (Arguments to the view) - grouping - extraArgs """ retVal = self.__checkPlotRequest(reportRequest) if not retVal["OK"]: return retVal - reporter = MainReporter(self.__acDB, self.serviceInfoDict["clientSetup"]) + reporter = MainReporter(self.__acDB) reportRequest["generatePlot"] = True return reporter.generate(reportRequest, self.getRemoteCredentials()) @@ -123,20 +120,20 @@ def export_generatePlot(self, reportRequest): def export_getReport(self, reportRequest): """ - Plot a accounting + Gets the report but does not generate a plot - Arguments: - - viewName : Name of view (easy!) + :param dict reportRequest: dictionary with arguments: + - viewName - startTime - endTime - - argsDict : Arguments to the view. + - argsDict (Arguments to the view) - grouping - extraArgs """ retVal = self.__checkPlotRequest(reportRequest) if not retVal["OK"]: return retVal - reporter = MainReporter(self.__acDB, self.serviceInfoDict["clientSetup"]) + reporter = MainReporter(self.__acDB) reportRequest["generatePlot"] = False return reporter.generate(reportRequest, self.getRemoteCredentials()) @@ -149,7 +146,7 @@ def export_listReports(self, typeName): Arguments: - none """ - reporter = MainReporter(self.__acDB, self.serviceInfoDict["clientSetup"]) + reporter = MainReporter(self.__acDB) return reporter.list(typeName) types_listUniqueKeyValues = [str] @@ -161,7 +158,7 @@ def export_listUniqueKeyValues(self, typeName): Arguments: - none """ - dbUtils = DBUtils(self.__acDB, self.serviceInfoDict["clientSetup"]) + dbUtils = DBUtils(self.__acDB) credDict = self.getRemoteCredentials() if typeName in gPoliciesList: policyFilter = gPoliciesList[typeName] @@ -179,17 +176,17 @@ def __generatePlotFromFileId(self, fileId): if not result["OK"]: return result plotRequest = result["Value"] - gLogger.info("Generating the plots..") + self.log.info("Generating the plots..") result = self.export_generatePlot(plotRequest) if not result["OK"]: - gLogger.error("Error while generating the plots", result["Message"]) + self.log.error("Error while generating the plots", result["Message"]) return result fileToReturn = "plot" if "extraArgs" in plotRequest: extraArgs = plotRequest["extraArgs"] if "thumbnail" in extraArgs and extraArgs["thumbnail"]: fileToReturn = "thumbnail" - gLogger.info("Returning {} file: {} ".format(fileToReturn, result["Value"][fileToReturn])) + self.log.info(f"Returning {fileToReturn} file: {result['Value'][fileToReturn]} ") return S_OK(result["Value"][fileToReturn]) def __sendErrorAsImg(self, msgText, fileHelper): @@ -202,13 +199,13 @@ def transfer_toClient(self, fileId, token, fileHelper): """ # First check if we've got to generate the plot if len(fileId) > 5 and fileId[1] == ":": - gLogger.info("Seems the file request is a plot generation request!") + self.log.verbose("Seems the file request is a plot generation request!") # Seems a request for a plot! try: result = self.__generatePlotFromFileId(fileId) except Exception as e: - gLogger.exception("Exception while generating plot") - result = S_ERROR("Error while generating plot: %s" % str(e)) + self.log.exception("Exception while generating plot") + result = S_ERROR(f"Error while generating plot: {str(e)}") if not result["OK"]: self.__sendErrorAsImg(result["Message"], fileHelper) fileHelper.sendEOF() diff --git a/src/DIRAC/AccountingSystem/private/DBUtils.py b/src/DIRAC/AccountingSystem/private/DBUtils.py index c4e32bcdac5..34970ee3e77 100644 --- a/src/DIRAC/AccountingSystem/private/DBUtils.py +++ b/src/DIRAC/AccountingSystem/private/DBUtils.py @@ -1,12 +1,12 @@ """ Class that collects utilities used in Accounting and Monitoring systems """ +from DIRAC.ConfigurationSystem.Client.Helpers import CSGlobals from DIRAC.Core.Utilities import TimeUtilities class DBUtils: - def __init__(self, db, setup): + def __init__(self, db): self._acDB = db - self._setup = setup def _retrieveBucketedData( self, typeName, startTime, endTime, selectFields, condDict=None, groupFields=None, orderFields=None @@ -32,7 +32,7 @@ def _retrieveBucketedData( if isinstance(condDict[key], (list, tuple)) and condDict[key]: validCondDict[key] = condDict[key] return self._acDB.retrieveBucketedData( - self._setup, typeName, startTime, endTime, selectFields, condDict, groupFields, orderFields + typeName, startTime, endTime, selectFields, condDict, groupFields, orderFields ) def _getUniqueValues(self, typeName, startTime, endTime, condDict, fieldList): @@ -60,11 +60,11 @@ def _groupByField(self, fieldIndex, dataList): return groupDict def _getBins(self, typeName, startTime, endTime): - return self._acDB.calculateBuckets(self._setup, typeName, startTime, endTime) + return self._acDB.calculateBuckets(typeName, startTime, endTime) def _getBucketLengthForTime(self, typeName, momentEpoch): nowEpoch = TimeUtilities.toEpoch() - return self._acDB.calculateBucketLengthForTime(self._setup, typeName, nowEpoch, momentEpoch) + return self._acDB.calculateBucketLengthForTime(typeName, nowEpoch, momentEpoch) def _spanToGranularity(self, granularity, bucketsData): """ @@ -280,7 +280,7 @@ def getKeyValues(self, typeName, condDict): """ Get all valid key values in a type """ - return self._acDB.getKeyValues(self._setup, typeName, condDict) + return self._acDB.getKeyValues(typeName, condDict) def _calculateProportionalGauges(self, dataDict): """ diff --git a/src/DIRAC/AccountingSystem/private/MainReporter.py b/src/DIRAC/AccountingSystem/private/MainReporter.py index 0064bd95f82..97ac037dfdd 100644 --- a/src/DIRAC/AccountingSystem/private/MainReporter.py +++ b/src/DIRAC/AccountingSystem/private/MainReporter.py @@ -3,17 +3,17 @@ import hashlib import re -from DIRAC import S_OK, S_ERROR, gConfig -from DIRAC.Core.Utilities.ObjectLoader import loadObjects -from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceSection +from DIRAC import S_ERROR, S_OK, gConfig +from DIRAC.AccountingSystem.private.Plotters.BaseReporter import BaseReporter from DIRAC.AccountingSystem.private.Policies import gPoliciesList -from DIRAC.AccountingSystem.private.Plotters.BaseReporter import BaseReporter as myBaseReporter +from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceSection +from DIRAC.Core.Utilities.ObjectLoader import loadObjects class PlottersList: def __init__(self): objectsLoaded = loadObjects( - "AccountingSystem/private/Plotters", re.compile(r".*[a-z1-9]Plotter\.py$"), myBaseReporter + "AccountingSystem/private/Plotters", re.compile(r".*[a-z1-9]Plotter\.py$"), BaseReporter ) self.__plotters = {} for objName in objectsLoaded: @@ -27,29 +27,27 @@ def getPlotterClass(self, typeName): class MainReporter: - def __init__(self, db, setup): + def __init__(self, db): self._db = db - self.setup = setup - self.csSection = getServiceSection("Accounting/ReportGenerator", setup=setup) + self.csSection = getServiceSection("Accounting/ReportGenerator") self.plotterList = PlottersList() def __calculateReportHash(self, reportRequest): requestToHash = dict(reportRequest) - granularity = gConfig.getValue("%s/CacheTimeGranularity" % self.csSection, 300) + granularity = gConfig.getValue(f"{self.csSection}/CacheTimeGranularity", 300) granularity *= 1000 for key in ("startTime", "endTime"): epoch = requestToHash[key] requestToHash[key] = epoch - epoch % granularity md5Hash = hashlib.md5() md5Hash.update(repr(requestToHash).encode()) - md5Hash.update(self.setup.encode()) return md5Hash.hexdigest() def generate(self, reportRequest, credDict): typeName = reportRequest["typeName"] plotterClass = self.plotterList.getPlotterClass(typeName) if not plotterClass: - return S_ERROR("There's no reporter registered for type %s" % typeName) + return S_ERROR(f"There's no reporter registered for type {typeName}") if typeName in gPoliciesList: retVal = gPoliciesList[typeName].checkRequest( reportRequest["reportName"], credDict, reportRequest["condDict"], reportRequest["grouping"] @@ -57,12 +55,12 @@ def generate(self, reportRequest, credDict): if not retVal["OK"]: return retVal reportRequest["hash"] = self.__calculateReportHash(reportRequest) - plotter = plotterClass(self._db, self.setup, reportRequest["extraArgs"]) + plotter = plotterClass(self._db, reportRequest["extraArgs"]) return plotter.generate(reportRequest) def list(self, typeName): plotterClass = self.plotterList.getPlotterClass(typeName) if not plotterClass: - return S_ERROR("There's no plotter registered for type %s" % typeName) - plotter = plotterClass(self._db, self.setup) + return S_ERROR(f"There's no plotter registered for type {typeName}") + plotter = plotterClass(self._db) return S_OK(plotter.plotsList()) diff --git a/src/DIRAC/AccountingSystem/private/Plotters/BaseReporter.py b/src/DIRAC/AccountingSystem/private/Plotters/BaseReporter.py index aadc73818b7..a422e1db8dc 100644 --- a/src/DIRAC/AccountingSystem/private/Plotters/BaseReporter.py +++ b/src/DIRAC/AccountingSystem/private/Plotters/BaseReporter.py @@ -16,7 +16,6 @@ class BaseReporter(DBUtils): - _PARAM_CHECK_FOR_NONE = "checkNone" _PARAM_CALCULATE_PROPORTIONAL_GAUGES = "calculateProportionalGauges" _PARAM_CONVERT_TO_GRANULARITY = "convertToGranularity" @@ -83,8 +82,8 @@ class BaseReporter(DBUtils): _typeKeyFields = [] _typeName = "" - def __init__(self, db, setup, extraArgs=None): - DBUtils.__init__(self, db, setup) + def __init__(self, db, extraArgs=None): + DBUtils.__init__(self, db) if isinstance(extraArgs, dict): self._extraArgs = extraArgs else: @@ -110,14 +109,12 @@ def _translateGrouping(self, grouping): def _averageConsolidation(self, total, count): if count == 0: return 0 - else: - return float(total) / float(count) + return float(total) / float(count) def _efficiencyConsolidation(self, total, count): if count == 0: return 0 - else: - return (float(total) / float(count)) * 100.0 + return (float(total) / float(count)) * 100.0 def generate(self, reportRequest): reportRequest["groupingFields"] = self._translateGrouping(reportRequest["grouping"]) @@ -125,7 +122,7 @@ def generate(self, reportRequest): reportName = reportRequest["reportName"] if reportName in self.__reportNameMapping: reportRequest["reportName"] = self.__reportNameMapping[reportName] - gLogger.info("Retrieving data for {}:{}".format(reportRequest["typeName"], reportRequest["reportName"])) + gLogger.info(f"Retrieving data for {reportRequest['typeName']}:{reportRequest['reportName']}") sT = time.time() retVal = self.__retrieveReportData(reportRequest, reportHash) reportGenerationTime = time.time() - sT @@ -134,7 +131,7 @@ def generate(self, reportRequest): if not reportRequest["generatePlot"]: return retVal reportData = retVal["Value"] - gLogger.info("Plotting data for {}:{}".format(reportRequest["typeName"], reportRequest["reportName"])) + gLogger.info(f"Plotting data for {reportRequest['typeName']}:{reportRequest['reportName']}") sT = time.time() retVal = self.__generatePlotForReport(reportRequest, reportHash, reportData) plotGenerationTime = time.time() - sT @@ -159,19 +156,19 @@ def plotsList(self): return sorted(k for k in self.__reportNameMapping) def __retrieveReportData(self, reportRequest, reportHash): - funcName = "_report%s" % reportRequest["reportName"] + funcName = f"_report{reportRequest['reportName']}" try: funcObj = getattr(self, funcName) except Exception: - return S_ERROR("Report %s is not defined" % reportRequest["reportName"]) + return S_ERROR(f"Report {reportRequest['reportName']} is not defined") return gDataCache.getReportData(reportRequest, reportHash, funcObj) def __generatePlotForReport(self, reportRequest, reportHash, reportData): - funcName = "_plot%s" % reportRequest["reportName"] + funcName = f"_plot{reportRequest['reportName']}" try: funcObj = getattr(self, funcName) except Exception: - return S_ERROR("Plot function for report %s is not defined" % reportRequest["reportName"]) + return S_ERROR(f"Plot function for report {reportRequest['reportName']} is not defined") return gDataCache.getReportPlot(reportRequest, reportHash, reportData, funcObj) ### @@ -186,7 +183,7 @@ def _getTimedData(self, startTime, endTime, selectFields, preCondDict, groupingF if self._PARAM_CONVERT_TO_GRANULARITY not in metadataDict: metadataDict[self._PARAM_CONVERT_TO_GRANULARITY] = "sum" elif metadataDict[self._PARAM_CONVERT_TO_GRANULARITY] not in self._VALID_PARAM_CONVERT_TO_GRANULARITY: - return S_ERROR("%s field metadata is invalid" % self._PARAM_CONVERT_TO_GRANULARITY) + return S_ERROR(f"{self._PARAM_CONVERT_TO_GRANULARITY} field metadata is invalid") if self._PARAM_CALCULATE_PROPORTIONAL_GAUGES not in metadataDict: metadataDict[self._PARAM_CALCULATE_PROPORTIONAL_GAUGES] = False # Make safe selections @@ -280,8 +277,7 @@ def _getSelectStringForGrouping(self, groupingFields): if len(groupingFields[1]) == 1: # If there's only one field, then we send the sql representation in pos 0 return groupingFields[0] - else: - return "CONCAT( %s )" % ", ".join(["%s, '-'" % sqlRep for sqlRep in groupingFields[0]]) + return "CONCAT( %s )" % ", ".join(["%s, '-'" % sqlRep for sqlRep in groupingFields[0]]) def _findSuitableRateUnit(self, dataDict, maxValue, unit): return self._findUnitMagic(dataDict, maxValue, unit, self._RATE_UNITS) @@ -291,14 +287,14 @@ def _findSuitableUnit(self, dataDict, maxValue, unit): def _findUnitMagic(self, reportDataDict, maxValue, unit, selectedUnits): if unit not in selectedUnits: - raise AttributeError("%s is not a known rate unit" % unit) + raise AttributeError(f"{unit} is not a known rate unit") baseUnitData = selectedUnits[unit][0] if "staticUnits" in self._extraArgs and self._extraArgs["staticUnits"]: unitData = selectedUnits[unit][0] else: unitList = selectedUnits[unit] unitIndex = -1 - for unitName, unitDivFactor, unitThreshold in unitList: + for _unitName, unitDivFactor, unitThreshold in unitList: unitIndex += 1 if maxValue / unitDivFactor < unitThreshold: break @@ -352,14 +348,14 @@ def __plotData(self, filename, dataDict, metadata, funcToPlot): self.__checkPlotMetadata(metadata) if not dataDict: funcToPlot = generateNoDataPlot - plotFileName = "%s.png" % filename + plotFileName = f"{filename}.png" finalResult = funcToPlot(plotFileName, dataDict, metadata) if not finalResult["OK"]: return finalResult thbMD = self.__checkThumbnailMetadata(metadata) if not thbMD: return S_OK({"plot": True, "thumbnail": False}) - thbFilename = "%s.thb.png" % filename + thbFilename = f"{filename}.thb.png" retVal = funcToPlot(thbFilename, dataDict, thbMD) if not retVal["OK"]: return retVal diff --git a/src/DIRAC/AccountingSystem/private/Plotters/DataOperationPlotter.py b/src/DIRAC/AccountingSystem/private/Plotters/DataOperationPlotter.py index 4b6c81d31fb..390fe5d6369 100644 --- a/src/DIRAC/AccountingSystem/private/Plotters/DataOperationPlotter.py +++ b/src/DIRAC/AccountingSystem/private/Plotters/DataOperationPlotter.py @@ -4,7 +4,6 @@ class DataOperationPlotter(BaseReporter): - _typeName = "DataOperation" _typeKeyFields = [dF[0] for dF in DataOperation().definitionKeyFields] @@ -67,7 +66,7 @@ def _plotFailedTransfers(self, reportRequest, plotInfo, filename): def __plotTransfers(self, reportRequest, plotInfo, filename, titleType, togetherFieldsToPlot): metadata = { - "title": "{} Transfers by {}".format(titleType, reportRequest["grouping"]), + "title": f"{titleType} Transfers by {reportRequest['grouping']}", "ylabel": plotInfo["unit"], "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], @@ -128,7 +127,7 @@ def _reportQuality(self, reportRequest): def _plotQuality(self, reportRequest, plotInfo, filename): metadata = { - "title": "Transfer quality by %s" % reportRequest["grouping"], + "title": f"Transfer quality by {reportRequest['grouping']}", "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], "span": plotInfo["granularity"], @@ -165,7 +164,7 @@ def _reportTransferedData(self, reportRequest): def _plotTransferedData(self, reportRequest, plotInfo, filename): metadata = { - "title": "Transfered data by %s" % reportRequest["grouping"], + "title": f"Transfered data by {reportRequest['grouping']}", "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], "span": plotInfo["granularity"], @@ -202,7 +201,7 @@ def _reportThroughput(self, reportRequest): def _plotThroughput(self, reportRequest, plotInfo, filename): metadata = { - "title": "Throughput by %s" % reportRequest["grouping"], + "title": f"Throughput by {reportRequest['grouping']}", "ylabel": plotInfo["unit"], "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], @@ -234,7 +233,7 @@ def _reportDataTransfered(self, reportRequest): def _plotDataTransfered(self, reportRequest, plotInfo, filename): metadata = { - "title": "Total data transfered by %s" % reportRequest["grouping"], + "title": f"Total data transfered by {reportRequest['grouping']}", "ylabel": "bytes", "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], diff --git a/src/DIRAC/AccountingSystem/private/Plotters/JobPlotter.py b/src/DIRAC/AccountingSystem/private/Plotters/JobPlotter.py index 7aad23619b1..100bb682a46 100644 --- a/src/DIRAC/AccountingSystem/private/Plotters/JobPlotter.py +++ b/src/DIRAC/AccountingSystem/private/Plotters/JobPlotter.py @@ -6,7 +6,6 @@ class JobPlotter(BaseReporter): - _typeName = "Job" _typeKeyFields = [dF[0] for dF in Job().definitionKeyFields] @@ -14,10 +13,9 @@ def _translateGrouping(self, grouping): if grouping == "Country": sqlRepr = 'upper( substring( %s, locate( ".", %s, length( %s ) - 4 ) + 1 ) )' return (sqlRepr, ["Site", "Site", "Site"], sqlRepr) - elif grouping == "Grid": + if grouping == "Grid": return ('substring_index( %s, ".", 1 )', ["Site"]) - else: - return ("%s", [grouping]) + return ("%s", [grouping]) _reportCPUEfficiencyName = "CPU efficiency" @@ -71,7 +69,7 @@ def _reportCPUEfficiency(self, reportRequest): def _plotCPUEfficiency(self, reportRequest, plotInfo, filename): metadata = { - "title": "Job CPU efficiency by %s" % reportRequest["grouping"], + "title": f"Job CPU efficiency by {reportRequest['grouping']}", "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], "span": plotInfo["granularity"], @@ -108,7 +106,7 @@ def _reportCPUUsed(self, reportRequest): def _plotCPUUsed(self, reportRequest, plotInfo, filename): metadata = { - "title": "CPU used by %s" % reportRequest["grouping"], + "title": f"CPU used by {reportRequest['grouping']}", "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], "span": plotInfo["granularity"], @@ -147,7 +145,7 @@ def _reportCPUUsage(self, reportRequest): def _plotCPUUsage(self, reportRequest, plotInfo, filename): metadata = { - "title": "CPU usage by %s" % reportRequest["grouping"], + "title": f"CPU usage by {reportRequest['grouping']}", "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], "span": plotInfo["granularity"], @@ -185,7 +183,7 @@ def _reportNormCPUUsed(self, reportRequest): def _plotNormCPUUsed(self, reportRequest, plotInfo, filename): metadata = { - "title": "Normalized CPU used by %s" % reportRequest["grouping"], + "title": f"Normalized CPU used by {reportRequest['grouping']}", "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], "span": plotInfo["granularity"], @@ -224,7 +222,7 @@ def _reportNormCPUUsage(self, reportRequest): def _plotNormCPUUsage(self, reportRequest, plotInfo, filename): metadata = { - "title": "Normalized CPU usage by %s" % reportRequest["grouping"], + "title": f"Normalized CPU usage by {reportRequest['grouping']}", "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], "span": plotInfo["granularity"], @@ -262,7 +260,7 @@ def _reportWallTime(self, reportRequest): def _plotWallTime(self, reportRequest, plotInfo, filename): metadata = { - "title": "Wall Time by %s" % reportRequest["grouping"], + "title": f"Wall Time by {reportRequest['grouping']}", "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], "span": plotInfo["granularity"], @@ -300,7 +298,7 @@ def _reportRunningJobs(self, reportRequest): def _plotRunningJobs(self, reportRequest, plotInfo, filename): metadata = { - "title": "Running jobs by %s" % reportRequest["grouping"], + "title": f"Running jobs by {reportRequest['grouping']}", "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], "span": plotInfo["granularity"], @@ -330,7 +328,7 @@ def _reportTotalCPUUsed(self, reportRequest): def _plotTotalCPUUsed(self, reportRequest, plotInfo, filename): metadata = { - "title": "CPU days used by %s" % reportRequest["grouping"], + "title": f"CPU days used by {reportRequest['grouping']}", "ylabel": "CPU days", "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], @@ -367,7 +365,7 @@ def _reportAccumulatedWallTime(self, reportRequest): def _plotAccumulatedWallTime(self, reportRequest, plotInfo, filename): metadata = { - "title": "Cumulative wall time by %s" % reportRequest["grouping"], + "title": f"Cumulative wall time by {reportRequest['grouping']}", "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], "span": plotInfo["granularity"], @@ -398,7 +396,7 @@ def _reportTotalWallTime(self, reportRequest): def _plotTotalWallTime(self, reportRequest, plotInfo, filename): metadata = { - "title": "Wall time days used by %s" % reportRequest["grouping"], + "title": f"Wall time days used by {reportRequest['grouping']}", "ylabel": "CPU days", "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], @@ -439,7 +437,7 @@ def _reportCumulativeNumberOfJobs(self, reportRequest): def _plotCumulativeNumberOfJobs(self, reportRequest, plotInfo, filename): metadata = { - "title": "Cumulative Jobs by %s" % reportRequest["grouping"], + "title": f"Cumulative Jobs by {reportRequest['grouping']}", "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], "span": plotInfo["granularity"], @@ -478,7 +476,7 @@ def _reportNumberOfJobs(self, reportRequest): def _plotNumberOfJobs(self, reportRequest, plotInfo, filename): metadata = { - "title": "Jobs by %s" % reportRequest["grouping"], + "title": f"Jobs by {reportRequest['grouping']}", "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], "span": plotInfo["granularity"], @@ -508,7 +506,7 @@ def _reportTotalNumberOfJobs(self, reportRequest): def _plotTotalNumberOfJobs(self, reportRequest, plotInfo, filename): metadata = { - "title": "Total Number of Jobs by %s" % reportRequest["grouping"], + "title": f"Total Number of Jobs by {reportRequest['grouping']}", "ylabel": "Jobs", "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], @@ -550,7 +548,7 @@ def _reportProcessingBandwidth(self, reportRequest): def _plotProcessingBandwidth(self, reportRequest, plotInfo, filename): metadata = { - "title": "Processing Bandwidth by %s" % reportRequest["grouping"], + "title": f"Processing Bandwidth by {reportRequest['grouping']}", "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], "span": plotInfo["granularity"], @@ -632,7 +630,7 @@ def _plotOutputDataSize(self, reportRequest, plotInfo, filename): def __plotFieldSizeinMB(self, reportRequest, plotInfo, filename, fieldTuple): metadata = { - "title": "{} by {}".format(fieldTuple[1], reportRequest["grouping"]), + "title": f"{fieldTuple[1]} by {reportRequest['grouping']}", "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], "span": plotInfo["granularity"], @@ -716,7 +714,7 @@ def _plotCumulativeOutputDataSize(self, reportRequest, plotInfo, filename): def __plotCumulativeFieldSizeinMB(self, reportRequest, plotInfo, filename, fieldTuple): metadata = { - "title": "Cumulative {} by {}".format(fieldTuple[1], reportRequest["grouping"]), + "title": f"Cumulative {fieldTuple[1]} by {reportRequest['grouping']}", "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], "span": plotInfo["granularity"], @@ -772,7 +770,7 @@ def _plotOuputDataFiles(self, reportRequest, plotInfo, filename): def __plotDataFiles(self, reportRequest, plotInfo, filename, fieldTuple): metadata = { - "title": "{} by {}".format(fieldTuple[1], reportRequest["grouping"]), + "title": f"{fieldTuple[1]} by {reportRequest['grouping']}", "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], "span": plotInfo["granularity"], @@ -798,7 +796,7 @@ def _reportHistogramCPUUsed(self, reportRequest): def _plotHistogramCPUUsed(self, reportRequest, plotInfo, filename): metadata = { - "title": "CPU usage by %s" % reportRequest["grouping"], + "title": f"CPU usage by {reportRequest['grouping']}", "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], } diff --git a/src/DIRAC/AccountingSystem/private/Plotters/NetworkPlotter.py b/src/DIRAC/AccountingSystem/private/Plotters/NetworkPlotter.py index 41bc38dc732..f821c6163b5 100644 --- a/src/DIRAC/AccountingSystem/private/Plotters/NetworkPlotter.py +++ b/src/DIRAC/AccountingSystem/private/Plotters/NetworkPlotter.py @@ -14,14 +14,12 @@ class NetworkPlotter(BaseReporter): - _typeName = "Network" _typeKeyFields = [dF[0] for dF in Network().definitionKeyFields] _reportPacketLossRateName = "Packet loss rate" def _reportPacketLossRate(self, reportRequest): - selectFields = ( self._getSelectStringForGrouping(reportRequest["groupingFields"]) + ", %s, %s, 100 - SUM(%s)/SUM(%s), 100", reportRequest["groupingFields"][1] + ["startTime", "bucketLength", "PacketLossRate", "entriesInBucket"], @@ -43,13 +41,12 @@ def _reportPacketLossRate(self, reportRequest): return S_OK({"data": dataDict, "granularity": granularity}) def _plotPacketLossRate(self, reportRequest, plotInfo, filename): - # prepare custom scale (10,20,...,100) scale_data = dict(zip(range(0, 101), range(100, -1, -1))) scale_ticks = list(range(0, 101, 10)) metadata = { - "title": "Packet loss rate by %s" % reportRequest["grouping"], + "title": f"Packet loss rate by {reportRequest['grouping']}", "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], "span": plotInfo["granularity"], @@ -63,7 +60,6 @@ def _plotPacketLossRate(self, reportRequest, plotInfo, filename): _reportMagnifiedPacketLossRateName = "Packet loss rate (magnified)" def _reportMagnifiedPacketLossRate(self, reportRequest): - selectFields = ( self._getSelectStringForGrouping(reportRequest["groupingFields"]) + ", %s, %s, 100 - IF(SUM(%s)/SUM(%s)*10 > 100, 100, SUM(%s)/SUM(%s)*10), 100", @@ -94,7 +90,6 @@ def _reportMagnifiedPacketLossRate(self, reportRequest): return S_OK({"data": dataDict, "granularity": granularity}) def _plotMagnifiedPacketLossRate(self, reportRequest, plotInfo, filename): - # prepare custom scale (1..10, 100) boundaries = list(np.arange(0, 10, 0.1)) boundaries.extend(range(10, 110, 10)) @@ -106,7 +101,7 @@ def _plotMagnifiedPacketLossRate(self, reportRequest, plotInfo, filename): scale_ticks.append(100) metadata = { - "title": "Magnified packet loss rate by %s" % reportRequest["grouping"], + "title": f"Magnified packet loss rate by {reportRequest['grouping']}", "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], "span": plotInfo["granularity"], @@ -120,7 +115,6 @@ def _plotMagnifiedPacketLossRate(self, reportRequest, plotInfo, filename): _reportAverageOneWayDelayName = "One-way delay (average)" def _reportAverageOneWayDelay(self, reportRequest): - selectFields = ( self._getSelectStringForGrouping(reportRequest["groupingFields"]) + ", %s, %s, SUM(%s)/SUM(%s)", reportRequest["groupingFields"][1] + ["startTime", "bucketLength", "OneWayDelay", "entriesInBucket"], @@ -144,7 +138,7 @@ def _reportAverageOneWayDelay(self, reportRequest): def _plotAverageOneWayDelay(self, reportRequest, plotInfo, filename): metadata = { - "title": "One-way delay by %s" % reportRequest["grouping"], + "title": f"One-way delay by {reportRequest['grouping']}", "ylabel": plotInfo["unit"], "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], @@ -158,7 +152,6 @@ def _plotAverageOneWayDelay(self, reportRequest, plotInfo, filename): _reportJitterName = "Jitter" def _reportJitter(self, reportRequest): - selectFields = ( self._getSelectStringForGrouping(reportRequest["groupingFields"]) + ", %s, %s, SUM(%s)/SUM(%s)", reportRequest["groupingFields"][1] + ["startTime", "bucketLength", "Jitter", "entriesInBucket"], @@ -182,7 +175,7 @@ def _reportJitter(self, reportRequest): def _plotJitter(self, reportRequest, plotInfo, filename): metadata = { - "title": "Jitter by %s" % reportRequest["grouping"], + "title": f"Jitter by {reportRequest['grouping']}", "ylabel": plotInfo["unit"], "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], @@ -196,7 +189,6 @@ def _plotJitter(self, reportRequest, plotInfo, filename): _reportJitterDelayRatioName = "Jitter/Delay" def _reportJitterDelayRatio(self, reportRequest): - selectFields = ( self._getSelectStringForGrouping(reportRequest["groupingFields"]) + ", %s, %s, SUM(%s)/SUM(%s)", reportRequest["groupingFields"][1] + ["startTime", "bucketLength", "Jitter", "OneWayDelay"], @@ -220,7 +212,7 @@ def _reportJitterDelayRatio(self, reportRequest): def _plotJitterDelayRatio(self, reportRequest, plotInfo, filename): metadata = { - "title": "Jitter over one-way delay by %s" % reportRequest["grouping"], + "title": f"Jitter over one-way delay by {reportRequest['grouping']}", "ylabel": "", "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], diff --git a/src/DIRAC/AccountingSystem/private/Plotters/PilotPlotter.py b/src/DIRAC/AccountingSystem/private/Plotters/PilotPlotter.py index 1a682f18a20..aeb4f6ae9d1 100644 --- a/src/DIRAC/AccountingSystem/private/Plotters/PilotPlotter.py +++ b/src/DIRAC/AccountingSystem/private/Plotters/PilotPlotter.py @@ -4,7 +4,6 @@ class PilotPlotter(BaseReporter): - _typeName = "Pilot" _typeKeyFields = [dF[0] for dF in Pilot().definitionKeyFields] @@ -36,7 +35,7 @@ def _reportCumulativeNumberOfJobs(self, reportRequest): def _plotCumulativeNumberOfJobs(self, reportRequest, plotInfo, filename): metadata = { - "title": "Cumulative Jobs by %s" % reportRequest["grouping"], + "title": f"Cumulative Jobs by {reportRequest['grouping']}", "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], "span": plotInfo["granularity"], @@ -73,7 +72,7 @@ def _reportNumberOfJobs(self, reportRequest): def _plotNumberOfJobs(self, reportRequest, plotInfo, filename): metadata = { - "title": "Jobs by %s" % reportRequest["grouping"], + "title": f"Jobs by {reportRequest['grouping']}", "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], "span": plotInfo["granularity"], @@ -109,7 +108,7 @@ def _reportCumulativeNumberOfPilots(self, reportRequest): def _plotCumulativeNumberOfPilots(self, reportRequest, plotInfo, filename): metadata = { - "title": "Cumulative Pilots by %s" % reportRequest["grouping"], + "title": f"Cumulative Pilots by {reportRequest['grouping']}", "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], "span": plotInfo["granularity"], @@ -146,7 +145,7 @@ def _reportNumberOfPilots(self, reportRequest): def _plotNumberOfPilots(self, reportRequest, plotInfo, filename): metadata = { - "title": "Pilots by %s" % reportRequest["grouping"], + "title": f"Pilots by {reportRequest['grouping']}", "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], "span": plotInfo["granularity"], @@ -181,13 +180,14 @@ def _reportJobsPerPilot(self, reportRequest): def _plotJobsPerPilot(self, reportRequest, plotInfo, filename): metadata = { - "title": "Jobs per pilot by %s" % reportRequest["grouping"], + "title": f"Jobs per pilot by {reportRequest['grouping']}", "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], "span": plotInfo["granularity"], "ylabel": "jobs/pilot", - "normalization": max(x for y in plotInfo["data"].values() for x in y.values()), } + if plotInfo["data"]: + metadata["normalization"] = max(x for y in plotInfo["data"].values() for x in y.values()) return self._generateQualityPlot(filename, plotInfo["data"], metadata) def _reportTotalNumberOfPilots(self, reportRequest): @@ -210,7 +210,7 @@ def _reportTotalNumberOfPilots(self, reportRequest): def _plotTotalNumberOfPilots(self, reportRequest, plotInfo, filename): metadata = { - "title": "Total Number of Pilots by %s" % reportRequest["grouping"], + "title": f"Total Number of Pilots by {reportRequest['grouping']}", "ylabel": "Pilots", "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], diff --git a/src/DIRAC/AccountingSystem/private/Plotters/PilotSubmissionPlotter.py b/src/DIRAC/AccountingSystem/private/Plotters/PilotSubmissionPlotter.py index 0309b0a7fbc..beed9056db1 100644 --- a/src/DIRAC/AccountingSystem/private/Plotters/PilotSubmissionPlotter.py +++ b/src/DIRAC/AccountingSystem/private/Plotters/PilotSubmissionPlotter.py @@ -46,7 +46,7 @@ def _plotSubmission(self, reportRequest, plotInfo, filename): """ metadata = { - "title": "Number of Submission by %s" % reportRequest["grouping"], + "title": f"Number of Submission by {reportRequest['grouping']}", "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], "span": plotInfo["granularity"], @@ -129,7 +129,7 @@ def _plotSubmissionEfficiency(self, reportRequest, plotInfo, filename): """ metadata = { - "title": "Pilot Submission efficiency by %s" % reportRequest["grouping"], + "title": f"Pilot Submission efficiency by {reportRequest['grouping']}", "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], "span": plotInfo["granularity"], diff --git a/src/DIRAC/AccountingSystem/private/Plotters/StorageOccupancyPlotter.py b/src/DIRAC/AccountingSystem/private/Plotters/StorageOccupancyPlotter.py index 4af1da2635a..2766c13c133 100644 --- a/src/DIRAC/AccountingSystem/private/Plotters/StorageOccupancyPlotter.py +++ b/src/DIRAC/AccountingSystem/private/Plotters/StorageOccupancyPlotter.py @@ -81,7 +81,7 @@ def plotter(self, reportRequest, plotInfo, filename): dataDict = plotInfo["graphDataDict"] metadata = { - "title": "Space grouped by %s" % reportRequest["grouping"], + "title": f"Space grouped by {reportRequest['grouping']}", "starttime": startEpoch, "endtime": endEpoch, "span": granularity, diff --git a/src/DIRAC/AccountingSystem/private/Plotters/WMSHistoryPlotter.py b/src/DIRAC/AccountingSystem/private/Plotters/WMSHistoryPlotter.py index f6e111a6c93..ed9f81f50fc 100644 --- a/src/DIRAC/AccountingSystem/private/Plotters/WMSHistoryPlotter.py +++ b/src/DIRAC/AccountingSystem/private/Plotters/WMSHistoryPlotter.py @@ -4,7 +4,6 @@ class WMSHistoryPlotter(BaseReporter): - _typeName = "WMSHistory" _typeKeyFields = [dF[0] for dF in WMSHistory().definitionKeyFields] @@ -38,7 +37,7 @@ def _reportNumberOfJobs(self, reportRequest): def _plotNumberOfJobs(self, reportRequest, plotInfo, filename): metadata = { - "title": "Jobs by %s" % reportRequest["grouping"], + "title": f"Jobs by {reportRequest['grouping']}", "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], "span": plotInfo["granularity"], @@ -71,7 +70,7 @@ def _reportNumberOfReschedules(self, reportRequest): def _plotNumberOfReschedules(self, reportRequest, plotInfo, filename): metadata = { - "title": "Reschedules by %s" % reportRequest["grouping"], + "title": f"Reschedules by {reportRequest['grouping']}", "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], "span": plotInfo["granularity"], @@ -107,7 +106,7 @@ def _reportAverageNumberOfJobs(self, reportRequest): def _plotAverageNumberOfJobs(self, reportRequest, plotInfo, filename): metadata = { - "title": "Average Number of Jobs by %s" % reportRequest["grouping"], + "title": f"Average Number of Jobs by {reportRequest['grouping']}", "ylabel": "Jobs", "starttime": reportRequest["startTime"], "endtime": reportRequest["endTime"], diff --git a/src/DIRAC/AccountingSystem/private/Policies/FilterExecutor.py b/src/DIRAC/AccountingSystem/private/Policies/FilterExecutor.py index d11403fcb3d..281e2822d9c 100644 --- a/src/DIRAC/AccountingSystem/private/Policies/FilterExecutor.py +++ b/src/DIRAC/AccountingSystem/private/Policies/FilterExecutor.py @@ -2,7 +2,6 @@ class FilterExecutor: - ALLKW = "all" def __init__(self): @@ -18,7 +17,7 @@ def applyFilters(self, iD, credDict, condDict, groupingList): gLogger.info(f"Applying filter {myFilter.__name__} for {iD}") retVal = myFilter(credDict, condDict, groupingList) if not retVal["OK"]: - gLogger.info("Filter {} for {} failed: {}".format(myFilter.__name__, iD, retVal["Message"])) + gLogger.info(f"Filter {myFilter.__name__} for {iD} failed: {retVal['Message']}") return retVal except Exception: gLogger.exception("Exception while applying filter", f"{myFilter.__name__} for {iD}") diff --git a/src/DIRAC/AccountingSystem/scripts/dirac_accounting_decode_fileid.py b/src/DIRAC/AccountingSystem/scripts/dirac_accounting_decode_fileid.py index 0b09782ae7e..e9156f79476 100755 --- a/src/DIRAC/AccountingSystem/scripts/dirac_accounting_decode_fileid.py +++ b/src/DIRAC/AccountingSystem/scripts/dirac_accounting_decode_fileid.py @@ -32,9 +32,9 @@ def main(): # Decode result = extractRequestFromFileId(fileId) if not result["OK"]: - gLogger.error("Could not decode fileId", "'{}', error was {}".format(fileId, result["Message"])) + gLogger.error("Could not decode fileId", f"'{fileId}', error was {result['Message']}") sys.exit(1) - gLogger.notice("Decode for '{}' is:\n{}".format(fileId, pprint.pformat(result["Value"]))) + gLogger.notice(f"Decode for '{fileId}' is:\n{pprint.pformat(result['Value'])}") sys.exit(0) diff --git a/src/DIRAC/ConfigurationSystem/Agent/Bdii2CSAgent.py b/src/DIRAC/ConfigurationSystem/Agent/Bdii2CSAgent.py index b759f504ca5..1704ad6e2b9 100644 --- a/src/DIRAC/ConfigurationSystem/Agent/Bdii2CSAgent.py +++ b/src/DIRAC/ConfigurationSystem/Agent/Bdii2CSAgent.py @@ -86,7 +86,7 @@ def initialize(self): self.voName.append(vomsVO) if self.voName: - self.log.info("Agent will manage VO(s) %s" % self.voName) + self.log.info(f"Agent will manage VO(s) {self.voName}") else: self.log.fatal("VirtualOrganization option not defined for agent") return S_ERROR() @@ -103,7 +103,7 @@ def execute(self): if not result["OK"]: self.log.warn("Could not download a fresh copy of the CS data", result["Message"]) - # Refresh the configuration from the master server + # Refresh the configuration from the controller server gConfig.forceRefresh(fromMaster=True) if self.processCEs: @@ -167,14 +167,14 @@ def __lookForNewCEs(self): body += ceString if siteDict: - body = "\nWe are glad to inform You about new CE(s) possibly suitable for %s:\n" % vo + body + body = f"\nWe are glad to inform You about new CE(s) possibly suitable for {vo}:\n" + body body += "\n\nTo suppress information about CE add its name to BannedCEs list.\n" - body += "Add new Sites/CEs for vo %s with the command:\n" % vo - body += "dirac-admin-add-resources --vo %s --ce\n" % vo + body += f"Add new Sites/CEs for vo {vo} with the command:\n" + body += f"dirac-admin-add-resources --vo {vo} --ce\n" if unknownCEs: body += "\n\n" - body += "There is no (longer) information about the following CEs for the %s VO.\n" % vo + body += f"There is no (longer) information about the following CEs for the {vo} VO.\n" body += "\n".join(sorted(unknownCEs)) body += "\n\n" @@ -191,7 +191,6 @@ def __lookForNewCEs(self): return S_OK() def __getGlue2CEInfo(self, vo): - if vo in self.voBdiiCEDict: return S_OK(self.voBdiiCEDict[vo]) self.log.info("Check for available CEs for VO", vo) @@ -208,7 +207,7 @@ def __getGlue2CEInfo(self, vo): if resultAlt["OK"]: totalResult["Value"].update(resultAlt["Value"]) else: - self.log.error("Failed getting information from %s " % bdii, resultAlt["Message"]) + self.log.error(f"Failed getting information from {bdii} ", resultAlt["Message"]) message = (message + "\n" + resultAlt["Message"]).strip() if mainResult["OK"]: @@ -265,7 +264,7 @@ def __purgeSites(self, ceBdiiDict): for ce in ces: res = getCESiteMapping(ce) if not res["OK"]: - self.log.error("Failed to get DIRAC site name for ce", "{}: {}".format(ce, res["Message"])) + self.log.error("Failed to get DIRAC site name for ce", f"{ce}: {res['Message']}") continue # if the ce is not in the CS the returned value will be empty if ce in res["Value"]: @@ -279,7 +278,6 @@ def __purgeSites(self, ceBdiiDict): return def __updateCS(self, bdiiChangeSet): - queueVODict = {} changeSet = set() for entry in bdiiChangeSet: @@ -317,7 +315,7 @@ def __updateCS(self, bdiiChangeSet): if not result["OK"]: self.log.error("Error while committing to CS", result["Message"]) else: - self.log.info("Successfully committed %d changes to CS" % len(changeList)) + self.log.info(f"Successfully committed {len(changeList)} changes to CS") return result else: self.log.info("No changes found") diff --git a/src/DIRAC/ConfigurationSystem/Agent/GOCDB2CSAgent.py b/src/DIRAC/ConfigurationSystem/Agent/GOCDB2CSAgent.py index 8a4dc0fd991..bf0623a36c3 100644 --- a/src/DIRAC/ConfigurationSystem/Agent/GOCDB2CSAgent.py +++ b/src/DIRAC/ConfigurationSystem/Agent/GOCDB2CSAgent.py @@ -53,11 +53,11 @@ def execute(self): for option, functionCall in GOCDB2CSAgent._functionMap.items(): optionValue = self.am_getOption(option, True) if optionValue: - result = functionCall(self) + result = functionCall(self) # pylint: disable=too-many-function-args if not result["OK"]: - self.log.error("{}() failed with message: {}".format(functionCall.__name__, result["Message"])) + self.log.error(f"{functionCall.__name__}() failed with message: {result['Message']}") else: - self.log.info("Successfully executed %s" % functionCall.__name__) + self.log.info(f"Successfully executed {functionCall.__name__}") return S_OK() @@ -72,21 +72,21 @@ def updatePerfSONARConfiguration(self): # get endpoints result = self.__getPerfSONAREndpoints() if not result["OK"]: - log.error("__getPerfSONAREndpoints() failed with message: %s" % result["Message"]) + log.error(f"__getPerfSONAREndpoints() failed with message: {result['Message']}") return S_ERROR("Unable to fetch perfSONAR endpoints from GOCDB.") endpointList = result["Value"] # add DIRAC site name result = self.__addDIRACSiteName(endpointList) if not result["OK"]: - log.error("__addDIRACSiteName() failed with message: %s" % result["Message"]) + log.error(f"__addDIRACSiteName() failed with message: {result['Message']}") return S_ERROR("Unable to extend the list with DIRAC site names.") extendedEndpointList = result["Value"] # prepare dictionary with new configuration result = self.__preparePerfSONARConfiguration(extendedEndpointList) if not result["OK"]: - log.error("__preparePerfSONARConfiguration() failed with message: %s" % result["Message"]) + log.error(f"__preparePerfSONARConfiguration() failed with message: {result['Message']}") return S_ERROR("Unable to prepare a new perfSONAR configuration.") finalConfiguration = result["Value"] @@ -110,16 +110,16 @@ def __getPerfSONAREndpoints(self): # get perfSONAR endpoints (latency and bandwidth) form GOCDB endpointList = [] for endpointType in ["Latency", "Bandwidth"]: - result = self.GOCDBClient.getServiceEndpointInfo("service_type", "net.perfSONAR.%s" % endpointType) + result = self.GOCDBClient.getServiceEndpointInfo("service_type", f"net.perfSONAR.{endpointType}") if not result["OK"]: - log.error("getServiceEndpointInfo() failed with message: %s" % result["Message"]) - return S_ERROR("Could not fetch %s endpoints from GOCDB" % endpointType.lower()) + log.error(f"getServiceEndpointInfo() failed with message: {result['Message']}") + return S_ERROR(f"Could not fetch {endpointType.lower()} endpoints from GOCDB") - log.debug("Number of {} endpoints: {}".format(endpointType.lower(), len(result["Value"]))) + log.debug(f"Number of {endpointType.lower()} endpoints: {len(result['Value'])}") endpointList.extend(result["Value"]) - log.debug("Number of perfSONAR endpoints: %s" % len(endpointList)) + log.debug(f"Number of perfSONAR endpoints: {len(endpointList)}") log.debug("End function.") return S_OK(endpointList) @@ -157,7 +157,7 @@ def __preparePerfSONARConfiguration(self, endpointList): for option in options: result = gConfig.getConfigurationTree(rootPath, extPath + "/", "/" + option) if not result["OK"]: - log.error("getConfigurationTree() failed with message: %s" % result["Message"]) + log.error(f"getConfigurationTree() failed with message: {result['Message']}") return S_ERROR("Unable to fetch perfSONAR endpoints from CS.") currentConfiguration.update(result["Value"]) @@ -175,10 +175,10 @@ def __preparePerfSONARConfiguration(self, endpointList): # inform what will be changed if addedEndpoints > 0: - self.log.info("%s new perfSONAR endpoints will be added to the configuration" % addedEndpoints) + self.log.info(f"{addedEndpoints} new perfSONAR endpoints will be added to the configuration") if disabledEndpoints > 0: - self.log.info("%s old perfSONAR endpoints will be disable in the configuration" % disabledEndpoints) + self.log.info(f"{disabledEndpoints} old perfSONAR endpoints will be disable in the configuration") if addedEndpoints == 0 and disabledEndpoints == 0: self.log.info("perfSONAR configuration is up-to-date") @@ -201,7 +201,7 @@ def __addDIRACSiteName(self, inputList): # get site name dictionary result = getDIRACGOCDictionary() if not result["OK"]: - log.error("getDIRACGOCDictionary() failed with message: %s" % result["Message"]) + log.error(f"getDIRACGOCDictionary() failed with message: {result['Message']}") return S_ERROR("Could not get site name dictionary") # reverse the dictionary (assume 1 to 1 relation) @@ -214,7 +214,7 @@ def __addDIRACSiteName(self, inputList): try: entry["DIRACSITENAME"] = GOCDIRACDict[entry["SITENAME"]] except KeyError: - self.log.warn("No dictionary entry for %s. " % entry["SITENAME"]) + self.log.warn(f"No dictionary entry for {entry['SITENAME']}. ") entry["DIRACSITENAME"] = None outputList.append(entry) @@ -235,7 +235,6 @@ def __updateConfiguration(self, setElements=None, delElements=None): # assure existence and proper value of a section or an option for path, value in setElements.items(): - if value is None: section = path else: @@ -245,27 +244,27 @@ def __updateConfiguration(self, setElements=None, delElements=None): try: result = self.csAPI.createSection(section) if not result["OK"]: - log.error("createSection() failed with message: %s" % result["Message"]) + log.error(f"createSection() failed with message: {result['Message']}") except Exception as e: - log.error("Exception in createSection(): %s" % repr(e).replace(",)", ")")) + log.error(f"Exception in createSection(): {repr(e).replace(',)', ')')}") if value is not None: try: result = self.csAPI.setOption(path, value) if not result["OK"]: - log.error("setOption() failed with message: %s" % result["Message"]) + log.error(f"setOption() failed with message: {result['Message']}") except Exception as e: - log.error("Exception in setOption(): %s" % repr(e).replace(",)", ")")) + log.error(f"Exception in setOption(): {repr(e).replace(',)', ')')}") # delete elements in the configuration for path in delElements: result = self.csAPI.delOption(path) if not result["OK"]: - log.warn("delOption() failed with message: %s" % result["Message"]) + log.warn(f"delOption() failed with message: {result['Message']}") result = self.csAPI.delSection(path) if not result["OK"]: - log.warn("delSection() failed with message: %s" % result["Message"]) + log.warn(f"delSection() failed with message: {result['Message']}") if self.dryRun: log.info("Dry Run: CS won't be updated") @@ -274,7 +273,7 @@ def __updateConfiguration(self, setElements=None, delElements=None): # update configuration stored by CS result = self.csAPI.commit() if not result["OK"]: - log.error("commit() failed with message: %s" % result["Message"]) + log.error(f"commit() failed with message: {result['Message']}") return S_ERROR("Could not commit changes to CS.") else: log.info("Committed changes to CS") diff --git a/src/DIRAC/ConfigurationSystem/Agent/RucioSynchronizerAgent.py b/src/DIRAC/ConfigurationSystem/Agent/RucioSynchronizerAgent.py index 0747937159a..fdf14b5762e 100644 --- a/src/DIRAC/ConfigurationSystem/Agent/RucioSynchronizerAgent.py +++ b/src/DIRAC/ConfigurationSystem/Agent/RucioSynchronizerAgent.py @@ -45,7 +45,7 @@ def getStorageElements(vo): :param vo: VO name that an SE supports :return: S_OK/S_ERROR, Value dictionary with key SE and value protocol list """ - log = gLogger.getLocalSubLogger("RucioSynchronizer/%s" % vo) + log = gLogger.getLocalSubLogger(f"RucioSynchronizer/{vo}") seProtocols = {} dms = DMSHelpers(vo=vo) for seName in dms.getStorageElements(): @@ -116,7 +116,7 @@ def getStorageElements(vo): space_token = value if params["scheme"] == "srm" and key == "WSUrl": params["extended_attributes"] = { - "web_service_path": "%s" % value, + "web_service_path": f"{value}", "space_token": space_token, } if key == "Protocol": @@ -174,7 +174,7 @@ def configHelper(voList): if result["OK"]: catSections = set(result["Value"]) else: - log.warn("No Services/Catalogs section in Operations, for ", "VO=%s (skipped)" % vo) + log.warn("No Services/Catalogs section in Operations, for ", f"VO={vo} (skipped)") continue selectedCatalog = list(catSections.intersection(catNames)) @@ -187,7 +187,7 @@ def configHelper(voList): continue if not selectedCatalog: - log.warn("VO is not using RucioFileCatalog (VO skipped)", "[VO: %s]" % vo) + log.warn("VO is not using RucioFileCatalog (VO skipped)", f"[VO: {vo}]") continue # check if the section name is in the catalog list to use. @@ -195,7 +195,7 @@ def configHelper(voList): fileCatalogs = opHelper.getValue("/Services/Catalogs/CatalogList", []) if fileCatalogs and selectedCatalog[0] not in fileCatalogs: - log.warn("VO is not using RucioFileCatalog - it is not in the catalog list", "[VO: %s]" % vo) + log.warn("VO is not using RucioFileCatalog - it is not in the catalog list", f"[VO: {vo}]") continue # now collect Rucio specific parameters for the VO params = {} @@ -367,9 +367,7 @@ def executeForVO(self, vo): try: client.add_protocol(rse=se, params=params) except Duplicate as err: - self.log.info( - "Protocol already exists on", "[RSE: {}, schema:{}]".format(se, params["scheme"]) - ) + self.log.info("Protocol already exists on", f"[RSE: {se}, schema:{params['scheme']}]") except Exception as err: self.log.error("Cannot create protocol on RSE", f"[RSE: {se}, Error: {str(err)}]") else: @@ -462,7 +460,7 @@ def executeForVO(self, vo): result = Operations().getSections("Shares") if result["OK"]: for dataLevel in result["Value"]: - result = Operations().getOptionsDict("Shares/%s" % dataLevel) + result = Operations().getOptionsDict(f"Shares/{dataLevel}") if not result["OK"]: self.log.error("Cannot get SEs:" % result["Message"]) continue @@ -470,7 +468,7 @@ def executeForVO(self, vo): for rse in rses: try: self.log.info("Setting", f"{dataLevel}Share for {rse} : {rseDict.get(rse, 0)}") - client.add_rse_attribute(rse, "%sShare" % dataLevel, rseDict.get(rse, 0)) + client.add_rse_attribute(rse, f"{dataLevel}Share", rseDict.get(rse, 0)) except Exception as err: self.log.error("Cannot create share:", "%sShare for %s", dataLevel, rse) else: @@ -561,9 +559,9 @@ def executeForVO(self, vo): scope = "user." + account if scope not in listScopes: try: - self.log.info("Will create a scope", "[Scope: %s]" % scope) + self.log.info("Will create a scope", f"[Scope: {scope}]") client.add_scope(account, scope) - self.log.info("Scope successfully added", "[Scope: %s]" % scope) + self.log.info("Scope successfully added", f"[Scope: {scope}]") except Exception as err: self.log.error("Cannot create a scope", f"[Scope: {scope}, Error: {str(err)}]") @@ -574,7 +572,7 @@ def executeForVO(self, vo): self.log.debug(" Will consider following Dirac groups for", f"[{vo} VO: {groups}]") else: groups = [] - self.log.debug("No Dirac groups for", "%s VO " % vo) + self.log.debug("No Dirac groups for", f"{vo} VO ") self.log.debug("No Rucio service accounts will be created") for group in groups: if group not in listAccounts: @@ -604,7 +602,7 @@ def executeForVO(self, vo): # Collect the group accounts from Dirac Configuration and create service accounts in Rucio result = getHosts() if not result["OK"]: - self.log.error("Cannot get host accounts:", "%s" % result["Message"]) + self.log.error("Cannot get host accounts:", f"{result['Message']}") else: hosts = result["Value"] for host in hosts: @@ -622,5 +620,5 @@ def executeForVO(self, vo): return S_OK() except Exception as exc: - self.log.exception("Synchronisation for VO failed. VO skipped ", "VO=%s" % vo, lException=exc) + self.log.exception("Synchronisation for VO failed. VO skipped ", f"VO={vo}", lException=exc) return S_ERROR(str(format_exc())) diff --git a/src/DIRAC/ConfigurationSystem/Agent/VOMS2CSAgent.py b/src/DIRAC/ConfigurationSystem/Agent/VOMS2CSAgent.py index 985b6454ccb..4df5742ec1e 100644 --- a/src/DIRAC/ConfigurationSystem/Agent/VOMS2CSAgent.py +++ b/src/DIRAC/ConfigurationSystem/Agent/VOMS2CSAgent.py @@ -25,12 +25,15 @@ corresponding options defined in the ``/Registry/VO/`` configuration section. """ + + from DIRAC import S_OK, gConfig, S_ERROR from DIRAC.Core.Base.AgentModule import AgentModule from DIRAC.Core.Utilities.Proxy import executeWithUserProxy from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOOption, getUserOption from DIRAC.ConfigurationSystem.Client.VOMS2CSSynchronizer import VOMS2CSSynchronizer from DIRAC.FrameworkSystem.Client.NotificationClient import NotificationClient +from DIRAC.FrameworkSystem.Client.TokenManagerClient import gTokenManager from DIRAC.Resources.Catalog.FileCatalog import FileCatalog @@ -50,6 +53,9 @@ def __init__(self, *args, **kwargs): self.autoLiftSuspendedStatus = True self.mailFrom = "noreply@dirac.system" self.syncPluginName = None + self.compareWithIAM = False + self.useIAM = False + self.forceNickname = False def initialize(self): """Initialize the default parameters""" @@ -63,6 +69,9 @@ def initialize(self): self.autoLiftSuspendedStatus = self.am_getOption("AutoLiftSuspendedStatus", self.autoLiftSuspendedStatus) self.makeFCEntry = self.am_getOption("MakeHomeDirectory", self.makeFCEntry) self.syncPluginName = self.am_getOption("SyncPluginName", self.syncPluginName) + self.compareWithIAM = self.am_getOption("CompareWithIAM", self.compareWithIAM) + self.useIAM = self.am_getOption("UseIAM", self.useIAM) + self.forceNickname = self.am_getOption("ForceNickname", self.forceNickname) self.detailedReport = self.am_getOption("DetailedReport", self.detailedReport) self.mailFrom = self.am_getOption("MailFrom", self.mailFrom) @@ -80,7 +89,6 @@ def initialize(self): return S_OK() def execute(self): - for vo in self.voList: voAdminUser = getVOOption(vo, "VOAdmin") voAdminMail = None @@ -96,6 +104,21 @@ def execute(self): autoLiftSuspendedStatus = getVOOption(vo, "AutoLiftSuspendedStatus", self.autoLiftSuspendedStatus) syncPluginName = getVOOption(vo, "SyncPluginName", self.syncPluginName) + compareWithIAM = getVOOption(vo, "CompareWithIAM", self.compareWithIAM) + useIAM = getVOOption(vo, "UseIAM", self.useIAM) + + accessToken = None + if compareWithIAM or useIAM: + res = gTokenManager.getToken( + userGroup=voAdminGroup, + requiredTimeLeft=3600, + scope=["scim:read"], + ) + if not res["OK"]: + return res + + accessToken = res["Value"]["access_token"] + vomsSync = VOMS2CSSynchronizer( vo, autoAddUsers=autoAddUsers, @@ -103,6 +126,10 @@ def execute(self): autoDeleteUsers=autoDeleteUsers, autoLiftSuspendedStatus=autoLiftSuspendedStatus, syncPluginName=syncPluginName, + compareWithIAM=compareWithIAM, + useIAM=useIAM, + accessToken=accessToken, + forceNickname=self.forceNickname, ) result = self.__syncCSWithVOMS( # pylint: disable=unexpected-keyword-arg @@ -111,9 +138,7 @@ def execute(self): proxyUserGroup=voAdminGroup, ) if not result["OK"]: - self.log.error( - "Failed to perform VOMS to CS synchronization:", "VO {}: {}".format(vo, result["Message"]) - ) + self.log.error("Failed to perform VOMS to CS synchronization:", f"VO {vo}: {result['Message']}") continue resultDict = result["Value"] newUsers = resultDict.get("NewUsers", []) @@ -123,6 +148,7 @@ def execute(self): csapi = resultDict.get("CSAPI") adminMessages = resultDict.get("AdminMessages", {"Errors": [], "Info": []}) voChanged = resultDict.get("VOChanged", False) + noNickname = resultDict.get("NoNickname", []) self.log.info( "Run user results", ": new %d, modified %d, deleted %d, new/suspended %d" @@ -131,7 +157,7 @@ def execute(self): if csapi.csModified: # We have accumulated all the changes, commit them now - self.log.info("There are changes to the CS ready to be committed", "for VO %s" % vo) + self.log.info("There are changes to the CS ready to be committed", f"for VO {vo}") if self.dryRun: self.log.info("Dry Run: CS won't be updated") csapi.showDiff() @@ -140,9 +166,9 @@ def execute(self): if not result["OK"]: self.log.error("Could not commit configuration changes", result["Message"]) return result - self.log.notice("Configuration committed", "for VO %s" % vo) + self.log.notice("Configuration committed", f"for VO {vo}") else: - self.log.info("No changes to the CS recorded at this cycle", "for VO %s" % vo) + self.log.info("No changes to the CS recorded at this cycle", f"for VO {vo}") # Add user home directory in the file catalog if self.makeFCEntry and newUsers: @@ -154,24 +180,29 @@ def execute(self): proxyUserGroup=voAdminGroup, ) if not result["OK"]: - self.log.error("Failed to create user home directories:", "VO {}: {}".format(vo, result["Message"])) + self.log.error("Failed to create user home directories:", f"VO {vo}: {result['Message']}") else: for user in result["Value"]["Failed"]: self.log.error( "Failed to create home directory", - "user: {}, operation: {}".format(user, result["Value"]["Failed"][user]), + f"user: {user}, operation: {result['Value']['Failed'][user]}", ) adminMessages["Errors"].append( "Failed to create home directory for user %s: operation %s" % (user, result["Value"]["Failed"][user]) ) for user in result["Value"]["Successful"]: - adminMessages["Info"].append("Created home directory for user %s" % user) + adminMessages["Info"].append(f"Created home directory for user {user}") if voChanged or self.detailedReport: mailMsg = "" if adminMessages["Errors"]: mailMsg += "\nErrors list:\n %s" % "\n ".join(adminMessages["Errors"]) + if self.forceNickname and noNickname: + mailMsg += "There are users without nicknames in the IAM\n" + for entry in noNickname: + mailMsg += str(entry) + mailMsg += "\n\n" if adminMessages["Info"]: mailMsg += "\nRun result:\n %s" % "\n ".join(adminMessages["Info"]) if self.detailedReport: @@ -185,7 +216,7 @@ def execute(self): if self.dryRun: self.log.info("Dry Run: mail won't be sent") self.log.info(mailMsg) - else: + elif voChanged: NotificationClient().sendMail( self.am_getOption("MailTo", voAdminMail), "VOMS2CSAgent run log", mailMsg, self.mailFrom ) @@ -198,9 +229,8 @@ def __syncCSWithVOMS(self, vomsSync): @executeWithUserProxy def __addHomeDirectory(self, vo, newUsers): - fc = FileCatalog(vo=vo) - defaultVOGroup = getVOOption(vo, "DefaultGroup", "%s_user" % vo) + defaultVOGroup = getVOOption(vo, "DefaultGroup", f"{vo}_user") failed = {} successful = {} diff --git a/src/DIRAC/ConfigurationSystem/Agent/test/Test_Bdii2CS.py b/src/DIRAC/ConfigurationSystem/Agent/test/Test_Bdii2CS.py index 3c7de2e02f5..06de61e7329 100644 --- a/src/DIRAC/ConfigurationSystem/Agent/test/Test_Bdii2CS.py +++ b/src/DIRAC/ConfigurationSystem/Agent/test/Test_Bdii2CS.py @@ -37,7 +37,6 @@ def tearDown(self): pass def test__getGlue2CEInfo_success(self): - expectedResult = {} expectedResult.update(ALTBDII) expectedResult.update(MAINBDII) @@ -61,7 +60,6 @@ def test__getGlue2CEInfo_success(self): self.assertNotIn("ce2b", ret["Value"]["site2"]["CEs"]) def test__getGlue2CEInfo_fail_10(self): - self.agent.alternativeBDIIs = ["server2"] with patch( MODNAME + ".getGlue2CEInfo", @@ -86,7 +84,6 @@ def test__getGlue2CEInfo_fail_10(self): self.assertEqual(ALTBDII, ret["Value"]) def test__getGlue2CEInfo_fail_01(self): - self.agent.alternativeBDIIs = ["server2"] with patch( MODNAME + ".getGlue2CEInfo", @@ -111,7 +108,6 @@ def test__getGlue2CEInfo_fail_01(self): self.assertEqual(MAINBDII, ret["Value"]) def test__getGlue2CEInfo_fail_11(self): - self.agent.alternativeBDIIs = ["server2"] with patch( MODNAME + ".getGlue2CEInfo", diff --git a/src/DIRAC/ConfigurationSystem/Client/CSAPI.py b/src/DIRAC/ConfigurationSystem/Client/CSAPI.py index 11ce7978a93..6a1c4d7a366 100644 --- a/src/DIRAC/ConfigurationSystem/Client/CSAPI.py +++ b/src/DIRAC/ConfigurationSystem/Client/CSAPI.py @@ -5,16 +5,16 @@ import datetime -from DIRAC import gLogger, gConfig, S_OK, S_ERROR +from DIRAC import S_ERROR, S_OK, gConfig, gLogger from DIRAC.ConfigurationSystem.Client.ConfigurationClient import ConfigurationClient -from DIRAC.Core.Utilities import List -from DIRAC.Core.Security.X509Chain import X509Chain # pylint: disable=import-error -from DIRAC.Core.Security import Locations -from DIRAC.ConfigurationSystem.private.Modificator import Modificator from DIRAC.ConfigurationSystem.Client.Helpers import CSGlobals from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations -from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSites, getCESiteMapping from DIRAC.ConfigurationSystem.Client.Helpers.Path import cfgPath +from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getCESiteMapping, getSites +from DIRAC.ConfigurationSystem.private.Modificator import Modificator +from DIRAC.Core.Security import Locations +from DIRAC.Core.Security.X509Chain import X509Chain # pylint: disable=import-error +from DIRAC.Core.Utilities import List class CSAPI: @@ -98,7 +98,7 @@ def initialize(self): return self.__initialized retVal = gConfig.getOption("/DIRAC/Configuration/MasterServer") if not retVal["OK"]: - self.__initialized = S_ERROR("Master server is not known. Is everything initialized?") + self.__initialized = S_ERROR("Controller server is not known. Is everything initialized?") return self.__initialized self.__rpcClient = ConfigurationClient(url=gConfig.getValue("/DIRAC/Configuration/MasterServer", "")) self.__csMod = Modificator( @@ -239,7 +239,7 @@ def listUsers(self, group=False): if not self.__initialized["OK"]: return self.__initialized if not group: - return S_OK(self.__csMod.getSections("%s/Users" % self.__baseSecurity)) + return S_OK(self.__csMod.getSections(f"{self.__baseSecurity}/Users")) users = self.__csMod.getValue(f"{self.__baseSecurity}/Groups/{group}/Users") return S_OK(List.fromChar(users) if users else []) @@ -250,7 +250,7 @@ def listHosts(self): """ if not self.__initialized["OK"]: return self.__initialized - return S_OK(self.__csMod.getSections("%s/Hosts" % self.__baseSecurity)) + return S_OK(self.__csMod.getSections(f"{self.__baseSecurity}/Hosts")) def describeUsers(self, users=None): """describe users by nickname @@ -280,7 +280,7 @@ def __describeEntity(self, mask, hosts=False): """ if not self.__initialized["OK"]: return self.__initialized - csSection = "{}/{}".format(self.__baseSecurity, ("Hosts" if hosts else "Users")) + csSection = f"{self.__baseSecurity}/{'Hosts' if hosts else 'Users'}" entities = self.__csMod.getSections(csSection) if mask: entities = [entity for entity in entities if entity in (mask or [])] @@ -307,7 +307,7 @@ def listGroups(self): """ if not self.__initialized["OK"]: return self.__initialized - return S_OK(self.__csMod.getSections("%s/Groups" % self.__baseSecurity)) + return S_OK(self.__csMod.getSections(f"{self.__baseSecurity}/Groups")) def describeGroups(self, mask=None): """ @@ -319,7 +319,7 @@ def describeGroups(self, mask=None): return self.__initialized groups = [ group - for group in self.__csMod.getSections("%s/Groups" % self.__baseSecurity) + for group in self.__csMod.getSections(f"{self.__baseSecurity}/Groups") if not mask or (mask and group in mask) ] groupsDict = {} @@ -348,14 +348,14 @@ def deleteUsers(self, users): usersData = result["Value"] for username in users: if username not in usersData: - gLogger.warn("User %s does not exist" % username) + gLogger.warn(f"User {username} does not exist") continue userGroups = usersData[username]["Groups"] for group in userGroups: self.__removeUserFromGroup(group, username) gLogger.info(f"Deleted user {username} from group {group}") self.__csMod.removeSection(f"{self.__baseSecurity}/Users/{username}") - gLogger.info("Deleted user %s" % username) + gLogger.info(f"Deleted user {username}") self.csModified = True return S_OK(True) @@ -424,7 +424,7 @@ def addUser(self, username, properties): for userGroup in properties["Groups"]: gLogger.info(f"Added user {username} to group {userGroup}") self.__addUserToGroup(userGroup, username) - gLogger.info("Registered user %s" % username) + gLogger.info(f"Registered user {username}") self.csModified = True return S_OK(True) @@ -451,7 +451,7 @@ def modifyUser(self, username, properties, createIfNonExistant=False): userData = result["Value"] if username not in userData: if createIfNonExistant: - gLogger.info("Registering user %s" % username) + gLogger.info(f"Registering user {username}") return self.addUser(username, properties) gLogger.error("User is not registered: ", repr(username)) return S_OK(False) @@ -491,10 +491,10 @@ def modifyUser(self, username, properties, createIfNonExistant=False): modified = False if modifiedUser: modified = True - gLogger.info("Modified user %s" % username) + gLogger.info(f"Modified user {username}") self.csModified = True else: - gLogger.info("Nothing to modify for user %s" % username) + gLogger.info(f"Nothing to modify for user {username}") return S_OK(modified) def addGroup(self, groupname, properties): @@ -521,7 +521,7 @@ def addGroup(self, groupname, properties): self.__csMod.createSection(f"{self.__baseSecurity}/Groups/{groupname}") for prop in properties: self.__csMod.setOptionValue(f"{self.__baseSecurity}/Groups/{groupname}/{prop}", properties[prop]) - gLogger.info("Registered group %s" % groupname) + gLogger.info(f"Registered group {groupname}") self.csModified = True return S_OK(True) @@ -548,7 +548,7 @@ def modifyGroup(self, groupname, properties, createIfNonExistant=False): groupData = result["Value"] if groupname not in groupData: if createIfNonExistant: - gLogger.info("Registering group %s" % groupname) + gLogger.info(f"Registering group {groupname}") return self.addGroup(groupname, properties) gLogger.error("Group is not registered", groupname) return S_OK(False) @@ -559,10 +559,10 @@ def modifyGroup(self, groupname, properties, createIfNonExistant=False): self.__csMod.setOptionValue(f"{self.__baseSecurity}/Groups/{groupname}/{prop}", properties[prop]) modifiedGroup = True if modifiedGroup: - gLogger.info("Modified group %s" % groupname) + gLogger.info(f"Modified group {groupname}") self.csModified = True else: - gLogger.info("Nothing to modify for group %s" % groupname) + gLogger.info(f"Nothing to modify for group {groupname}") return S_OK(True) def addHost(self, hostname, properties): @@ -593,7 +593,7 @@ def addHost(self, hostname, properties): self.__csMod.createSection(f"{self.__baseSecurity}/Hosts/{hostname}") for prop in properties: self.__csMod.setOptionValue(f"{self.__baseSecurity}/Hosts/{hostname}/{prop}", properties[prop]) - gLogger.info("Registered host %s" % hostname) + gLogger.info(f"Registered host {hostname}") self.csModified = True return S_OK(True) @@ -612,25 +612,15 @@ def getOpsSection(): Where is the shifters section? """ vo = CSGlobals.getVO() - setup = CSGlobals.getSetup() if vo: - res = gConfig.getSections(f"/Operations/{vo}/{setup}/Shifter") - if res["OK"]: - return S_OK(f"/Operations/{vo}/{setup}/Shifter") - - res = gConfig.getSections("/Operations/%s/Defaults/Shifter" % vo) + res = gConfig.getSections(f"/Operations/{vo}/Shifter") if res["OK"]: - return S_OK("/Operations/%s/Defaults/Shifter" % vo) + return S_OK(f"/Operations/{vo}/Shifter") - else: - res = gConfig.getSections("/Operations/%s/Shifter" % setup) - if res["OK"]: - return S_OK("/Operations/%s/Shifter" % setup) - - res = gConfig.getSections("/Operations/Defaults/Shifter") - if res["OK"]: - return S_OK("/Operations/Defaults/Shifter") + res = gConfig.getSections("/Operations/Defaults/Shifter") + if res["OK"]: + return S_OK("/Operations/Defaults/Shifter") return S_ERROR("No shifter section") @@ -649,7 +639,7 @@ def getOpsSection(): currentShifterRoles = currentShifterRoles["Value"] currentShiftersDict = {} for currentShifterRole in currentShifterRoles: - currentShifter = opsH.getOptionsDict("Shifter/%s" % currentShifterRole) + currentShifter = opsH.getOptionsDict(f"Shifter/{currentShifterRole}") if not currentShifter["OK"]: return currentShifter currentShifter = currentShifter["Value"] @@ -671,13 +661,13 @@ def getOpsSection(): gLogger.info("Adding shifter section") vo = CSGlobals.getVO() if vo: - section = "/Operations/%s/Defaults/Shifter" % vo + section = f"/Operations/{vo}/Shifter" else: section = "/Operations/Defaults/Shifter" res = self.__csMod.createSection(section) if not res: - gLogger.error("Section %s not created" % section) - return S_ERROR("Section %s not created" % section) + gLogger.error(f"Section {section} not created") + return S_ERROR(f"Section {section} not created") else: gLogger.error(section["Message"]) return section @@ -719,7 +709,7 @@ def modifyHost(self, hostname, properties, createIfNonExistant=False): hostData = result["Value"] if hostname not in hostData: if createIfNonExistant: - gLogger.info("Registering host %s" % hostname) + gLogger.info(f"Registering host {hostname}") return self.addHost(hostname, properties) gLogger.error("Host is not registered", hostname) return S_OK(False) @@ -730,10 +720,10 @@ def modifyHost(self, hostname, properties, createIfNonExistant=False): self.__csMod.setOptionValue(f"{self.__baseSecurity}/Hosts/{hostname}/{prop}", properties[prop]) modifiedHost = True if modifiedHost: - gLogger.info("Modified host %s" % hostname) + gLogger.info(f"Modified host {hostname}") self.csModified = True else: - gLogger.info("Nothing to modify for host %s" % hostname) + gLogger.info(f"Nothing to modify for host {hostname}") return S_OK(True) def syncUsersWithCFG(self, usersCFG): @@ -757,9 +747,9 @@ def syncUsersWithCFG(self, usersCFG): return S_OK(done) def sortUsersAndGroups(self): - self.__csMod.sortAlphabetically("%s/Users" % self.__baseSecurity) - self.__csMod.sortAlphabetically("%s/Hosts" % self.__baseSecurity) - for group in self.__csMod.getSections("%s/Groups" % self.__baseSecurity): + self.__csMod.sortAlphabetically(f"{self.__baseSecurity}/Users") + self.__csMod.sortAlphabetically(f"{self.__baseSecurity}/Hosts") + for group in self.__csMod.getSections(f"{self.__baseSecurity}/Groups"): usersOptionPath = f"{self.__baseSecurity}/Groups/{group}/Users" users = self.__csMod.getValue(usersOptionPath) if users: @@ -769,8 +759,8 @@ def sortUsersAndGroups(self): self.__csMod.setOptionValue(usersOptionPath, sortedUsers) def checkForUnexistantUsersInGroups(self): - allUsers = self.__csMod.getSections("%s/Users" % self.__baseSecurity) - allGroups = self.__csMod.getSections("%s/Groups" % self.__baseSecurity) + allUsers = self.__csMod.getSections(f"{self.__baseSecurity}/Users") + allGroups = self.__csMod.getSections(f"{self.__baseSecurity}/Groups") for group in allGroups: usersInGroup = self.__csMod.getValue(f"{self.__baseSecurity}/Groups/{group}/Users") if usersInGroup: @@ -790,7 +780,7 @@ def commitChanges(self, sortUsers=True): self.sortUsersAndGroups() retVal = self.__csMod.commit() if not retVal["OK"]: - gLogger.error("Can't commit new configuration data", "%s" % retVal["Message"]) + gLogger.error("Can't commit new configuration data", f"{retVal['Message']}") return retVal return self.downloadCSData() return S_OK() @@ -802,7 +792,7 @@ def commit(self): if self.csModified: retVal = self.__csMod.commit() if not retVal["OK"]: - gLogger.error("Can't commit new configuration data", "%s" % retVal["Message"]) + gLogger.error("Can't commit new configuration data", f"{retVal['Message']}") return retVal return self.downloadCSData() return S_OK() @@ -825,7 +815,7 @@ def modifyValue(self, optionPath, newValue): gLogger.verbose(f"Changing {optionPath} from \n{prevVal} \nto \n{newValue}") self.__csMod.setOptionValue(optionPath, newValue) self.csModified = True - return S_OK("Modified %s" % optionPath) + return S_OK(f"Modified {optionPath}") def setOption(self, optionPath, optionValue): """Create an option at the specified path.""" @@ -848,9 +838,9 @@ def delOption(self, optionPath): if not self.__initialized["OK"]: return self.__initialized if not self.__csMod.removeOption(optionPath): - return S_ERROR("Couldn't delete option %s" % optionPath) + return S_ERROR(f"Couldn't delete option {optionPath}") self.csModified = True - return S_OK("Deleted option %s" % optionPath) + return S_OK(f"Deleted option {optionPath}") def createSection(self, sectionPath, comment=""): """Create a new section""" @@ -867,7 +857,7 @@ def delSection(self, sectionPath): if not self.__initialized["OK"]: return self.__initialized if not self.__csMod.removeSection(sectionPath): - return S_ERROR("Could not delete section %s " % sectionPath) + return S_ERROR(f"Could not delete section {sectionPath} ") self.csModified = True return S_OK() @@ -881,7 +871,7 @@ def copySection(self, originalPath, targetPath): if not result["OK"]: return result if not self.__csMod.mergeSectionFromCFG(targetPath, sectionCfg): - return S_ERROR("Could not merge cfg into section %s" % targetPath) + return S_ERROR(f"Could not merge cfg into section {targetPath}") self.csModified = True return S_OK() @@ -904,7 +894,7 @@ def mergeCFGUnderSection(self, sectionPath, cfg): if not result["OK"]: return result if not self.__csMod.mergeSectionFromCFG(sectionPath, cfg): - return S_ERROR("Could not merge cfg into section %s" % sectionPath) + return S_ERROR(f"Could not merge cfg into section {sectionPath}") self.csModified = True return S_OK() @@ -925,7 +915,7 @@ def getCurrentCFG(self): def showDiff(self): """Just shows the differences accumulated within the Modificator object""" diffData = self.__csMod.showCurrentDiff() - gLogger.notice("Accumulated diff with master CS") + gLogger.notice("Accumulated diff with Controller CS") for line in diffData: if line[0] in ("+", "-"): gLogger.notice(line) diff --git a/src/DIRAC/ConfigurationSystem/Client/CSCLI.py b/src/DIRAC/ConfigurationSystem/Client/CSCLI.py index 8e1be84ae3a..46b8b602c9b 100644 --- a/src/DIRAC/ConfigurationSystem/Client/CSCLI.py +++ b/src/DIRAC/ConfigurationSystem/Client/CSCLI.py @@ -29,21 +29,21 @@ def _showTraceback(): def _printComment(comment): commentList = comment.split("\n") for commentLine in commentList[:-1]: - print("# %s" % commentLine.strip()) + print(f"# {commentLine.strip()}") def _appendExtensionIfMissing(filename): dotPosition = filename.rfind(".") if dotPosition > -1: filename = filename[:dotPosition] - return "%s.cfg" % filename + return f"{filename}.cfg" class CSCLI(CLI): def __init__(self): CLI.__init__(self) self.connected = False - self.masterURL = "unset" + self.controllerURL = "unset" self.writeEnabled = False self.modifiedData = False self.rpcClient = None @@ -56,7 +56,7 @@ def __init__(self): self.backupFilename = "dataChanges" # store history histfilename = os.path.basename(sys.argv[0]) - historyFile = os.path.expanduser("~/.dirac/%s.history" % histfilename[0:-3]) + historyFile = os.path.expanduser(f"~/.dirac/{histfilename[0:-3]}.history") mkDir(os.path.dirname(historyFile)) if os.path.isfile(historyFile): readline.read_history_file(historyFile) @@ -83,11 +83,11 @@ def _setConnected(self, connected, writeEnabled): self.writeEnabled = writeEnabled if connected: if writeEnabled: - self.prompt = "({})-{}> ".format(self.masterURL, colorize("Connected", "green")) + self.prompt = f"({self.controllerURL})-{colorize('Connected', 'green')}> " else: - self.prompt = "({})-{}> ".format(self.masterURL, colorize("Connected (RO)", "yellow")) + self.prompt = f"({self.controllerURL})-{colorize('Connected (RO)', 'yellow')}> " else: - self.prompt = "({})-{}> ".format(self.masterURL, colorize("Disconnected", "red")) + self.prompt = f"({self.controllerURL})-{colorize('Disconnected', 'red')}> " def do_quit(self, dummy): """ @@ -99,12 +99,12 @@ def do_quit(self, dummy): if self.modifiedData: print("Changes are about to be written to file for later use.") self.do_writeToFile(self.backupFilename) - print("Changes written to %s.cfg" % self.backupFilename) + print(f"Changes written to {self.backupFilename}.cfg") sys.exit(0) def _setStatus(self, connected=True): if not connected: - self.masterURL = "unset" + self.controllerURL = "unset" self._setConnected(False, False) else: retVal = self.rpcClient.writeEnabled() @@ -114,27 +114,27 @@ def _setStatus(self, connected=True): else: self._setConnected(True, False) else: - print("Server returned an error: %s" % retVal["Message"]) + print(f"Server returned an error: {retVal['Message']}") self._setConnected(True, False) def _tryConnection(self): - print("Trying connection to %s" % self.masterURL) + print(f"Trying connection to {self.controllerURL}") try: - self.rpcClient = ConfigurationClient(url=self.masterURL) + self.rpcClient = ConfigurationClient(url=self.controllerURL) self._setStatus() except Exception as x: - gLogger.error("Couldn't connect to master CS server", f"{self.masterURL} ({str(x)})") + gLogger.error("Couldn't connect to controller CS server", f"{self.controllerURL} ({str(x)})") self._setStatus(False) def do_connect(self, args=""): """ - Connects to configuration master server (in specified url if provided). + Connects to configuration controller server (in specified url if provided). Usage: connect """ if not args or not isinstance(args, str): - self.masterURL = gConfigurationData.getMasterServer() - if self.masterURL != "unknown" and self.masterURL: + self.controllerURL = gConfigurationData.getMasterServer() + if self.controllerURL != "unknown" and self.controllerURL: self._tryConnection() else: self._setStatus(False) @@ -144,7 +144,7 @@ def do_connect(self, args=""): print("Must specify witch url to connect") self._setStatus(False) else: - self.masterURL = splitted[0].strip() + self.controllerURL = splitted[0].strip() self._tryConnection() def do_sections(self, args): @@ -161,11 +161,11 @@ def do_sections(self, args): else: baseSection = "/" if not self.modificator.existsSection(baseSection): - print("Section %s does not exist" % baseSection) + print(f"Section {baseSection} does not exist") return sectionList = self.modificator.getSections(baseSection) if not sectionList: - print("Section %s is empty" % baseSection) + print(f"Section {baseSection} is empty") return for section in sectionList: section = f"{baseSection}/{section}" @@ -187,11 +187,11 @@ def do_options(self, args): print("Which section?") return if not self.modificator.existsSection(section): - print("Section %s does not exist" % section) + print(f"Section {section} does not exist") return optionsList = self.modificator.getOptions(section) if not optionsList: - print("Section %s has no options" % section) + print(f"Section {section} has no options") return for option in optionsList: _printComment(self.modificator.getComment(f"{section}/{option}")) @@ -217,7 +217,7 @@ def do_get(self, args): _printComment(self.modificator.getComment(optionPath)) self.printPair(option, self.modificator.getValue(optionPath), "=") else: - print("Option %s does not exist" % optionPath) + print(f"Option {optionPath} does not exist") except Exception: _showTraceback() @@ -247,14 +247,14 @@ def do_writeToServer(self, dummy): choice = input("Do you really want to send changes to server? yes/no [no]: ") choice = choice.lower() if choice in ("yes", "y"): - print("Uploading changes to %s (It may take some seconds)..." % self.masterURL) + print(f"Uploading changes to {self.controllerURL} (It may take some seconds)...") response = self.modificator.commit() if response["OK"]: self.modifiedData = False print("Data sent to server.") self.modificator.loadFromRemote() else: - print("Error sending data, server said: %s" % response["Message"]) + print(f"Error sending data, server said: {response['Message']}") return else: print("Commit aborted") @@ -298,7 +298,7 @@ def do_removeOption(self, args): print("Must specify option to delete") return optionPath = argsList[0].strip() - choice = input("Are you sure you want to delete %s? yes/no [no]: " % optionPath) + choice = input(f"Are you sure you want to delete {optionPath}? yes/no [no]: ") choice = choice.lower() if choice in ("yes", "y", "true"): if self.modificator.removeOption(optionPath): @@ -308,7 +308,7 @@ def do_removeOption(self, args): else: print("Aborting removal.") except Exception as x: - print("Error removing option, %s" % str(x)) + print(f"Error removing option, {str(x)}") def do_removeSection(self, args): """ @@ -322,7 +322,7 @@ def do_removeSection(self, args): print("Must specify section to delete") return section = argsList[0].strip() - choice = input("Are you sure you want to delete %s? yes/no [no]: " % section) + choice = input(f"Are you sure you want to delete {section}? yes/no [no]: ") choice = choice.lower() if choice in ("yes", "y", "true"): if self.modificator.removeSection(section): @@ -332,7 +332,7 @@ def do_removeSection(self, args): else: print("Aborting removal.") except Exception as x: - print("Error removing section, %s" % str(x)) + print(f"Error removing section, {str(x)}") def do_setComment(self, args): """ @@ -433,7 +433,7 @@ def do_showHistory(self, args): if len(argsList) > 0: limit = int(argsList[0]) history = self.modificator.getHistory(limit) - print("%s recent commits:" % limit) + print(f"{limit} recent commits:") for entry in history: self.printPair(entry[0], entry[1], "@") except Exception: @@ -495,7 +495,7 @@ def do_rollbackToVersion(self, args): print("What version to rollback?") return version = " ".join(argsList[0:2]) - choice = input("Do you really want to rollback to version %s? yes/no [no]: " % version) + choice = input(f"Do you really want to rollback to version {version}? yes/no [no]: ") choice = choice.lower() if choice in ("yes", "y"): response = self.modificator.rollbackToVersion(version) @@ -504,7 +504,7 @@ def do_rollbackToVersion(self, args): print("Rolled back.") self.modificator.loadFromRemote() else: - print("Error sending data, server said: %s" % response["Message"]) + print(f"Error sending data, server said: {response['Message']}") except Exception: _showTraceback() diff --git a/src/DIRAC/ConfigurationSystem/Client/CSShellCLI.py b/src/DIRAC/ConfigurationSystem/Client/CSShellCLI.py index c716158c4ac..f2c04055f38 100644 --- a/src/DIRAC/ConfigurationSystem/Client/CSShellCLI.py +++ b/src/DIRAC/ConfigurationSystem/Client/CSShellCLI.py @@ -63,6 +63,13 @@ def do_connect(self, line): self.update_prompt() print("done.") + def do_reload(self, _line): + """reload + Reload contents of the remote configuration""" + result = self.modificator.loadFromRemote() + if not result["OK"]: + print("Reload failed: ", result["Message"]) + def do_disconnect(self, _line): """Disconnect from CS""" if self.connected and self.dirty: @@ -154,6 +161,13 @@ def do_rmdir(self, line): complete_rmdir = complete_cd + def do_sort(self, line): + """sort + Sort sections alphabetically""" + if self.connected: + self.modificator.sortAlphabetically(self.root, ascending=True) + self.dirty = True + def do_rm(self, line): """rm Delete an option in the CS""" @@ -193,7 +207,12 @@ def do_commit(self, _line): """commit Commit the modifications to the CS""" if self.connected and self.dirty: - self.modificator.commit() + result = self.modificator.commit() + if not result["OK"]: + print("Commit failed: ", result["Message"]) + # Reload to allow further commits + self.do_reload("") + self.dirty = False def default(self, line): """Override [Cmd.default(line)] function.""" diff --git a/src/DIRAC/ConfigurationSystem/Client/ConfigurationClient.py b/src/DIRAC/ConfigurationSystem/Client/ConfigurationClient.py index f0b60e8f81c..c3cb52b713e 100644 --- a/src/DIRAC/ConfigurationSystem/Client/ConfigurationClient.py +++ b/src/DIRAC/ConfigurationSystem/Client/ConfigurationClient.py @@ -63,7 +63,7 @@ class ConfigurationClient(Client): def __init__(self, **kwargs): # By default we use Configuration/Server as url, client do the resolution - # In some case url has to be static (when a slave register to the master server for example) + # In some case url has to be static (when a worker register to the controller server for example) # It's why we can use 'url' as keyword arguments if "url" not in kwargs: kwargs["url"] = "Configuration/Server" diff --git a/src/DIRAC/ConfigurationSystem/Client/Helpers/Local.py b/src/DIRAC/ConfigurationSystem/Client/Helpers/Local.py deleted file mode 100644 index 93800818994..00000000000 --- a/src/DIRAC/ConfigurationSystem/Client/Helpers/Local.py +++ /dev/null @@ -1,11 +0,0 @@ -from DIRAC import gConfig -from DIRAC.ConfigurationSystem.Client.Helpers.Path import cfgPath - -gBaseLocalSiteSection = "/LocalSite" - - -def gridEnv(): - """ - Return location of gridenv file to get a UI environment - """ - return gConfig.getValue(cfgPath(gBaseLocalSiteSection, "GridEnv"), "") diff --git a/src/DIRAC/ConfigurationSystem/Client/Helpers/Operations.py b/src/DIRAC/ConfigurationSystem/Client/Helpers/Operations.py index 8e3be488eb7..5796ee25278 100644 --- a/src/DIRAC/ConfigurationSystem/Client/Helpers/Operations.py +++ b/src/DIRAC/ConfigurationSystem/Client/Helpers/Operations.py @@ -6,56 +6,19 @@ Operations/ Defaults/ + someOption = someValue + aSecondOption = aSecondValue + specificVo/ someSection/ - someOption = someValue - aSecondOption = aSecondValue - Production/ - someSection/ - someOption = someValueInProduction - aSecondOption = aSecondValueInProduction - Certification/ - someSection/ - someOption = someValueInCertification + someOption = someValueInVO The following calls would give different results based on the setup:: Operations().getValue('someSection/someOption') - - someValueInProduction if we are in 'Production' setup - - someValueInCertification if we are in 'Certification' setup - - Operations().getValue('someSection/aSecondOption') - - aSecondValueInProduction if we are in 'Production' setup - - aSecondValue if we are in 'Certification' setup <- looking in Defaults - since there's no Certification/someSection/aSecondOption - + - someValueInVO if we are in 'specificVo' vo + - someValue if we are in any other VO - At the same time, for multi-VO installations, it is also possible to specify different options per-VO, - like the following:: - - Operations/ - aVOName/ - Defaults/ - someSection/ - someOption = someValue - aSecondOption = aSecondValue - Production/ - someSection/ - someOption = someValueInProduction - aSecondOption = aSecondValueInProduction - Certification/ - someSection/ - someOption = someValueInCertification - anotherVOName/ - Defaults/ - someSectionName/ - someOptionX = someValueX - aSecondOption = aSecondValue - setupName/ - someSection/ - someOption = someValueInProduction - aSecondOption = aSecondValueInProduction - - For this case it becomes then important for the Operations() objects to know the VO name + It becomes then important for the Operations() objects to know the VO name for which we want the information, and this can be done in the following ways. 1. by specifying the VO name directly:: @@ -70,12 +33,13 @@ but this works iff the object is instantiated by a proxy (and not, e.g., using a server certificate) """ -import os import _thread + from diraccfg import CFG -from DIRAC import S_OK, S_ERROR, gConfig -from DIRAC.ConfigurationSystem.Client.Helpers import Registry, CSGlobals + +from DIRAC import S_ERROR, S_OK from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData +from DIRAC.ConfigurationSystem.Client.Helpers import CSGlobals, Registry from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup from DIRAC.Core.Utilities import LockRing from DIRAC.Core.Utilities.DErrno import ESECTION @@ -98,9 +62,7 @@ def __init__(self, vo=False, group=False, setup=False): """ self.__uVO = vo self.__uGroup = group - self.__uSetup = setup self.__vo = False - self.__setup = False self.__discoverSettings() def __discoverSettings(self): @@ -119,12 +81,6 @@ def __discoverSettings(self): result = getVOfromProxyGroup() if result["OK"]: self.__vo = result["Value"] - # Set the setup - self.__setup = False - if self.__uSetup: - self.__setup = self.__uSetup - else: - self.__setup = CSGlobals.getSetup() def __getCache(self): Operations.__cacheLock.acquire() @@ -134,7 +90,7 @@ def __getCache(self): Operations.__cache = {} Operations.__cacheVersion = currentVersion - cacheKey = (self.__vo, self.__setup) + cacheKey = (self.__vo,) if cacheKey in Operations.__cache: return Operations.__cache[cacheKey] @@ -155,14 +111,13 @@ def __getCache(self): pass def __getSearchPaths(self): - paths = ["/Operations/Defaults", "/Operations/%s" % self.__setup] + paths = ["/Operations/Defaults"] if not self.__vo: globalVO = CSGlobals.getVO() if not globalVO: return paths self.__vo = CSGlobals.getVO() - paths.append("/Operations/%s/Defaults" % self.__vo) - paths.append(f"/Operations/{self.__vo}/{self.__setup}") + paths.append(f"/Operations/{self.__vo}/") return paths def getValue(self, optionPath, defaultValue=None): @@ -172,10 +127,10 @@ def __getCFG(self, sectionPath): cacheCFG = self.__getCache() section = cacheCFG.getRecursive(sectionPath) if not section: - return S_ERROR(ESECTION, "%s in Operations does not exist" % sectionPath) + return S_ERROR(ESECTION, f"{sectionPath} in Operations does not exist") sectionCFG = section["value"] if isinstance(sectionCFG, str): - return S_ERROR("%s in Operations is not a section" % sectionPath) + return S_ERROR(f"{sectionPath} in Operations is not a section") return S_OK(sectionCFG) def getSections(self, sectionPath, listOrdered=False): @@ -202,26 +157,6 @@ def getOptionsDict(self, sectionPath): data[opName] = sectionCFG[opName] return S_OK(data) - def getPath(self, option, vo=False, setup=False): - """ - Generate the CS path for an option: - - - if vo is not defined, the helper's vo will be used for multi VO installations - - if setup evaluates False (except None) -> The helpers setup will be used - - if setup is defined -> whatever is defined will be used as setup - - if setup is None -> Defaults will be used - - :param option: path with respect to the Operations standard path - :type option: string - """ - - for path in self.__getSearchPaths(): - optionPath = os.path.join(path, option) - value = gConfig.getValue(optionPath, "NoValue") - if value != "NoValue": - return optionPath - return "" - def getMonitoringBackends(self, monitoringType=None): """ Chooses the type of backend to use (Monitoring and/or Accounting) depending on the MonitoringType. @@ -231,6 +166,6 @@ def getMonitoringBackends(self, monitoringType=None): :param string MonitoringType: monitoring type to specify """ if monitoringType and self.getValue(f"MonitoringBackends/{monitoringType}"): - return self.getValue(f"MonitoringBackends/{monitoringType}") + return self.getValue(f"MonitoringBackends/{monitoringType}", []) else: return self.getValue("MonitoringBackends/Default", ["Accounting"]) diff --git a/src/DIRAC/ConfigurationSystem/Client/Helpers/Path.py b/src/DIRAC/ConfigurationSystem/Client/Helpers/Path.py index 11fb2b493a4..7cb2d87aea7 100644 --- a/src/DIRAC/ConfigurationSystem/Client/Helpers/Path.py +++ b/src/DIRAC/ConfigurationSystem/Client/Helpers/Path.py @@ -18,7 +18,7 @@ def cfgPath(*args): return os.path.normpath(os.path.join(*(str(k) for k in args))) -def cfgInstallPath(*args): +def cfgInstallPath(*args) -> str: """ Path to Installation/Configuration Options """ diff --git a/src/DIRAC/ConfigurationSystem/Client/Helpers/Registry.py b/src/DIRAC/ConfigurationSystem/Client/Helpers/Registry.py index 5f40fa9fcda..159649574ec 100644 --- a/src/DIRAC/ConfigurationSystem/Client/Helpers/Registry.py +++ b/src/DIRAC/ConfigurationSystem/Client/Helpers/Registry.py @@ -1,7 +1,13 @@ """ Helper for /Registry section """ + import errno +from threading import Lock + +from cachetools import TTLCache, cached + + from DIRAC import S_OK, S_ERROR from DIRAC.ConfigurationSystem.Client.Config import gConfig from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import getVO @@ -23,14 +29,14 @@ def getUsernameForDN(dn, usersList=None): """ dn = dn.strip() if not usersList: - result = gConfig.getSections("%s/Users" % gBaseRegistrySection) + result = gConfig.getSections(f"{gBaseRegistrySection}/Users") if not result["OK"]: return result usersList = result["Value"] for username in usersList: if dn in gConfig.getValue(f"{gBaseRegistrySection}/Users/{username}/DN", []): return S_OK(username) - return S_ERROR("No username found for dn %s" % dn) + return S_ERROR(f"No username found for dn {dn}") def getDNForUsername(username): @@ -41,7 +47,7 @@ def getDNForUsername(username): :return: S_OK(str)/S_ERROR() """ dnList = gConfig.getValue(f"{gBaseRegistrySection}/Users/{username}/DN", []) - return S_OK(dnList) if dnList else S_ERROR("No DN found for user %s" % username) + return S_OK(dnList) if dnList else S_ERROR(f"No DN found for user {username}") def getDNForHost(host): @@ -52,7 +58,7 @@ def getDNForHost(host): :return: S_OK(list)/S_ERROR() -- list of DNs """ dnList = gConfig.getValue(f"{gBaseRegistrySection}/Hosts/{host}/DN", []) - return S_OK(dnList) if dnList else S_ERROR("No DN found for host %s" % host) + return S_OK(dnList) if dnList else S_ERROR(f"No DN found for host {host}") def getGroupsForDN(dn): @@ -77,7 +83,7 @@ def __getGroupsWithAttr(attrName, value): :return: S_OK(list)/S_ERROR() -- contain list of groups """ - result = gConfig.getSections("%s/Groups" % gBaseRegistrySection) + result = gConfig.getSections(f"{gBaseRegistrySection}/Groups") if not result["OK"]: return result groupsList = result["Value"] @@ -107,7 +113,7 @@ def getGroupsForVO(vo): :return: S_OK(list)/S_ERROR() """ if getVO(): # tries to get default VO in /DIRAC/VirtualOrganization - return gConfig.getSections("%s/Groups" % gBaseRegistrySection) + return gConfig.getSections(f"{gBaseRegistrySection}/Groups") if not vo: return S_ERROR("No VO requested") return __getGroupsWithAttr("VO", vo) @@ -131,14 +137,14 @@ def getHostnameForDN(dn): :return: S_OK()/S_ERROR() """ dn = dn.strip() - result = gConfig.getSections("%s/Hosts" % gBaseRegistrySection) + result = gConfig.getSections(f"{gBaseRegistrySection}/Hosts") if not result["OK"]: return result hostList = result["Value"] for hostname in hostList: if dn in gConfig.getValue(f"{gBaseRegistrySection}/Hosts/{hostname}/DN", []): return S_OK(hostname) - return S_ERROR("No hostname found for dn %s" % dn) + return S_ERROR(f"No hostname found for dn {dn}") def getDefaultUserGroup(): @@ -146,7 +152,7 @@ def getDefaultUserGroup(): :return: str """ - return gConfig.getValue("/%s/DefaultGroup" % gBaseRegistrySection, "user") + return gConfig.getValue(f"/{gBaseRegistrySection}/DefaultGroup", "user") def findDefaultGroupForDN(dn): @@ -163,23 +169,23 @@ def findDefaultGroupForDN(dn): return findDefaultGroupForUser(result["Value"]) -def findDefaultGroupForUser(userName): +def findDefaultGroupForUser(username): """Get default group for user - :param str userName: user name + :param str username: user name :return: S_OK(str)/S_ERROR() """ - defGroups = getUserOption(userName, "DefaultGroup", []) - defGroups += gConfig.getValue("%s/DefaultGroup" % gBaseRegistrySection, ["user"]) - result = getGroupsForUser(userName) + defGroups = getUserOption(username, "DefaultGroup", []) + defGroups += gConfig.getValue(f"{gBaseRegistrySection}/DefaultGroup", ["user"]) + result = getGroupsForUser(username) if not result["OK"]: return result userGroups = result["Value"] for group in defGroups: if group in userGroups: return S_OK(group) - return S_OK(userGroups[0]) if userGroups else S_ERROR("User %s has no groups" % userName) + return S_OK(userGroups[0]) if userGroups else S_ERROR(f"User {username} has no groups") def getAllUsers(): @@ -187,7 +193,7 @@ def getAllUsers(): :return: list """ - result = gConfig.getSections("%s/Users" % gBaseRegistrySection) + result = gConfig.getSections(f"{gBaseRegistrySection}/Users") return result["Value"] if result["OK"] else [] @@ -196,7 +202,7 @@ def getAllGroups(): :return: list """ - result = gConfig.getSections("%s/Groups" % gBaseRegistrySection) + result = gConfig.getSections(f"{gBaseRegistrySection}/Groups") return result["Value"] if result["OK"] else [] @@ -349,16 +355,16 @@ def hostHasProperties(hostName, propList): return __matchProps(propList, getPropertiesForHost(hostName)) -def getUserOption(userName, optName, defaultValue=""): +def getUserOption(username, optName, defaultValue=""): """Get user option - :param str userName: user name + :param str username: user name :param str optName: option name :param defaultValue: default value :return: defaultValue or str """ - return gConfig.getValue(f"{gBaseRegistrySection}/Users/{userName}/{optName}", defaultValue) + return gConfig.getValue(f"{gBaseRegistrySection}/Users/{username}/{optName}", defaultValue) def getGroupOption(groupName, optName, defaultValue=""): @@ -390,7 +396,7 @@ def getHosts(): :return: S_OK()/S_ERROR() """ - return gConfig.getSections("%s/Hosts" % gBaseRegistrySection) + return gConfig.getSections(f"{gBaseRegistrySection}/Hosts") def getVOOption(voName, optName, defaultValue=""): @@ -410,7 +416,7 @@ def getBannedIPs(): :return: list """ - return gConfig.getValue("%s/BannedIPs" % gBaseRegistrySection, []) + return gConfig.getValue(f"{gBaseRegistrySection}/BannedIPs", []) def getVOForGroup(group): @@ -430,7 +436,7 @@ def getIdPForGroup(group): :return: str """ - return getVOOption(getVOForGroup(group), "IdP") + return getVOOption(getVOForGroup(group), "IdProvider") def getDefaultVOMSAttribute(): @@ -438,7 +444,7 @@ def getDefaultVOMSAttribute(): :return: str """ - return gConfig.getValue("%s/DefaultVOMSAttribute" % gBaseRegistrySection, "") + return gConfig.getValue(f"{gBaseRegistrySection}/DefaultVOMSAttribute", "") def getVOMSAttributeForGroup(group): @@ -456,7 +462,7 @@ def getDefaultVOMSVO(): :return: str """ - return gConfig.getValue("%s/DefaultVOMSVO" % gBaseRegistrySection, "") or getVO() + return gConfig.getValue(f"{gBaseRegistrySection}/DefaultVOMSVO", "") or getVO() def getVOMSVOForGroup(group): @@ -481,7 +487,7 @@ def getGroupsWithVOMSAttribute(vomsAttr): :return: list """ groups = [] - for group in gConfig.getSections("%s/Groups" % (gBaseRegistrySection)).get("Value", []): + for group in gConfig.getSections(f"{gBaseRegistrySection}/Groups").get("Value", []): if vomsAttr == gConfig.getValue(f"{gBaseRegistrySection}/Groups/{group}/VOMSRole", ""): groups.append(group) return groups @@ -493,7 +499,7 @@ def getVOs(): :return: S_OK(list)/S_ERROR() """ voName = getVO() - return S_OK([voName]) if voName else gConfig.getSections("%s/VO" % gBaseRegistrySection) + return S_OK([voName]) if voName else gConfig.getSections(f"{gBaseRegistrySection}/VO") def getVOMSServerInfo(requestedVO=""): @@ -586,14 +592,14 @@ def getUsernameForID(ID, usersList=None): :return: S_OK(str)/S_ERROR() """ if not usersList: - result = gConfig.getSections("%s/Users" % gBaseRegistrySection) + result = gConfig.getSections(f"{gBaseRegistrySection}/Users") if not result["OK"]: return result usersList = result["Value"] for username in usersList: if ID in gConfig.getValue(f"{gBaseRegistrySection}/Users/{username}/ID", []): return S_OK(username) - return S_ERROR("No username found for ID %s" % ID) + return S_ERROR(f"No username found for ID {ID}") def getCAForUsername(username): @@ -604,7 +610,7 @@ def getCAForUsername(username): :return: S_OK(str)/S_ERROR() """ dnList = gConfig.getValue(f"{gBaseRegistrySection}/Users/{username}/CA", []) - return S_OK(dnList) if dnList else S_ERROR("No CA found for user %s" % username) + return S_OK(dnList) if dnList else S_ERROR(f"No CA found for user {username}") def getDNProperty(userDN, value, defaultValue=None): @@ -619,7 +625,7 @@ def getDNProperty(userDN, value, defaultValue=None): result = getUsernameForDN(userDN) if not result["OK"]: return result - pathDNProperties = "{}/Users/{}/DNProperties".format(gBaseRegistrySection, result["Value"]) + pathDNProperties = f"{gBaseRegistrySection}/Users/{result['Value']}/DNProperties" result = gConfig.getSections(pathDNProperties) if result["OK"]: for section in result["Value"]: @@ -628,6 +634,10 @@ def getDNProperty(userDN, value, defaultValue=None): return S_OK(defaultValue) +_cache_getProxyProvidersForDN = TTLCache(maxsize=1000, ttl=60) + + +@cached(_cache_getProxyProvidersForDN, lock=Lock()) def getProxyProvidersForDN(userDN): """Get proxy providers by user DN @@ -725,5 +735,5 @@ def getIDFromDN(userDN): :return: S_OK(str)/S_ERROR() """ if not userDN.startswith(ID_DN_PREFIX): - return S_ERROR("%s DN does not contain user ID." % userDN) + return S_ERROR(f"{userDN} DN does not contain user ID.") return S_OK(userDN[len(ID_DN_PREFIX) :]) diff --git a/src/DIRAC/ConfigurationSystem/Client/Helpers/Resources.py b/src/DIRAC/ConfigurationSystem/Client/Helpers/Resources.py index 0aed942041c..06d0b40e718 100644 --- a/src/DIRAC/ConfigurationSystem/Client/Helpers/Resources.py +++ b/src/DIRAC/ConfigurationSystem/Client/Helpers/Resources.py @@ -3,12 +3,9 @@ import re from urllib import parse -from DIRAC import S_OK, S_ERROR, gConfig, gLogger -from DIRAC.ConfigurationSystem.Client.Helpers import Registry, Operations +from DIRAC import S_ERROR, S_OK, gConfig, gLogger from DIRAC.ConfigurationSystem.Client.Helpers.Path import cfgPath -from DIRAC.Core.Utilities.List import uniqueElements, fromChar -from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager - +from DIRAC.Core.Utilities.List import fromChar, uniqueElements gBaseResourcesSection = "/Resources" @@ -79,12 +76,11 @@ def getGOCSiteName(diracSiteName): cfgPath(gBaseResourcesSection, "Sites", diracSiteName.split(".")[0], diracSiteName, "Name") ) if not gocDBName: - return S_ERROR("No GOC site name for %s in CS (Not a grid site ?)" % diracSiteName) + return S_ERROR(f"No GOC site name for {diracSiteName} in CS (Not a grid site ?)") return S_OK(gocDBName) def getGOCSites(diracSites=None): - if diracSites is None: diracSites = getSites() if not diracSites["OK"]: @@ -124,7 +120,7 @@ def getDIRACSiteName(gocSiteName): if diracSites: return S_OK(diracSites) - return S_ERROR("There's no site with GOCDB name = %s in DIRAC CS" % gocSiteName) + return S_ERROR(f"There's no site with GOCDB name = {gocSiteName} in DIRAC CS") def getGOCFTSName(diracFTSName): @@ -137,7 +133,7 @@ def getGOCFTSName(diracFTSName): gocFTSName = gConfig.getValue(cfgPath(gBaseResourcesSection, "FTSEndpoints", "FTS3", diracFTSName)) if not gocFTSName: - return S_ERROR("No GOC FTS server name for %s in CS (Not a grid site ?)" % diracFTSName) + return S_ERROR(f"No GOC FTS server name for {diracFTSName} in CS (Not a grid site ?)") return S_OK(gocFTSName) @@ -201,36 +197,45 @@ def getSiteGrid(site): def getQueue(site, ce, queue): """Get parameters of the specified queue""" grid = site.split(".")[0] - result = gConfig.getOptionsDict(f"/Resources/Sites/{grid}/{site}/CEs/{ce}") - if not result["OK"]: + + # Get CE parameters + if not (result := gConfig.getOptionsDict(f"/Resources/Sites/{grid}/{site}/CEs/{ce}"))["OK"]: return result - resultDict = result["Value"] + ceDict = result["Value"] - # Get queue defaults - result = gConfig.getOptionsDict(f"/Resources/Sites/{grid}/{site}/CEs/{ce}/Queues/{queue}") - if not result["OK"]: + tags = set(fromChar(ceDict.get("Tag")) or []) + requiredTags = set(fromChar(ceDict.get("RequiredTag")) or []) + + # Get queue parameters + if not (result := gConfig.getOptionsDict(f"/Resources/Sites/{grid}/{site}/CEs/{ce}/Queues/{queue}"))["OK"]: return result - resultDict.update(result["Value"]) - - # Handle tag lists for the queue - for tagFieldName in ("Tag", "RequiredTag"): - tags = [] - ceTags = resultDict.get(tagFieldName) - if ceTags: - tags = fromChar(ceTags) - queueTags = resultDict.get(tagFieldName) - if queueTags: - queueTags = fromChar(queueTags) - tags = list(set(tags + queueTags)) - if tags: - resultDict[tagFieldName] = tags + queueDict = result["Value"] + + # Union the sets to combine tags and required tags from CE and queue + tags = tags.union(set(fromChar(queueDict.get("Tag")) or [])) + requiredTags = requiredTags.union(set(fromChar(queueDict.get("RequiredTag")) or [])) + + resultDict = {**ceDict, **queueDict} + if tags: + resultDict["Tag"] = list(tags) + if requiredTags: + resultDict["RequiredTag"] = list(requiredTags) resultDict["Queue"] = queue return S_OK(resultDict) -def getQueues(siteList=None, ceList=None, ceTypeList=None, community=None, mode=None): - """Get CE/queue options according to the specified selection""" +def getQueues(siteList=None, ceList=None, ceTypeList=None, community=None, tags=None): + """Get CE/queue options according to the specified selection + + :param str list siteList: sites to be selected + :param str list ceList: CEs to be selected + :param str list ceTypeList: CE types to be selected + :param str community: selected VO + :param str list tags: tags required for selection + + :return: S_OK/S_ERROR with Value - dictionary of selected Site/CE/Queue parameters + """ result = gConfig.getSections("/Resources/Sites") if not result["OK"]: @@ -240,17 +245,14 @@ def getQueues(siteList=None, ceList=None, ceTypeList=None, community=None, mode= grids = result["Value"] for grid in grids: - result = gConfig.getSections("/Resources/Sites/%s" % grid) + result = gConfig.getSections(f"/Resources/Sites/{grid}") if not result["OK"]: continue sites = result["Value"] for site in sites: if siteList and site not in siteList: continue - if community: - comList = gConfig.getValue(f"/Resources/Sites/{grid}/{site}/VO", []) - if comList and community.lower() not in [cl.lower() for cl in comList]: - continue + siteCEParameters = {} result = gConfig.getOptionsDict(f"/Resources/Sites/{grid}/{site}/CEs") if result["OK"]: @@ -260,20 +262,14 @@ def getQueues(siteList=None, ceList=None, ceTypeList=None, community=None, mode= continue ces = result["Value"] for ce in ces: - if mode: - ceMode = gConfig.getValue(f"/Resources/Sites/{grid}/{site}/CEs/{ce}/SubmissionMode", "Direct") - if not ceMode or ceMode.lower() != mode.lower(): - continue + ceTags = gConfig.getValue(f"/Resources/Sites/{grid}/{site}/CEs/{ce}/Tag", []) if ceTypeList: ceType = gConfig.getValue(f"/Resources/Sites/{grid}/{site}/CEs/{ce}/CEType", "") if not ceType or ceType not in ceTypeList: continue if ceList and ce not in ceList: continue - if community: - comList = gConfig.getValue(f"/Resources/Sites/{grid}/{site}/CEs/{ce}/VO", []) - if comList and community.lower() not in [cl.lower() for cl in comList]: - continue + ceOptionsDict = dict(siteCEParameters) result = gConfig.getOptionsDict(f"/Resources/Sites/{grid}/{site}/CEs/{ce}") if not result["OK"]: @@ -285,9 +281,28 @@ def getQueues(siteList=None, ceList=None, ceTypeList=None, community=None, mode= queues = result["Value"] for queue in queues: if community: - comList = gConfig.getValue(f"/Resources/Sites/{grid}/{site}/CEs/{ce}/Queues/{queue}/VO", []) + # Community can be defined on site, CE or queue level + paths = [ + f"/Resources/Sites/{grid}/{site}/CEs/{ce}/Queues/{queue}/VO", + f"/Resources/Sites/{grid}/{site}/CEs/{ce}/VO", + f"/Resources/Sites/{grid}/{site}/VO", + ] + + # Try each path in order, stopping when we find a non-empty list + for path in paths: + comList = gConfig.getValue(path, []) + if comList: + break + + # If we found a list and the community is not in it, skip this iteration if comList and community.lower() not in [cl.lower() for cl in comList]: continue + + if tags: + queueTags = gConfig.getValue(f"/Resources/Sites/{grid}/{site}/CEs/{ce}/Queues/{queue}/Tag", []) + queueTags = set(ceTags + queueTags) + if not queueTags or not set(tags).issubset(queueTags): + continue resultDict.setdefault(site, {}) resultDict[site].setdefault(ce, ceOptionsDict) resultDict[site][ce].setdefault("Queues", {}) @@ -370,7 +385,7 @@ def getDIRACPlatform(OSList): platforms += os2PlatformDict[os] if not platforms: - return S_ERROR("No compatible DIRAC platform found for %s" % ",".join(OSList)) + return S_ERROR(f"No compatible DIRAC platform found for {','.join(OSList)}") platforms.sort(key=_platformSortKey, reverse=True) @@ -387,7 +402,7 @@ def getDIRACPlatforms(): def getCatalogPath(catalogName): """Return the configuration path of the description for a a given catalog""" - return "/Resources/FileCatalogs/%s" % catalogName + return f"/Resources/FileCatalogs/{catalogName}" def getBackendConfig(backendID): @@ -395,7 +410,7 @@ def getBackendConfig(backendID): :params backendID: string representing a backend identifier. Ex: stdout, file, f02 """ - return gConfig.getOptionsDict("Resources/LogBackends/%s" % backendID) + return gConfig.getOptionsDict(f"Resources/LogBackends/{backendID}") def getFilterConfig(filterID): @@ -403,7 +418,7 @@ def getFilterConfig(filterID): :params filterID: string representing a filter identifier. """ - return gConfig.getOptionsDict("Resources/LogFilters/%s" % filterID) + return gConfig.getOptionsDict(f"Resources/LogFilters/{filterID}") def getInfoAboutProviders(of=None, providerName=None, option="", section=""): @@ -446,189 +461,6 @@ def getInfoAboutProviders(of=None, providerName=None, option="", section=""): return S_OK(gConfig.getValue(f"{gBaseResourcesSection}/{of}Providers/{providerName}/{section}/{option}")) -def findGenericCloudCredentials(vo=False, group=False): - """Get the cloud credentials to use for a specific VO and/or group.""" - if not group and not vo: - return S_ERROR("Need a group or a VO to determine the Generic cloud credentials") - if not vo: - vo = Registry.getVOForGroup(group) - if not vo: - return S_ERROR("Group %s does not have a VO associated" % group) - opsHelper = Operations.Operations(vo=vo) - cloudGroup = opsHelper.getValue("Cloud/GenericCloudGroup", "") - cloudDN = opsHelper.getValue("Cloud/GenericCloudDN", "") - if not cloudDN: - cloudUser = opsHelper.getValue("Cloud/GenericCloudUser", "") - if cloudUser: - result = Registry.getDNForUsername(cloudUser) - if result["OK"]: - cloudDN = result["Value"][0] - else: - return S_ERROR("Failed to find suitable CloudDN") - if cloudDN and cloudGroup: - gLogger.verbose(f"Cloud credentials from CS: {cloudDN}@{cloudGroup}") - result = gProxyManager.userHasProxy(cloudDN, cloudGroup, 86400) - if not result["OK"]: - return result - return S_OK((cloudDN, cloudGroup)) - return S_ERROR("Cloud credentials not found") - - -def getVMTypes(siteList=None, ceList=None, vmTypeList=None, vo=None): - """Get CE/vmType options filtered by the provided parameters.""" - - result = gConfig.getSections("/Resources/Sites") - if not result["OK"]: - return result - - resultDict = {} - - grids = result["Value"] - for grid in grids: - result = gConfig.getSections("/Resources/Sites/%s" % grid) - if not result["OK"]: - continue - sites = result["Value"] - for site in sites: - if siteList is not None and site not in siteList: - continue - if vo: - voList = gConfig.getValue(f"/Resources/Sites/{grid}/{site}/VO", []) - if voList and vo not in voList: - continue - result = gConfig.getSections(f"/Resources/Sites/{grid}/{site}/Cloud") - if not result["OK"]: - continue - ces = result["Value"] - for ce in ces: - if ceList is not None and ce not in ceList: - continue - if vo: - voList = gConfig.getValue(f"/Resources/Sites/{grid}/{site}/Cloud/{ce}/VO", []) - if voList and vo not in voList: - continue - result = gConfig.getOptionsDict(f"/Resources/Sites/{grid}/{site}/Cloud/{ce}") - if not result["OK"]: - continue - ceOptionsDict = result["Value"] - result = gConfig.getSections(f"/Resources/Sites/{grid}/{site}/Cloud/{ce}/VMTypes") - if not result["OK"]: - result = gConfig.getSections(f"/Resources/Sites/{grid}/{site}/Cloud/{ce}/Images") - if not result["OK"]: - return result - vmTypes = result["Value"] - for vmType in vmTypes: - if vmTypeList is not None and vmType not in vmTypeList: - continue - if vo: - voList = gConfig.getValue(f"/Resources/Sites/{grid}/{site}/Cloud/{ce}/VMTypes/{vmType}/VO", []) - if not voList: - voList = gConfig.getValue( - f"/Resources/Sites/{grid}/{site}/Cloud/{ce}/Images/{vmType}/VO", [] - ) - if voList and vo not in voList: - continue - resultDict.setdefault(site, {}) - resultDict[site].setdefault(ce, ceOptionsDict) - resultDict[site][ce].setdefault("VMTypes", {}) - result = gConfig.getOptionsDict(f"/Resources/Sites/{grid}/{site}/Cloud/{ce}/VMTypes/{vmType}") - if not result["OK"]: - result = gConfig.getOptionsDict(f"/Resources/Sites/{grid}/{site}/Cloud/{ce}/Images/{vmType}") - if not result["OK"]: - continue - vmTypeOptionsDict = result["Value"] - resultDict[site][ce]["VMTypes"][vmType] = vmTypeOptionsDict - - return S_OK(resultDict) - - -def getVMTypeConfig(site, ce="", vmtype=""): - """Get the VM image type parameters of the specified queue""" - tags = [] - reqtags = [] - grid = site.split(".")[0] - if not ce: - result = gConfig.getSections(f"/Resources/Sites/{grid}/{site}/Cloud") - if not result["OK"]: - return result - ceList = result["Value"] - if len(ceList) == 1: - ce = ceList[0] - else: - return S_ERROR("No cloud endpoint specified") - - result = gConfig.getOptionsDict(f"/Resources/Sites/{grid}/{site}/Cloud/{ce}") - if not result["OK"]: - return result - resultDict = result["Value"] - ceTags = resultDict.get("Tag") - if ceTags: - tags = fromChar(ceTags) - ceTags = resultDict.get("RequiredTag") - if ceTags: - reqtags = fromChar(ceTags) - resultDict["CEName"] = ce - - if vmtype: - result = gConfig.getOptionsDict(f"/Resources/Sites/{grid}/{site}/Cloud/{ce}/VMTypes/{vmtype}") - if not result["OK"]: - return result - resultDict.update(result["Value"]) - queueTags = resultDict.get("Tag") - if queueTags: - queueTags = fromChar(queueTags) - tags = list(set(tags + queueTags)) - queueTags = resultDict.get("RequiredTag") - if queueTags: - queueTags = fromChar(queueTags) - reqtags = list(set(reqtags + queueTags)) - - if tags: - resultDict["Tag"] = tags - if reqtags: - resultDict["RequiredTag"] = reqtags - resultDict["VMType"] = vmtype - resultDict["Site"] = site - return S_OK(resultDict) - - -def getPilotBootstrapParameters(vo="", runningPod=""): - """Get all of the settings required to bootstrap a cloud instance.""" - op = Operations.Operations(vo=vo) - result = op.getOptionsDict("Cloud") - opParameters = {} - if result["OK"]: - opParameters = result["Value"] - opParameters["VO"] = vo - # FIXME: The majority of these settings can be removed once the old vm-pilot - # scripts have been removed. - opParameters["ReleaseProject"] = op.getValue("Cloud/ReleaseProject", "DIRAC") - opParameters["ReleaseVersion"] = op.getValue("Cloud/ReleaseVersion", op.getValue("Pilot/Version")) - opParameters["Setup"] = gConfig.getValue("/DIRAC/Setup", "unknown") - opParameters["SubmitPool"] = op.getValue("Cloud/SubmitPool") - opParameters["CloudPilotCert"] = op.getValue("Cloud/CloudPilotCert") - opParameters["CloudPilotKey"] = op.getValue("Cloud/CloudPilotKey") - opParameters["pilotFileServer"] = op.getValue("Pilot/pilotFileServer") - result = op.getOptionsDict("Cloud/%s" % runningPod) - if result["OK"]: - opParameters.update(result["Value"]) - - # Get standard pilot version now - if "Version" in opParameters: - gLogger.warn( - "Cloud bootstrap version now uses standard Pilot/Version setting. " - "Please remove all obsolete (Cloud/Version) setting(s)." - ) - pilotVersions = op.getValue("Pilot/Version") - if isinstance(pilotVersions, str): - pilotVersions = [pilotVersions] - if not pilotVersions: - return S_ERROR("Failed to get pilot version.") - opParameters["Version"] = pilotVersions[0].strip() - - return S_OK(opParameters) - - def _platformSortKey(version: str) -> list[str]: # Loosely based on distutils.version.LooseVersion parts = [] diff --git a/src/DIRAC/ConfigurationSystem/Client/Helpers/test/Test_Helpers.py b/src/DIRAC/ConfigurationSystem/Client/Helpers/test/Test_Helpers.py index 570eb4dc969..6a60f64006c 100644 --- a/src/DIRAC/ConfigurationSystem/Client/Helpers/test/Test_Helpers.py +++ b/src/DIRAC/ConfigurationSystem/Client/Helpers/test/Test_Helpers.py @@ -1,12 +1,16 @@ from itertools import zip_longest +from diraccfg import CFG import pytest from unittest.mock import MagicMock +from DIRAC import gConfig +from DIRAC.ConfigurationSystem.Client import ConfigurationData from DIRAC.ConfigurationSystem.Client.Helpers.Resources import ( getDIRACPlatform, getCompatiblePlatforms, _platformSortKey, + getQueue, ) @@ -63,7 +67,6 @@ ], ) def test_getDIRACPlatform(mocker, mockGCReplyInput, requested, expectedRes, expectedValue): - mockGCReply.return_value = mockGCReplyInput mocker.patch("DIRAC.Interfaces.API.Dirac.gConfig.getOptionsDict", side_effect=mockGCReply) @@ -123,3 +126,79 @@ def test_getCompatiblePlatforms(mocker, mockGCReplyInput, requested, expectedRes assert res["OK"] is expectedRes, res if expectedRes: assert set(res["Value"]) == set(expectedValue), res["Value"] + + +config = """ +Resources +{ + Sites + { + LHCb + { + LHCb.CERN.cern + { + CEs + { + ce1.cern.ch + { + CEType = AREX + architecture = x86_64 + OS = linux_CentOS_7.9.2009 + Tag = Token + Queues + { + nordugrid-SLURM-grid + { + SI00 = 2775 + MaxRAM = 128534 + CPUTime = 3836159 + maxCPUTime = 5760 + Tag = MultiProcessor + MaxWaitingJobs = 10 + MaxTotalJobs = 200 + LocalCEType = Pool/Singularity + OS = linux_AlmaLinux_9.4.2104 + } + } + } + } + } + } + } +} +""" + + +def test_getQueue(): + """Test getQueue function.""" + + # Set up the configuration file + ConfigurationData.localCFG = CFG() + cfg = CFG() + cfg.loadFromBuffer(config) + gConfig.loadCFG(cfg) + + # Test getQueue + site = "LHCb.CERN.cern" + ce = "ce1.cern.ch" + queue = "nordugrid-SLURM-grid" + + result = getQueue(site, ce, queue) + assert result["OK"] + + expectedDict = { + "CEType": "AREX", + "Queue": "nordugrid-SLURM-grid", + "architecture": "x86_64", + "SI00": "2775", + "MaxRAM": "128534", + "CPUTime": "3836159", + "maxCPUTime": "5760", + "Tag": ["MultiProcessor", "Token"], + "MaxWaitingJobs": "10", + "MaxTotalJobs": "200", + "LocalCEType": "Pool/Singularity", + "OS": "linux_AlmaLinux_9.4.2104", + } + assert sorted(result["Value"].pop("Tag")) == sorted(expectedDict.pop("Tag")) + assert result["Value"] == expectedDict diff --git a/src/DIRAC/ConfigurationSystem/Client/LocalConfiguration.py b/src/DIRAC/ConfigurationSystem/Client/LocalConfiguration.py index c953d78e8b6..5b7c0086120 100755 --- a/src/DIRAC/ConfigurationSystem/Client/LocalConfiguration.py +++ b/src/DIRAC/ConfigurationSystem/Client/LocalConfiguration.py @@ -11,7 +11,11 @@ from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData from DIRAC.ConfigurationSystem.private.Refresher import gRefresher -from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceSection, getAgentSection, getExecutorSection +from DIRAC.ConfigurationSystem.Client.PathFinder import ( + getServiceSection, + getAgentSection, + getExecutorSection, +) from DIRAC.Core.Utilities.Devloader import Devloader @@ -56,8 +60,7 @@ def disableParsingCommandLine(self): def __getAbsolutePath(self, optionPath): if optionPath[0] == "/": return optionPath - else: - return f"{self.currentSectionPath}/{optionPath}" + return f"{self.currentSectionPath}/{optionPath}" def addMandatoryEntry(self, optionPath): """ @@ -157,9 +160,9 @@ def registerCmdOpt(self, shortOption, longOption, helpString, function=False): raise Exception("No short or long options defined") for optTuple in self.commandOptionList: if shortOption and optTuple[0] == shortOption: - raise Exception("Short switch %s is already defined!" % shortOption) + raise Exception(f"Short switch {shortOption} is already defined!") if longOption and optTuple[1] == longOption: - raise Exception("Long switch %s is already defined!" % longOption) + raise Exception(f"Long switch {longOption} is already defined!") self.commandOptionList.append((shortOption, longOption, helpString, function)) def registerCmdArg(self, description, mandatory=True, values=None, default=None): @@ -207,7 +210,7 @@ def registerCmdArg(self, description, mandatory=True, values=None, default=None) description = [description] # Single argument that can have two names, e.g.: elif isinstance(description, tuple): - argMarking = "<%s>" % "|".join([d.split(":")[0].strip() for d in description]) + argMarking = f"<{'|'.join([d.split(':')[0].strip() for d in description])}>" # List arguments, e.g.: CE [CE] elif isinstance(description, list): argMarking = "{0} [{0}]".format(description[0].split(":")[0].strip()) @@ -306,6 +309,7 @@ def initialize(self, *, returnErrors=False): def __initLogger(self, componentName, logSection, forceInit=False): gLogger.initialize(componentName, logSection, forceInit=forceInit) + gLogger.disableLogsFromExternalLibs() if self.__debugMode == 1: gLogger.setLevel("VERBOSE") @@ -315,6 +319,7 @@ def __initLogger(self, componentName, logSection, forceInit=False): elif self.__debugMode >= 3: gLogger.setLevel("DEBUG") gLogger.showHeaders(True) + gLogger.enableLogsFromExternalLibs() def loadUserData(self): """ @@ -357,22 +362,22 @@ def __parseCommandLine(self): longOptionList = [] for optionTuple in self.commandOptionList: if shortOption.find(optionTuple[0]) < 0: - shortOption += "%s" % optionTuple[0] + shortOption += f"{optionTuple[0]}" else: if optionTuple[0]: - gLogger.error("Short option -%s has been already defined" % optionTuple[0]) + gLogger.error(f"Short option -{optionTuple[0]} has been already defined") if not optionTuple[1] in longOptionList: - longOptionList.append("%s" % optionTuple[1]) + longOptionList.append(f"{optionTuple[1]}") else: if optionTuple[1]: - gLogger.error("Long option --%s has been already defined" % optionTuple[1]) + gLogger.error(f"Long option --{optionTuple[1]} has been already defined") try: opts, args = getopt.gnu_getopt(sys.argv[self.firstOptionIndex :], shortOption, longOptionList) except getopt.GetoptError as x: # x = option "-k" not recognized # print help information and exit - gLogger.fatal("Error when parsing command line arguments: %s" % str(x)) + gLogger.fatal(f"Error when parsing command line arguments: {str(x)}") self.showHelp(exitCode=2) for opt, val in opts: @@ -407,8 +412,8 @@ def __parseCommandLine(self): groupArgs = [] step = 0 - for i in range(len(self.commandArgumentList)): - argMarking, description, mandatory, values, default = self.commandArgumentList[i] + for i, commandArgument in enumerate(self.commandArgumentList): + argMarking, description, mandatory, values, default = commandArgument # Check whether the required arguments are given in the command line if len(self.commandArgList) <= (i + step): @@ -428,7 +433,7 @@ def __parseCommandLine(self): if values and cArg not in values: gLogger.fatal( "Error when parsing command line arguments: " - '"%s" does not match the allowed values for %s' % (cArg, argMarking) + f'"{cArg}" does not match the allowed values for {argMarking}' ) self.showHelp(exitCode=1) @@ -439,30 +444,56 @@ def __parseCommandLine(self): def __loadCFGFiles(self): """ Loads possibly several cfg files, in order: - 1. ~/.dirac.cfg - 2. cfg files pointed by DIRACSYSCONFIG env variable (comma-separated) - 3. cfg files specified in addCFGFile calls - 4. cfg files that come from the command line + 1. cfg files pointed by DIRACSYSCONFIG env variable (comma-separated) + 2. ~/.dirac.cfg + 3. DIRAC.rootPath/etc/dirac.cfg + 4. cfg files specified in addCFGFile calls + 5. cfg files that come from the command line """ errorsList = [] + foundCFGFile = False + + # 1. $DIRACSYSCONFIG if "DIRACSYSCONFIG" in os.environ: diracSysConfigFiles = os.environ["DIRACSYSCONFIG"].replace(" ", "").split(",") for diracSysConfigFile in reversed(diracSysConfigFiles): - gLogger.debug("Loading file from DIRACSYSCONFIG %s" % diracSysConfigFile) + gLogger.debug(f"Loading file from DIRACSYSCONFIG {diracSysConfigFile}") + if os.path.isfile(diracSysConfigFile): + foundCFGFile = True gConfigurationData.loadFile(diracSysConfigFile) + + # 2. ~/.dirac.cfg + if os.path.isfile(os.path.expanduser("~/.dirac.cfg")): + foundCFGFile = True gConfigurationData.loadFile(os.path.expanduser("~/.dirac.cfg")) + + # 3. defaultCFGFile = os.path.join(DIRAC.rootPath, "etc", "dirac.cfg") + if os.path.isfile(os.path.join(DIRAC.rootPath, "etc", "dirac.cfg")): + foundCFGFile = True + + # 4. cfg files specified in addCFGFile calls for fileName in self.additionalCFGFiles: - gLogger.debug("Loading file %s" % fileName) + if os.path.isfile(fileName): + foundCFGFile = True + gLogger.debug(f"Loading file {fileName}") retVal = gConfigurationData.loadFile(fileName) if not retVal["OK"]: - gLogger.debug("Could not load file {}: {}".format(fileName, retVal["Message"])) + gLogger.debug(f"Could not load file {fileName}: {retVal['Message']}") errorsList.append(retVal["Message"]) + + # 5. cfg files that come from the command line for fileName in self.cliAdditionalCFGFiles: - gLogger.debug("Loading file %s" % fileName) + if os.path.isfile(fileName): + foundCFGFile = True + gLogger.debug(f"Loading file {fileName}") retVal = gConfigurationData.loadFile(fileName) if not retVal["OK"]: - gLogger.debug("Could not load file {}: {}".format(fileName, retVal["Message"])) + gLogger.debug(f"Could not load file {fileName}: {retVal['Message']}") errorsList.append(retVal["Message"]) + + if not foundCFGFile: + gLogger.warn("No CFG file loaded, was that intentional?") + return errorsList def __addUserDataToConfiguration(self): @@ -474,18 +505,20 @@ def __addUserDataToConfiguration(self): try: if self.componentType == "service": self.__setDefaultSection(getServiceSection(self.componentName)) + elif self.componentType == "tornado": + self.__setDefaultSection("/Systems/Tornado") elif self.componentType == "agent": self.__setDefaultSection(getAgentSection(self.componentName)) elif self.componentType == "executor": self.__setDefaultSection(getExecutorSection(self.componentName)) elif self.componentType == "web": - self.__setDefaultSection("/%s" % self.componentName) + self.__setDefaultSection(f"/{self.componentName}") elif self.componentType == "script": if self.componentName and self.componentName[0] == "/": self.__setDefaultSection(self.componentName) self.componentName = self.componentName[1:] else: - self.__setDefaultSection("/Scripts/%s" % self.componentName) + self.__setDefaultSection(f"/Scripts/{self.componentName}") else: self.__setDefaultSection("/") except Exception as e: @@ -500,7 +533,7 @@ def __addUserDataToConfiguration(self): if func: retVal = func(optionValue) if not isinstance(retVal, dict): - errorsList.append("Callback for switch '%s' does not return S_OK or S_ERROR" % optionName) + errorsList.append(f"Callback for switch '{optionName}' does not return S_OK or S_ERROR") elif not retVal["OK"]: errorsList.append(retVal["Message"]) else: @@ -600,9 +633,16 @@ def setConfigurationForScript(self, scriptName): self.componentName = scriptName self.componentType = "script" + def setConfigurationForTornado(self): + """ + Declare this is a Tornado component + """ + self.componentName = "Tornado/Tornado" + self.componentType = "tornado" + def __setSectionByCmd(self, value): if value[0] != "/": - return S_ERROR("%s is not a valid section. It should start with '/'" % value) + return S_ERROR(f"{value} is not a valid section. It should start with '/'") self.currentSectionPath = value return S_OK() @@ -610,7 +650,7 @@ def __setOptionByCmd(self, value): valueList = value.split("=") if len(valueList) < 2: # FIXME: in the method above an exception is raised, check consitency - return S_ERROR("-o expects a option=value argument.\nFor example %s -o Port=1234" % sys.argv[0]) + return S_ERROR(f"-o expects a option=value argument.\nFor example {sys.argv[0]} -o Port=1234") self.__setOptionValue(valueList[0], "=".join(valueList[1:])) return S_OK() @@ -646,7 +686,7 @@ def showLicense(self, dummy=False): with open(lpath) as fd: sys.stdout.write(fd.read()) except OSError: - sys.stdout.write("Can't find GPLv3 license at %s. Somebody stole it!\n" % lpath) + sys.stdout.write(f"Can't find GPLv3 license at {lpath}. Somebody stole it!\n") sys.stdout.write("Please check out http://www.gnu.org/licenses/gpl-3.0.html for more info\n") DIRAC.exit(0) @@ -666,7 +706,7 @@ def showHelp(self, dummy=False, exitCode=0): else: gLogger.notice("\nUsage:") gLogger.notice( - " %s [options] ..." % os.path.basename(sys.argv[0]).split(".")[0].replace("_", "-"), + f" {os.path.basename(sys.argv[0]).split('.')[0].replace('_', '-')} [options] ...", " ".join([t[0] for t in self.commandArgumentList]), ) if dummy: @@ -713,9 +753,9 @@ def showHelp(self, dummy=False, exitCode=0): for aName, dText in descriptions: maxArgLen = max(len(aName), maxArgLen) if values: - dText += " [%s]" % ", ".join(values) + dText += f" [{', '.join(values)}]" if default: - dText += " [default: %s]" % default + dText += f" [default: {default}]" if not mandatory: dText += " (optional)" # Do not duplicate descriptions @@ -724,7 +764,7 @@ def showHelp(self, dummy=False, exitCode=0): # Print Arguments block gLogger.notice("\nArguments:") for arg, doc in allDescriptions: - gLogger.notice(" {}:{} {}".format(arg, " " * (maxArgLen - len(arg)), doc)) + gLogger.notice(f" {arg}:{' ' * (maxArgLen - len(arg))} {doc}") elif self.__helpArgumentsDoc: gLogger.notice(self.__helpArgumentsDoc) diff --git a/src/DIRAC/ConfigurationSystem/Client/PathFinder.py b/src/DIRAC/ConfigurationSystem/Client/PathFinder.py index 8278295c185..1946cee118d 100755 --- a/src/DIRAC/ConfigurationSystem/Client/PathFinder.py +++ b/src/DIRAC/ConfigurationSystem/Client/PathFinder.py @@ -7,14 +7,6 @@ from DIRAC.ConfigurationSystem.Client.Helpers import Path -def getDIRACSetup(): - """Get DIRAC default setup name - - :return: str - """ - return gConfigurationData.extractOptionFromCFG("/DIRAC/Setup") - - def divideFullName(entityName, componentName=None): """Convert component full name to tuple @@ -29,48 +21,15 @@ def divideFullName(entityName, componentName=None): fields = [field.strip() for field in entityName.split("/") if field.strip()] if len(fields) == 2: return tuple(fields) - raise RuntimeError("Service (%s) name must be with the form system/service" % entityName) - + raise RuntimeError(f"Service ({entityName}) name must be with the form system/service") -def getSystemInstance(system, setup=False): - """Find system instance name - - :param str system: system name - :param str setup: setup name - - :return: str - """ - optionPath = Path.cfgPath("/DIRAC/Setups", setup or getDIRACSetup(), system) - instance = gConfigurationData.extractOptionFromCFG(optionPath) - if not instance: - raise RuntimeError("Option %s is not defined" % optionPath) - return instance - - -def getSystemSection(system, instance=False, setup=False): - """Get system section - - :param str system: system name - :param str instance: instance name - :param str setup: setup name - :return: str -- system section path - """ - system, _ = divideFullName(system, "_") # for backward compatibility - return Path.cfgPath( - "/Systems", - system, - instance or getSystemInstance(system, setup=setup), - ) - - -def getComponentSection(system, component=False, setup=False, componentCategory="Services"): +def getComponentSection(system, component=False, componentCategory="Services"): """Function returns the path to the component. :param str system: system name or component name prefixed by the system in which it is placed. e.g. 'WorkloadManagement/SandboxStoreHandler' :param str component: component name, e.g. 'SandboxStoreHandler' - :param str setup: Name of the setup. :param str componentCategory: Category of the component, it can be: 'Agents', 'Services', 'Executors' or 'Databases'. @@ -80,14 +39,14 @@ def getComponentSection(system, component=False, setup=False, componentCategory= :raise RuntimeException: If in the system - the system part does not correspond to any known system in DIRAC. Examples: - getComponentSection('WorkloadManagement/SandboxStoreHandler', setup='Production', componentCategory='Services') - getComponentSection('WorkloadManagement', 'SandboxStoreHandler', 'Production') + getComponentSection('WorkloadManagement/SandboxStoreHandler', componentCategory='Services') + getComponentSection('WorkloadManagement', 'SandboxStoreHandler') """ system, component = divideFullName(system, component) - return Path.cfgPath(getSystemSection(system, setup=setup), componentCategory, component) + return Path.cfgPath(f"/Systems/{system}", componentCategory, component) -def getAPISection(system, endpointName=False, setup=False): +def getAPISection(system, endpointName=False): """Get API section in a system :param str system: system name @@ -95,66 +54,61 @@ def getAPISection(system, endpointName=False, setup=False): :return: str """ - return getComponentSection(system, component=endpointName, setup=setup, componentCategory="APIs") + return getComponentSection(system, component=endpointName, componentCategory="APIs") -def getServiceSection(system, serviceName=False, setup=False): +def getServiceSection(system, serviceName=False): """Get service section in a system :param str system: system name :param str serviceName: service name - :param str setup: setup name :return: str """ - return getComponentSection(system, component=serviceName, setup=setup) + return getComponentSection(system, component=serviceName) -def getAgentSection(system, agentName=False, setup=False): +def getAgentSection(system, agentName=False): """Get agent section in a system :param str system: system name :param str agentName: agent name - :param str setup: setup name :return: str """ - return getComponentSection(system, component=agentName, setup=setup, componentCategory="Agents") + return getComponentSection(system, component=agentName, componentCategory="Agents") -def getExecutorSection(system, executorName=None, component=False, setup=False): +def getExecutorSection(system, executorName=None): """Get executor section in a system :param str system: system name :param str executorName: executor name - :param str setup: setup name :return: str """ - return getComponentSection(system, component=executorName, setup=setup, componentCategory="Executors") + return getComponentSection(system, component=executorName, componentCategory="Executors") -def getDatabaseSection(system, dbName=False, setup=False): +def getDatabaseSection(system, dbName=False): """Get DB section in a system :param str system: system name :param str dbName: DB name - :param str setup: setup name :return: str """ - return getComponentSection(system, component=dbName, setup=setup, componentCategory="Databases") + return getComponentSection(system, component=dbName, componentCategory="Databases") -def getSystemURLSection(system, setup=False): +def getSystemURLSection(system): """Get URLs section in a system :param str system: system name - :param str setup: setup name :return: str """ - return Path.cfgPath(getSystemSection(system, setup=setup), "URLs") + return Path.cfgPath(f"/Systems/{system}", "URLs") def checkComponentURL(componentURL, system=None, component=None, pathMandatory=False): @@ -183,27 +137,25 @@ def checkComponentURL(componentURL, system=None, component=None, pathMandatory=F return url.geturl() -def getSystemURLs(system, setup=False, failover=False): +def getSystemURLs(system, failover=False): """Generate url. :param str system: system name or full name e.g.: Framework/ProxyManager - :param str setup: DIRAC setup name, can be defined in dirac.cfg :param bool failover: to add failover URLs to end of result list :return: dict -- complete urls. e.g. [dips://some-domain:3424/Framework/Service] """ urlDict = {} - for service in gConfigurationData.getOptionsFromCFG("%s/URLs" % getSystemSection(system, setup=setup)) or []: - urlDict[service] = getServiceURLs(system, service, setup=setup, failover=failover) + for service in gConfigurationData.getOptionsFromCFG(f"/Systems/{system}/URLs") or []: + urlDict[service] = getServiceURLs(system, service, failover=failover) return urlDict -def getServiceURLs(system, service=None, setup=False, failover=False): +def getServiceURLs(system, service=None, failover=False): """Generate url. :param str system: system name or full name e.g.: Framework/ProxyManager :param str service: service name, like 'ProxyManager'. - :param str setup: DIRAC setup name, can be defined in dirac.cfg :param bool failover: to add failover URLs to end of result list :return: list -- complete urls. e.g. [dips://some-domain:3424/Framework/Service] @@ -211,7 +163,7 @@ def getServiceURLs(system, service=None, setup=False, failover=False): system, service = divideFullName(system, service) resList = [] mainServers = None - systemSection = getSystemSection(system, setup=setup) + systemSection = f"/Systems/{system}" # Add failover URLs at the end of the list failover = "Failover" if failover else "" @@ -221,7 +173,6 @@ def getServiceURLs(system, service=None, setup=False, failover=False): # Be sure that urls not None for url in urls or []: - # Trying if we are refering to the list of main servers # which would be like dips://$MAINSERVERS$:1234/System/Component if "$MAINSERVERS$" in url: @@ -229,7 +180,7 @@ def getServiceURLs(system, service=None, setup=False, failover=False): # Operations cannot be imported at the beginning because of a bootstrap problem from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations - mainServers = Operations(setup=setup).getValue("MainServers", []) + mainServers = Operations().getValue("MainServers", []) if not mainServers: raise Exception("No Main servers defined") @@ -249,31 +200,42 @@ def getServiceURLs(system, service=None, setup=False, failover=False): return resList -def getServiceURL(system, service=None, setup=False): +def useLegacyAdapter(system, service=None) -> bool: + """Should DiracX be used for this service via the legacy adapter mechanism + + :param str system: system name or full name e.g.: Framework/ProxyManager + :param str service: service name, like 'ProxyManager'. + + :return: bool -- True if DiracX should be used + """ + system, service = divideFullName(system, service) + value = gConfigurationData.extractOptionFromCFG(f"/DiracX/LegacyClientEnabled/{system}/{service}") + return (value or "no").lower() in ("y", "yes", "true", "1") + + +def getServiceURL(system, service=None): """Generate url. :param str system: system name or full name e.g.: Framework/ProxyManager :param str service: service name, like 'ProxyManager'. - :param str setup: DIRAC setup name, can be defined in dirac.cfg :return: str -- complete list of urls. e.g. dips://some-domain:3424/Framework/Service, dips://.. """ system, service = divideFullName(system, service) - urls = getServiceURLs(system, service=service, setup=setup) + urls = getServiceURLs(system, service=service) return ",".join(urls) if urls else "" -def getServiceFailoverURL(system, service=None, setup=False): +def getServiceFailoverURL(system, service=None): """Get failover URLs for service :param str system: system name or full name, like 'Framework/Service'. :param str service: service name, like 'ProxyManager'. - :param str setup: DIRAC setup name, can be defined in dirac.cfg :return: str -- complete list of urls """ system, service = divideFullName(system, service) - systemSection = getSystemSection(system, setup=setup) + systemSection = f"/Systems/{system}" failovers = gConfigurationData.extractOptionFromCFG(f"{systemSection}/FailoverURLs/{service}") if not failovers: return "" @@ -293,8 +255,14 @@ def getGatewayURLs(system="", service=None): siteName = gConfigurationData.extractOptionFromCFG("/LocalSite/Site") if not siteName: return False - gateways = gConfigurationData.extractOptionFromCFG("/DIRAC/Gateways/%s" % siteName) + gateways = gConfigurationData.extractOptionFromCFG(f"/DIRAC/Gateways/{siteName}") if not gateways: return False gateways = List.randomize(List.fromChar(gateways, ",")) return [checkComponentURL(u, system, service) for u in gateways if u] if system and service else gateways + + +def getDisabledDiracxVOs() -> list[str]: + """Get the list of VOs for which DiracX is enabled""" + vos = gConfigurationData.extractOptionFromCFG("/DiracX/DisabledVOs") + return List.fromChar(vos or "", ",") diff --git a/src/DIRAC/ConfigurationSystem/Client/SyncPlugins/CERNLDAPSyncPlugin.py b/src/DIRAC/ConfigurationSystem/Client/SyncPlugins/CERNLDAPSyncPlugin.py index b78369f28e2..a925c1b580e 100644 --- a/src/DIRAC/ConfigurationSystem/Client/SyncPlugins/CERNLDAPSyncPlugin.py +++ b/src/DIRAC/ConfigurationSystem/Client/SyncPlugins/CERNLDAPSyncPlugin.py @@ -12,7 +12,9 @@ class CERNLDAPSyncPlugin: def __init__(self): """Initialise the plugin and underlying LDAP connection.""" self._server = ldap3.Server("ldap://xldap.cern.ch") - self._connection = ldap3.Connection(self._server, client_strategy=ldap3.SAFE_SYNC, auto_bind=True) + self._connection = ldap3.Connection( + self._server, client_strategy=ldap3.SAFE_SYNC, auto_bind=ldap3.AUTO_BIND_NO_TLS + ) def verifyAndUpdateUserInfo(self, username, userDict): """Add the "CERNAccountType" and "PrimaryCERNAccount" values to the CS attributes. @@ -61,11 +63,11 @@ def _getUserInfo(self, commonName): """ status, result, response, _ = self._connection.search( "OU=Users,OU=Organic Units,DC=cern,DC=ch", - "(CN=%s)" % commonName, + f"(CN={commonName})", attributes=["cernAccountOwner", "cernAccountType"], ) if not status: - raise ValueError("Bad status from LDAP search: %s" % result) + raise ValueError(f"Bad status from LDAP search: {result}") if len(response) != 1: raise ValueError(f"Expected exactly one match for CN={commonName} but found {len(response)}") # https://github.com/PyCQA/pylint/issues/4148 diff --git a/src/DIRAC/ConfigurationSystem/Client/Utilities.py b/src/DIRAC/ConfigurationSystem/Client/Utilities.py index 9aaf35241b4..63c38da2354 100644 --- a/src/DIRAC/ConfigurationSystem/Client/Utilities.py +++ b/src/DIRAC/ConfigurationSystem/Client/Utilities.py @@ -1,6 +1,3 @@ -######################################################################## -# Author : Andrei Tsaregorodtsev -######################################################################## """ Utilities for managing DIRAC configuration: """ @@ -13,7 +10,6 @@ from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getDIRACSiteName from DIRAC.ConfigurationSystem.Client.PathFinder import getDatabaseSection from DIRAC.Core.Utilities.Glue2 import getGlue2CEInfo -from DIRAC.ConfigurationSystem.Client.PathFinder import getSystemInstance def getGridVOs(): @@ -177,6 +173,8 @@ def makeNewQueueName(queueName, ceType): ceInfo["Queues"].update(newQueues) + defaultLocalCEType = gConfig.getValue("/Resources/Computing/DefaultLocalCEType", "") + changeSet = set() for site in ceBdiiDict: result = getDIRACSiteName(site) @@ -232,7 +230,6 @@ def makeNewQueueName(queueName, ceType): si00 = ceDict.get("SI00", "Unknown") ceType = ceDict.get("CEType", "Unknown") ram = ceDict.get("MaxRAM", "Unknown") - submissionMode = ceDict.get("SubmissionMode", "Unknown") # Current BDII CE info newarch = ceBdiiDict[site]["CEs"][ce].get("GlueHostArchitecturePlatformType", "").strip() @@ -250,11 +247,8 @@ def makeNewQueueName(queueName, ceType): if newCEType: break if newCEType == "ARC-CE": - newCEType = "ARC" + newCEType = "AREX" - newSubmissionMode = None - if newCEType in ["ARC", "CREAM"]: - newSubmissionMode = "Direct" newRAM = ceInfo.get("GlueHostMainMemoryRAMSize", "").strip() # Protect from unreasonable values if newRAM and int(newRAM) > 150000: @@ -264,10 +258,10 @@ def makeNewQueueName(queueName, ceType): addToChangeSet((ceSection, "architecture", arch, newarch), changeSet) addToChangeSet((ceSection, "OS", OS, newOS), changeSet) addToChangeSet((ceSection, "SI00", si00, newsi00), changeSet) + addToChangeSet((ceSection, "CEType", ceType, newCEType), changeSet) + addToChangeSet((ceSection, "MaxRAM", ram, newRAM), changeSet) - if submissionMode == "Unknown" and newSubmissionMode: - addToChangeSet((ceSection, "SubmissionMode", submissionMode, newSubmissionMode), changeSet) for queue, queueInfo in ceInfo["Queues"].items(): queueStatus = queueInfo["GlueCEStateStatus"] @@ -313,7 +307,7 @@ def makeNewQueueName(queueName, ceType): reqTag = queueDict.get("RequiredTag", "") # LocalCEType can be empty (equivalent to "InProcess") # or "Pool", "Singularity", but also "Pool/Singularity" - localCEType = queueDict.get("LocalCEType", "") + localCEType = queueDict.get("LocalCEType", defaultLocalCEType) try: localCEType_inner = localCEType.split("/")[1] except IndexError: @@ -375,15 +369,6 @@ def getDBParameters(fullname): """Retrieve Database parameters from CS :param str fullname: should be of the form / - defaultHost is the host to return if the option is not found in the CS. - Not used as the method will fail if it cannot be found - defaultPort is the port to return if the option is not found in the CS - defaultUser is the user to return if the option is not found in the CS. - Not usePassword is the password to return if the option is not found in the CS. - Not used as the method will fail if it cannot be found - defaultDB is the db to return if the option is not found in the CS. - Not used as the method will fail if it cannot be found - defaultQueueSize is the QueueSize to return if the option is not found in the CS :return: S_OK(dict)/S_ERROR() - dictionary with the keys: 'host', 'port', 'user', 'password', 'db' and 'queueSize' @@ -392,6 +377,25 @@ def getDBParameters(fullname): cs_path = getDatabaseSection(fullname) parameters = {} + # Check mandatory parameters first: Password, User, Host and DBName + result = gConfig.getOption(cs_path + "/Password") + if not result["OK"]: + # No individual password found, try at the common place + result = gConfig.getOption("/Systems/Databases/Password") + if not result["OK"]: + return S_ERROR("Failed to get the configuration parameter: Password") + dbPass = result["Value"] + parameters["Password"] = dbPass + + result = gConfig.getOption(cs_path + "/User") + if not result["OK"]: + # No individual user name found, try at the common place + result = gConfig.getOption("/Systems/Databases/User") + if not result["OK"]: + return S_ERROR("Failed to get the configuration parameter: User") + dbUser = result["Value"] + parameters["User"] = dbUser + result = gConfig.getOption(cs_path + "/Host") if not result["OK"]: # No host name found, try at the common place @@ -407,6 +411,13 @@ def getDBParameters(fullname): dbHost = "localhost" parameters["Host"] = dbHost + result = gConfig.getOption(cs_path + "/DBName") + if not result["OK"]: + return S_ERROR("Failed to get the configuration parameter: DBName") + dbName = result["Value"] + parameters["DBName"] = dbName + + # Check optional parameters: Port # Mysql standard dbPort = 3306 result = gConfig.getOption(cs_path + "/Port") @@ -419,30 +430,6 @@ def getDBParameters(fullname): dbPort = int(result["Value"]) parameters["Port"] = dbPort - result = gConfig.getOption(cs_path + "/User") - if not result["OK"]: - # No individual user name found, try at the common place - result = gConfig.getOption("/Systems/Databases/User") - if not result["OK"]: - return S_ERROR("Failed to get the configuration parameter: User") - dbUser = result["Value"] - parameters["User"] = dbUser - - result = gConfig.getOption(cs_path + "/Password") - if not result["OK"]: - # No individual password found, try at the common place - result = gConfig.getOption("/Systems/Databases/Password") - if not result["OK"]: - return S_ERROR("Failed to get the configuration parameter: Password") - dbPass = result["Value"] - parameters["Password"] = dbPass - - result = gConfig.getOption(cs_path + "/DBName") - if not result["OK"]: - return S_ERROR("Failed to get the configuration parameter: DBName") - dbName = result["Value"] - parameters["DBName"] = dbName - return S_OK(parameters) @@ -454,9 +441,98 @@ def getElasticDBParameters(fullname): :return: S_OK(dict)/S_ERROR() """ + def _getCACerts(cs_path): + result = gConfig.getOption(cs_path + "/ca_certs") + if not result["OK"]: + # No CA certificate found, try at the common place + result = gConfig.getOption("/Systems/NoSQLDatabases/ca_certs") + if not result["OK"]: + return None + else: + ca_certs = result["Value"] + else: + ca_certs = result["Value"] + return ca_certs + cs_path = getDatabaseSection(fullname) parameters = {} + # Check if connection is through certificates and get certificate parameters + # OpenSearch use certs + result = gConfig.getOption(cs_path + "/CRT") + if not result["OK"]: + # No CRT option found, try at the common place + result = gConfig.getOption("/Systems/NoSQLDatabases/CRT") + if not result["OK"]: + gLogger.debug("Failed to get the configuration parameter: CRT. Using False") + certs = False + else: + certs = result["Value"].lower() in ("true", "yes", "y", "1") + else: + certs = result["Value"].lower() in ("true", "yes", "y", "1") + parameters["CRT"] = certs + + # If connection is through certificates get the mandatory parameters: ca_certs, client_key, client_cert + if parameters["CRT"]: + parameters["Password"] = None + parameters["User"] = None + + ca_certs = _getCACerts(cs_path) + if ca_certs is None: + return S_ERROR("Failed to get the configuration parameter: ca_certs.") + parameters["ca_certs"] = ca_certs + + # OpenSearch client_key + result = gConfig.getOption(cs_path + "/client_key") + if not result["OK"]: + # No client private key found, try at the common place + result = gConfig.getOption("/Systems/NoSQLDatabases/client_key") + if not result["OK"]: + return S_ERROR("Failed to get the configuration parameter: client_key.") + else: + client_key = result["Value"] + else: + client_key = result["Value"] + parameters["client_key"] = client_key + + # OpenSearch client_cert + result = gConfig.getOption(cs_path + "/client_cert") + if not result["OK"]: + # No cient certificate found, try at the common place + result = gConfig.getOption("/Systems/NoSQLDatabases/client_cert") + if not result["OK"]: + return S_ERROR("Failed to get the configuration parameter: client_cert.") + else: + client_cert = result["Value"] + else: + client_cert = result["Value"] + parameters["client_cert"] = client_cert + # If connection is not through certificates get the mandatory parameters: Password, User + else: + result = gConfig.getOption(cs_path + "/Password") + if not result["OK"]: + # No individual password found, try at the common place + result = gConfig.getOption("/Systems/NoSQLDatabases/Password") + if not result["OK"]: + return S_ERROR("Failed to get the configuration parameter: Password.") + dbPass = result["Value"] + parameters["Password"] = dbPass + + result = gConfig.getOption(cs_path + "/User") + if not result["OK"]: + # No individual user name found, try at the common place + result = gConfig.getOption("/Systems/NoSQLDatabases/User") + if not result["OK"]: + return S_ERROR("Failed to get the configuration parameter: User.") + dbUser = result["Value"] + parameters["User"] = dbUser + + # ca_certs is not mandatory + ca_certs = _getCACerts(cs_path) + if ca_certs: + parameters["ca_certs"] = ca_certs + + # Check optional parameters: Host, Port, SSL result = gConfig.getOption(cs_path + "/Host") if not result["OK"]: # No host name found, try at the common place @@ -476,13 +552,13 @@ def getElasticDBParameters(fullname): dbHost = "localhost" parameters["Host"] = dbHost - # Elasticsearch standard port + # OpenSearch standard port result = gConfig.getOption(cs_path + "/Port") if not result["OK"]: # No individual port number found, try at the common place result = gConfig.getOption("/Systems/NoSQLDatabases/Port") if not result["OK"]: - gLogger.warn("No configuration parameter set for Port, assuming URL points to right location") + gLogger.debug("No configuration parameter set for Port, assuming URL points to right location") dbPort = None else: dbPort = int(result["Value"]) @@ -490,105 +566,19 @@ def getElasticDBParameters(fullname): dbPort = int(result["Value"]) parameters["Port"] = dbPort - result = gConfig.getOption(cs_path + "/User") - if not result["OK"]: - # No individual user name found, try at the common place - result = gConfig.getOption("/Systems/NoSQLDatabases/User") - if not result["OK"]: - gLogger.warn( - "Failed to get the configuration parameter: User. Assuming no user/password is provided/needed" - ) - dbUser = None - else: - dbUser = result["Value"] - else: - dbUser = result["Value"] - parameters["User"] = dbUser - - result = gConfig.getOption(cs_path + "/Password") - if not result["OK"]: - # No individual password found, try at the common place - result = gConfig.getOption("/Systems/NoSQLDatabases/Password") - if not result["OK"]: - gLogger.warn( - "Failed to get the configuration parameter: Password. Assuming no user/password is provided/needed" - ) - dbPass = None - else: - dbPass = result["Value"] - else: - dbPass = result["Value"] - parameters["Password"] = dbPass - result = gConfig.getOption(cs_path + "/SSL") if not result["OK"]: # No SSL option found, try at the common place result = gConfig.getOption("/Systems/NoSQLDatabases/SSL") if not result["OK"]: - gLogger.warn("Failed to get the configuration parameter: SSL. Assuming SSL is needed") + gLogger.debug("Failed to get the configuration parameter: SSL. Assuming SSL is needed") ssl = True else: - ssl = False if result["Value"].lower() in ("false", "no", "n") else True + ssl = result["Value"].lower() in ("true", "yes", "y", "1") else: - ssl = False if result["Value"].lower() in ("false", "no", "n") else True + ssl = result["Value"].lower() in ("true", "yes", "y", "1") parameters["SSL"] = ssl - # Elasticsearch use certs - result = gConfig.getOption(cs_path + "/CRT") - if not result["OK"]: - # No CRT option found, try at the common place - result = gConfig.getOption("/Systems/NoSQLDatabases/CRT") - if not result["OK"]: - gLogger.warn("Failed to get the configuration parameter: CRT. Using False") - certs = False - else: - certs = result["Value"] - else: - certs = result["Value"] - parameters["CRT"] = certs - - # Elasticsearch ca_certs - result = gConfig.getOption(cs_path + "/ca_certs") - if not result["OK"]: - # No CA certificate found, try at the common place - result = gConfig.getOption("/Systems/NoSQLDatabases/ca_certs") - if not result["OK"]: - gLogger.warn("Failed to get the configuration parameter: ca_certs. Using None") - ca_certs = None - else: - ca_certs = result["Value"] - else: - ca_certs = result["Value"] - parameters["ca_certs"] = ca_certs - - # Elasticsearch client_key - result = gConfig.getOption(cs_path + "/client_key") - if not result["OK"]: - # No client private key found, try at the common place - result = gConfig.getOption("/Systems/NoSQLDatabases/client_key") - if not result["OK"]: - gLogger.warn("Failed to get the configuration parameter: client_key. Using None") - client_key = None - else: - client_key = result["Value"] - else: - client_key = result["Value"] - parameters["client_key"] = client_key - - # Elasticsearch client_cert - result = gConfig.getOption(cs_path + "/client_cert") - if not result["OK"]: - # No cient certificate found, try at the common place - result = gConfig.getOption("/Systems/NoSQLDatabases/client_cert") - if not result["OK"]: - gLogger.warn("Failed to get the configuration parameter: client_cert. Using None") - client_cert = None - else: - client_cert = result["Value"] - else: - client_cert = result["Value"] - parameters["client_cert"] = client_cert - return S_OK(parameters) @@ -605,7 +595,7 @@ def getDIRACGOCDictionary(): result = gConfig.getConfigurationTree("/Resources/Sites", "Name") if not result["OK"]: - log.error("getConfigurationTree() failed with message: %s" % result["Message"]) + log.error(f"getConfigurationTree() failed with message: {result['Message']}") return S_ERROR("Configuration is corrupted") siteNamesTree = result["Value"] @@ -631,7 +621,7 @@ def getAuthAPI(): :return: str """ - return gConfig.getValue("/Systems/Framework/%s/URLs/AuthAPI" % getSystemInstance("Framework")) + return gConfig.getValue(f"/Systems/Framework/URLs/AuthAPI") def getAuthorizationServerMetadata(issuer=None, ignoreErrors=False): @@ -660,7 +650,7 @@ def getAuthorizationServerMetadata(issuer=None, ignoreErrors=False): try: data["issuer"] = getAuthAPI() except Exception as e: - return S_ERROR("No issuer found in DIRAC authorization server: %s" % repr(e)) + return S_ERROR(f"No issuer found in DIRAC authorization server: {repr(e)}") return S_OK(data) if data["issuer"] else S_ERROR("Cannot find DIRAC Authorization Server issuer.") @@ -670,5 +660,5 @@ def isDownloadProxyAllowed(): :return: S_OK(bool)/S_ERROR() """ - cs_path = "/Systems/Framework/%s/APIs/Auth" % getSystemInstance("Framework") + cs_path = f"/Systems/Framework/APIs/Auth" return gConfig.getValue(cs_path + "/allowProxyDownload", True) diff --git a/src/DIRAC/ConfigurationSystem/Client/VOMS2CSSynchronizer.py b/src/DIRAC/ConfigurationSystem/Client/VOMS2CSSynchronizer.py index e439dc0a1b6..13c663ecb5a 100644 --- a/src/DIRAC/ConfigurationSystem/Client/VOMS2CSSynchronizer.py +++ b/src/DIRAC/ConfigurationSystem/Client/VOMS2CSSynchronizer.py @@ -1,23 +1,27 @@ """ VOMS2CSSyncronizer is a helper class containing the logic for synchronization of the VOMS user data with the DIRAC Registry """ + from collections import defaultdict -from DIRAC import S_OK, S_ERROR, gLogger, gConfig +from diraccfg import CFG -from DIRAC.Core.Security.VOMSService import VOMSService -from DIRAC.Core.Utilities.List import fromChar -from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader -from DIRAC.Core.Utilities.PrettyPrint import printTable +from DIRAC import S_ERROR, S_OK, gConfig, gLogger from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI from DIRAC.ConfigurationSystem.Client.Helpers.Registry import ( - getVOOption, - getVOMSRoleGroupMapping, - getUsersInVO, getAllUsers, - getUserOption, getGroupsForUser, + getUserOption, + getUsersInVO, + getVOMSRoleGroupMapping, + getVOOption, ) +from DIRAC.Core.Security.IAMService import IAMService +from DIRAC.Core.Security.VOMSService import VOMSService +from DIRAC.Core.Utilities.List import fromChar +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader +from DIRAC.Core.Utilities.PrettyPrint import printTable +from DIRAC.Core.Utilities.ReturnValues import convertToReturnValue, returnValueOrRaise def _getUserNameFromMail(mail): @@ -64,6 +68,8 @@ def _getUserNameFromDN(dn, vo): key, value = "CN", entry else: key, value = entry.split("=") + key = key.strip() + value = value.strip() if key.upper() == "CN": ind = value.find("(") # Strip of possible words in parenthesis in the name @@ -125,6 +131,10 @@ def __init__( autoDeleteUsers=False, autoLiftSuspendedStatus=False, syncPluginName=None, + compareWithIAM=False, + useIAM=False, + accessToken=None, + forceNickname=False, ): """VOMS2CSSynchronizer class constructor @@ -134,6 +144,9 @@ def __init__( :param autoDeleteUsers: flag to automatically delete users from CS if no more in VOMS :param autoLiftSuspendedStatus: flag to automatically remove Suspended status in CS :param syncPluginName: name of the plugin to validate or extend users' info + :param compareWithIAM: if true, also dump the list of users from IAM and compare + :param useIAM: if True, use Iam instead of VOMS + :param accessToken: if talking to IAM, needs a token with scim:read property :return: None """ @@ -143,7 +156,7 @@ def __init__( self.vo = vo self.vomsVOName = getVOOption(vo, "VOMSName", "") if not self.vomsVOName: - raise Exception("VOMS name not defined for VO %s" % vo) + raise Exception(f"VOMS name not defined for VO {vo}") self.adminMsgs = {"Errors": [], "Info": []} self.vomsUserDict = {} self.autoModifyUsers = autoModifyUsers @@ -152,11 +165,16 @@ def __init__( self.autoLiftSuspendedStatus = autoLiftSuspendedStatus self.voChanged = False self.syncPlugin = None + self.iamSrv = None + self.compareWithIAM = compareWithIAM + self.useIAM = useIAM + self.accessToken = accessToken + self.forceNickname = forceNickname if syncPluginName: objLoader = ObjectLoader() _class = objLoader.loadObject( - "ConfigurationSystem.Client.SyncPlugins.%sSyncPlugin" % syncPluginName, "%sSyncPlugin" % syncPluginName + f"ConfigurationSystem.Client.SyncPlugins.{syncPluginName}SyncPlugin", f"{syncPluginName}SyncPlugin" ) if not _class["OK"]: @@ -164,6 +182,62 @@ def __init__( self.syncPlugin = _class["Value"]() + def compare_entry(self, iam_entry, voms_entry, is_robot): + """Compare a VOMS and IAM entry""" + + if iam_entry.get("mail") != voms_entry.get("mail"): + self.log.info( + "Difference in mails", + f"{iam_entry['nickname']} - mail : {iam_entry.get('mail')} vs {voms_entry.get('mail')}", + ) + if is_robot: + self.log.info("\t this is expected for robots !") + + for field in ("CA", "certSuspended", "suspended", "mail", "nickname"): + if iam_entry.get(field) != voms_entry.get(field): + self.log.info( + f"Difference in {field}", + f"{iam_entry['nickname']} - {field} : {iam_entry.get(field)} vs {voms_entry.get(field)}", + ) + + if sorted(iam_entry["Roles"]) != sorted(voms_entry["Roles"]): + self.log.info( + "Difference in roles", + f"{iam_entry['nickname']} - Roles : {iam_entry['Roles']} vs {voms_entry['Roles']}", + ) + + def compareUsers(self, voms_users, iam_users): + missing_in_iam = set(voms_users) - set(iam_users) + if missing_in_iam: + self.log.info("Missing entries in IAM:", missing_in_iam) + else: + self.log.info("No entry missing in IAM, GOOD !") + # suspended_in_voms = {dn for dn in voms_users if voms_users[dn]["suspended"]} + missing_in_voms = set(iam_users) - set(voms_users) + + if missing_in_voms: + self.log.info("Entries in IAM that are not in VOMS:", missing_in_voms) + else: + self.log.info("No extra entry entries in IAM, GOOD !") + + for dn in set(iam_users) & set(voms_users): + is_robot = "CN=Robot:" in dn + self.compare_entry(iam_users[dn], voms_users[dn], is_robot=is_robot) + + @convertToReturnValue + def _getUsers(self): + if self.compareWithIAM or self.useIAM: + self.iamSrv = IAMService(self.accessToken, vo=self.vo, forceNickname=self.forceNickname) + iam_users = returnValueOrRaise(self.iamSrv.getUsers()) + if self.useIAM: + return iam_users + + vomsSrv = VOMSService(self.vo) + voms_users = returnValueOrRaise(vomsSrv.getUsers()) + if self.compareWithIAM: + self.compareUsers(voms_users.get("Users", {}), iam_users.get("Users", {})) + return voms_users + def syncCSWithVOMS(self): """Performs the synchronization of the DIRAC registry with the VOMS data. The resulting CSAPI object containing modifications is returned as part of the output dictionary. @@ -184,15 +258,13 @@ def syncCSWithVOMS(self): noVOMSGroups = result["Value"]["NoVOMS"] noSyncVOMSGroups = result["Value"]["NoSyncVOMS"] - vomsSrv = VOMSService(self.vo) - - # Get VOMS users - result = vomsSrv.getUsers() + result = self._getUsers() if not result["OK"]: - self.log.error("Could not retrieve user information from VOMS", result["Message"]) + self.log.error("Could not retrieve user information", result["Message"]) return result - - self.vomsUserDict = result["Value"] + if getUserErrors := result["Value"]["Errors"]: + self.adminMsgs["Errors"].extend(getUserErrors) + self.vomsUserDict = result["Value"]["Users"] message = f"There are {len(self.vomsUserDict)} user entries in VOMS for VO {self.vomsVOName}" self.adminMsgs["Info"].append(message) self.log.info("VOMS user entries", message) @@ -236,7 +308,7 @@ def syncCSWithVOMS(self): nonVOUserDict = result["Value"] # Process users - defaultVOGroup = getVOOption(self.vo, "DefaultGroup", "%s_user" % self.vo) + defaultVOGroup = getVOOption(self.vo, "DefaultGroup", f"{self.vo}_user") # If a user is (previously put by hand) in a "QuarantineGroup", # then the default group will be ignored. # So, this option is only considered for the case of existing users. @@ -263,6 +335,10 @@ def syncCSWithVOMS(self): # Check the nickName in the same VO to see if the user is already registered # with another DN nickName = self.vomsUserDict[dn].get("nickname") + if not nickName and self.forceNickname: + resultDict["NoNickname"].append(self.vomsUserDict[dn]) + self.log.error("No nickname defined for", self.vomsUserDict[dn]) + continue if nickName in diracUserDict or nickName in newAddedUserDict: diracName = nickName # This is a flag for adding the new DN to an already existing user @@ -270,24 +346,27 @@ def syncCSWithVOMS(self): # We have a real new user if not diracName: + # if we have a nickname, we use the nickname no + # matter what so we can have users from different + # VOs with the same nickname / username if nickName: - newDiracName = nickName + newDiracName = nickName.strip() else: newDiracName = self.getUserName(dn) + # If the chosen user name exists already, append a distinguishing suffix + ind = 1 + trialName = newDiracName + while newDiracName in allDiracUsers: + # We have a user with the same name but with a different DN + newDiracName = "%s_%d" % (trialName, ind) + ind += 1 + # Do not consider users with Suspended status in VOMS if self.vomsUserDict[dn]["suspended"] or self.vomsUserDict[dn]["certSuspended"]: resultDict["SuspendedUsers"].append(newDiracName) continue - # If the chosen user name exists already, append a distinguishing suffix - ind = 1 - trialName = newDiracName - while newDiracName in allDiracUsers: - # We have a user with the same name but with a different DN - newDiracName = "%s_%d" % (trialName, ind) - ind += 1 - # We now have everything to add the new user userDict = {"DN": dn, "CA": self.vomsUserDict[dn]["CA"], "Email": self.vomsUserDict[dn]["mail"]} groupsWithRole = [] @@ -309,7 +388,7 @@ def syncCSWithVOMS(self): ) continue - message = "\n Added new user %s:\n" % newDiracName + message = f"\n Added new user {newDiracName}:\n" for key in userDict: message += f" {key}: {str(userDict[key])}\n" self.adminMsgs["Info"].append(message) @@ -318,11 +397,16 @@ def syncCSWithVOMS(self): self.log.info(f"Adding new user {newDiracName}: {str(userDict)}") result = self.csapi.modifyUser(newDiracName, userDict, createIfNonExistant=True) if not result["OK"]: - self.log.warn("Failed adding new user %s" % newDiracName) + self.log.warn(f"Failed adding new user {newDiracName}", result["Message"]) resultDict["NewUsers"].append(newDiracName) newAddedUserDict[newDiracName] = userDict continue + # If we have a new user with multiple DN, + # it's a bit tricky, so first create it with a single one + # and at the next iteration add more DNs + if diracName in newAddedUserDict: + continue # We have an already existing user modified = False suspendedInVOMS = self.vomsUserDict[dn]["suspended"] or self.vomsUserDict[dn]["certSuspended"] @@ -393,31 +477,31 @@ def syncCSWithVOMS(self): if not existingGroups and diracName in allDiracUsers: groups = getGroupsForUser(diracName) if groups["OK"]: - self.log.info("Found groups for user {} {}".format(diracName, groups["Value"])) + self.log.info(f"Found groups for user {diracName} {groups['Value']}") userDict["Groups"] = list(set(groups["Value"] + keepGroups)) addedGroups = list(set(userDict["Groups"]) - set(groups["Value"])) modified = True - message = "\n Modified user %s:\n" % diracName - message += " Added to group(s) %s\n" % ",".join(addedGroups) + message = f"\n Modified user {diracName}:\n" + message += f" Added to group(s) {','.join(addedGroups)}\n" self.adminMsgs["Info"].append(message) # Check if something changed before asking CSAPI to modify if diracName in diracUserDict: - message = "\n Modified user %s:\n" % diracName + message = f"\n Modified user {diracName}:\n" modMsg = "" for key in userDict: if key == "Groups": addedGroups = set(userDict[key]) - set(diracUserDict.get(diracName, {}).get(key, [])) removedGroups = set(diracUserDict.get(diracName, {}).get(key, [])) - set(userDict[key]) if addedGroups: - modMsg += " Added to group(s) %s\n" % ",".join(addedGroups) + modMsg += f" Added to group(s) {','.join(addedGroups)}\n" if removedGroups: - modMsg += " Removed from group(s) %s\n" % ",".join(removedGroups) + modMsg += f" Removed from group(s) {','.join(removedGroups)}\n" elif key == "Suspended": if userDict["Suspended"] == "None": modMsg += " Suspended status removed\n" else: - modMsg += " User Suspended in VOs: %s\n" % userDict["Suspended"] + modMsg += f" User Suspended in VOs: {userDict['Suspended']}\n" else: oldValue = str(diracUserDict.get(diracName, {}).get(key, "")) if str(userDict[key]) != oldValue: @@ -451,8 +535,8 @@ def syncCSWithVOMS(self): if result["OK"] and result["Value"]: self.log.info(f"Modified user {user}: {str(userDict)}") self.voChanged = True - message = "\n Modified user %s:\n" % user - modMsg = " Removed from group(s) %s\n" % ",".join(removedGroups) + message = f"\n Modified user {user}:\n" + modMsg = f" Removed from group(s) {','.join(removedGroups)}\n" self.adminMsgs["Info"].append(message + modMsg) resultDict["ModifiedUsers"].append(user) continue @@ -487,13 +571,13 @@ def syncCSWithVOMS(self): if oldUsers: self.voChanged = True if self.autoDeleteUsers: - self.log.info("The following users will be deleted: %s" % str(oldUsers)) + self.log.info(f"The following users will be deleted: {str(oldUsers)}") result = self.csapi.deleteUsers(oldUsers) if result["OK"]: - self.adminMsgs["Info"].append("The following users are deleted from CS:\n %s\n" % str(oldUsers)) + self.adminMsgs["Info"].append(f"The following users are deleted from CS:\n {str(oldUsers)}\n") resultDict["DeletedUsers"] = oldUsers else: - self.adminMsgs["Errors"].append("Error in deleting users from CS:\n %s" % str(oldUsers)) + self.adminMsgs["Errors"].append(f"Error in deleting users from CS:\n {str(oldUsers)}") self.log.error("Error while user deletion from CS", result) else: self.adminMsgs["Info"].append( @@ -501,6 +585,16 @@ def syncCSWithVOMS(self): ) self.log.info("The following users to be checked for deletion:", "\n\t".join(sorted(oldUsers))) + # Try to fill in the DiracX section + if self.useIAM: + iam_subs = self.iamSrv.getUsersSub() + diracx_vo_config = {"DiracX": {"CsSync": {"VOs": {self.vo: {"UserSubjects": iam_subs}}}}} + iam_sub_cfg = CFG() + iam_sub_cfg.loadFromDict(diracx_vo_config) + result = self.csapi.mergeFromCFG(iam_sub_cfg) + if not result["OK"]: + return result + resultDict["CSAPI"] = self.csapi resultDict["AdminMessages"] = self.adminMsgs resultDict["VOChanged"] = self.voChanged @@ -519,7 +613,7 @@ def getVOUserData(self, refreshFlag=False): # Get DIRAC users diracUsers = getUsersInVO(self.vo) if not diracUsers: - return S_ERROR("No VO users found for %s" % self.vo) + return S_ERROR(f"No VO users found for {self.vo}") if refreshFlag: result = self.csapi.downloadCSData() @@ -571,13 +665,13 @@ def getVOUserReport(self): if multiDNUsers: output += "\nUsers with multiple DNs:\n" for user in multiDNUsers: - output += " %s:\n" % user + output += f" {user}:\n" for dn in multiDNUsers[user]: - output += " %s\n" % dn + output += f" {dn}\n" if suspendedUsers: - output += "\n%d suspended users:\n" % len(suspendedUsers) - output += " %s" % ",".join(suspendedUsers) + output += f"\n{len(suspendedUsers)} suspended users:\n" + output += f" {','.join(suspendedUsers)}" return S_OK(output) diff --git a/src/DIRAC/ConfigurationSystem/Client/test/Test_LocalConfiguration.py b/src/DIRAC/ConfigurationSystem/Client/test/Test_LocalConfiguration.py index f2113b493e7..d27b17ca70c 100644 --- a/src/DIRAC/ConfigurationSystem/Client/test/Test_LocalConfiguration.py +++ b/src/DIRAC/ConfigurationSystem/Client/test/Test_LocalConfiguration.py @@ -205,7 +205,7 @@ def test_register_arguments(localCFG, argsData, expected): for a in arg[2].split("\n"): argBlock += "\n" + a if values: - argBlock += " [%s]" % ", ".join(values) + argBlock += f" [{', '.join(values)}]" if default: argBlock += " [default: defVal]" if not mandatory: diff --git a/src/DIRAC/ConfigurationSystem/Client/test/Test_PathFinder.py b/src/DIRAC/ConfigurationSystem/Client/test/Test_PathFinder.py index 0025eb2a1ac..3348140e438 100644 --- a/src/DIRAC/ConfigurationSystem/Client/test/Test_PathFinder.py +++ b/src/DIRAC/ConfigurationSystem/Client/test/Test_PathFinder.py @@ -12,33 +12,19 @@ mergedCFG = CFG() mergedCFG.loadFromBuffer( """ -DIRAC -{ - Setup=TestSetup - Setups - { - TestSetup - { - WorkloadManagement=MyWM - } - } -} Systems { WorkloadManagement { - MyWM + URLs + { + Service1 = dips://server1:1234/WorkloadManagement/Service1 + Service2 = dips://$MAINSERVERS$:5678/WorkloadManagement/Service2 + } + FailoverURLs { - URLs - { - Service1 = dips://server1:1234/WorkloadManagement/Service1 - Service2 = dips://$MAINSERVERS$:5678/WorkloadManagement/Service2 - } - FailoverURLs - { - Service2 = dips://failover1:5678/WorkloadManagement/Service2 - Service2 += dips://failover2:5678/WorkloadManagement/Service2 - } + Service2 = dips://failover1:5678/WorkloadManagement/Service2 + Service2 += dips://failover2:5678/WorkloadManagement/Service2 } } } @@ -63,53 +49,44 @@ def pathFinder(monkeypatch): return PathFinder -def test_getDIRACSetup(pathFinder): - """Test getDIRACSetup""" - assert pathFinder.getDIRACSetup() == "TestSetup" - - @pytest.mark.parametrize( - "system, componentName, setup, componentType, result", + "system, componentName, componentType, result", [ ( "WorkloadManagement/SandboxStoreHandler", False, - False, "Services", - "/Systems/WorkloadManagement/MyWM/Services/SandboxStoreHandler", + "/Systems/WorkloadManagement/Services/SandboxStoreHandler", ), ( "WorkloadManagement", "SandboxStoreHandler", - False, "Services", - "/Systems/WorkloadManagement/MyWM/Services/SandboxStoreHandler", + "/Systems/WorkloadManagement/Services/SandboxStoreHandler", ), # tricky case one could expect that if entity string is wrong # than some kind of error will be returned, but it is not the case ( "WorkloadManagement/SimpleLogConsumer", False, - False, "NonRonsumersNon", - "/Systems/WorkloadManagement/MyWM/NonRonsumersNon/SimpleLogConsumer", + "/Systems/WorkloadManagement/NonRonsumersNon/SimpleLogConsumer", ), ], ) -def test_getComponentSection(pathFinder, system, componentName, setup, componentType, result): +def test_getComponentSection(pathFinder, system, componentName, componentType, result): """Test getComponentSection""" - assert pathFinder.getComponentSection(system, componentName, setup, componentType) == result + assert pathFinder.getComponentSection(system, componentName, componentType) == result @pytest.mark.parametrize( - "system, setup, result", + "system, result", [ - ("WorkloadManagement", False, "/Systems/WorkloadManagement/MyWM/URLs"), - ("WorkloadManagement", "TestSetup", "/Systems/WorkloadManagement/MyWM/URLs"), + ("WorkloadManagement", "/Systems/WorkloadManagement/MyWM/URLs"), ], ) -def test_getSystemURLSection(pathFinder, system, setup, result): - assert pathFinder.getSystemURLs(system, setup) +def test_getSystemURLSection(pathFinder, system, result): + assert pathFinder.getSystemURLs(system) @pytest.mark.parametrize( @@ -205,11 +182,10 @@ def test_getServiceURLs(pathFinder, serviceName, service, failover, result): @pytest.mark.parametrize( - "system, setup, failover, result", + "system, failover, result", [ ( "WorkloadManagement", - None, False, { "Service1": {"dips://server1:1234/WorkloadManagement/Service1"}, @@ -221,7 +197,6 @@ def test_getServiceURLs(pathFinder, serviceName, service, failover, result): ), ( "WorkloadManagement", - None, True, { "Service1": {"dips://server1:1234/WorkloadManagement/Service1"}, @@ -235,8 +210,8 @@ def test_getServiceURLs(pathFinder, serviceName, service, failover, result): ), ], ) -def test_getSystemURLs(pathFinder, system, setup, failover, result): - sysDict = pathFinder.getSystemURLs(system, setup=setup, failover=failover) +def test_getSystemURLs(pathFinder, system, failover, result): + sysDict = pathFinder.getSystemURLs(system, failover=failover) for service in sysDict: assert set(sysDict[service]) == result[service] diff --git a/src/DIRAC/ConfigurationSystem/ConfigTemplate.cfg b/src/DIRAC/ConfigurationSystem/ConfigTemplate.cfg index 3f40397c5b0..5776ba856db 100644 --- a/src/DIRAC/ConfigurationSystem/ConfigTemplate.cfg +++ b/src/DIRAC/ConfigurationSystem/ConfigTemplate.cfg @@ -1,9 +1,10 @@ Services { ##BEGIN Server + # This is the master CS, which is exposed via Tornado but at port 9135 Server { - HandlerPath = DIRAC/ConfigurationSystem/Service/ConfigurationHandler.py + HandlerPath = DIRAC/ConfigurationSystem/Service/TornadoConfigurationHandler.py Port = 9135 # Subsection to configure authorization over the service Authorization @@ -21,6 +22,27 @@ Services } } ##END + ##BEGIN TornadoServer + # This is the slave CS, exposed via standard Tornado + TornadoConfiguration + { + Protocol = https + # Subsection to configure authorization over the service + Authorization + { + # Default authorization + Default = authenticated + #Define who can commit new configuration + commitNewData = CSAdministrator + # Define who can roll back the configuration to a previous version + rollbackToVersion = CSAdministrator + # Define who can get version contents + getVersionContents = ServiceAdministrator + getVersionContents += CSAdministrator + forceGlobalConfigurationUpdate = CSAdministrator + } + } + ##END } Agents { @@ -71,6 +93,13 @@ Agents DryRun = True # Name of the plugin to validate or expand user's info. See :py:mod:`DIRAC.ConfigurationSystem.Client.SyncPlugins.DummySyncPlugin` SyncPluginName = + # If set to true, will query the VO IAM server for the list of user, and print + # a comparison of what is with VOMS + CompareWithIAM = False + # If set to true, will only query IAM and return the list of users from there + UseIAM = False + # If set to true only users with a nickname attribute defined in the IAM are created in DIRAC + ForceNickname = False } ##END ##BEGIN GOCDB2CSAgent diff --git a/src/DIRAC/ConfigurationSystem/Service/ConfigurationHandler.py b/src/DIRAC/ConfigurationSystem/Service/ConfigurationHandler.py index af020b3e4d2..398123a402a 100755 --- a/src/DIRAC/ConfigurationSystem/Service/ConfigurationHandler.py +++ b/src/DIRAC/ConfigurationSystem/Service/ConfigurationHandler.py @@ -108,7 +108,7 @@ def export_getVersionContents(cls, versionList): if retVal["OK"]: contentsList.append(retVal["Value"]) else: - return S_ERROR("Can't get contents for version {}: {}".format(version, retVal["Message"])) + return S_ERROR(f"Can't get contents for version {version}: {retVal['Message']}") return S_OK(contentsList) types_rollbackToVersion = [str] @@ -116,7 +116,7 @@ def export_getVersionContents(cls, versionList): def export_rollbackToVersion(self, version): retVal = gServiceInterface.getVersionContents(version) if not retVal["OK"]: - return S_ERROR("Can't get contents for version {}: {}".format(version, retVal["Message"])) + return S_ERROR(f"Can't get contents for version {version}: {retVal['Message']}") credDict = self.getRemoteCredentials() if "DN" not in credDict or "username" not in credDict: return S_ERROR("You must be authenticated!") diff --git a/src/DIRAC/ConfigurationSystem/Service/TornadoConfigurationHandler.py b/src/DIRAC/ConfigurationSystem/Service/TornadoConfigurationHandler.py index 2ce620f9a0b..970f2526bda 100644 --- a/src/DIRAC/ConfigurationSystem/Service/TornadoConfigurationHandler.py +++ b/src/DIRAC/ConfigurationSystem/Service/TornadoConfigurationHandler.py @@ -59,9 +59,9 @@ def export_getCompressedDataIfNewer(self, sClientVersion): def export_publishSlaveServer(self, sURL): """ - Used by slave server to register as a slave server. + Used by worker server to register as a worker server. - :param sURL: The url of the slave server. + :param sURL: The url of the worker server. """ self.ServiceInterface.publishSlaveServer(sURL) return S_OK() @@ -105,7 +105,7 @@ def export_getVersionContents(self, versionList): if retVal["OK"]: contentsList.append(retVal["Value"]) else: - return S_ERROR("Can't get contents for version {}: {}".format(version, retVal["Message"])) + return S_ERROR(f"Can't get contents for version {version}: {retVal['Message']}") return S_OK(contentsList) def export_rollbackToVersion(self, version): @@ -116,7 +116,7 @@ def export_rollbackToVersion(self, version): """ retVal = self.ServiceInterface.getVersionContents(version) if not retVal["OK"]: - return S_ERROR("Can't get contents for version {}: {}".format(version, retVal["Message"])) + return S_ERROR(f"Can't get contents for version {version}: {retVal['Message']}") credDict = self.getRemoteCredentials() if "DN" not in credDict or "username" not in credDict: return S_ERROR("You must be authenticated!") diff --git a/src/DIRAC/ConfigurationSystem/private/ConfigurationClient.py b/src/DIRAC/ConfigurationSystem/private/ConfigurationClient.py index b15e0e1464b..441406ba84c 100755 --- a/src/DIRAC/ConfigurationSystem/private/ConfigurationClient.py +++ b/src/DIRAC/ConfigurationSystem/private/ConfigurationClient.py @@ -136,7 +136,7 @@ def getOption(self, optionPath, typeValue=None): if optionValue is None: return S_ERROR( - "Path %s does not exist or it's not an option" % optionPath, + f"Path {optionPath} does not exist or it's not an option", callStack=["ConfigurationClient.getOption"], ) @@ -195,7 +195,7 @@ def getSections(self, sectionPath, listOrdered=True): if isinstance(sectionList, list): return S_OK(sectionList) else: - return S_ERROR("Path %s does not exist or it's not a section" % sectionPath) + return S_ERROR(f"Path {sectionPath} does not exist or it's not a section") def getOptions(self, sectionPath, listOrdered=True): """Get configuration options @@ -210,7 +210,7 @@ def getOptions(self, sectionPath, listOrdered=True): if isinstance(optionList, list): return S_OK(optionList) else: - return S_ERROR("Path %s does not exist or it's not a section" % sectionPath) + return S_ERROR(f"Path {sectionPath} does not exist or it's not a section") def getOptionsDict(self, sectionPath): """Get configuration options in dictionary @@ -227,7 +227,7 @@ def getOptionsDict(self, sectionPath): optionsDict[option] = gConfigurationData.extractOptionFromCFG(f"{sectionPath}/{option}") return S_OK(optionsDict) else: - return S_ERROR("Path %s does not exist or it's not a section" % sectionPath) + return S_ERROR(f"Path {sectionPath} does not exist or it's not a section") def getOptionsDictRecursively(self, sectionPath): """Get configuration options in dictionary recursively @@ -237,7 +237,7 @@ def getOptionsDictRecursively(self, sectionPath): :return: S_OK(dict)/S_ERROR() """ if not gConfigurationData.mergedCFG.isSection(sectionPath): - return S_ERROR("Path %s does not exist or it's not a section" % sectionPath) + return S_ERROR(f"Path {sectionPath} does not exist or it's not a section") return S_OK(gConfigurationData.mergedCFG.getAsDict(sectionPath)) def getConfigurationTree(self, root="", *filters): @@ -272,7 +272,7 @@ def getConfigurationTree(self, root="", *filters): # get options of current root options = self.getOptionsDict(root) if not options["OK"]: - return S_ERROR("getOptionsDict() failed with message: %s" % options["Message"]) + return S_ERROR(f"getOptionsDict() failed with message: {options['Message']}") for key, value in options["Value"].items(): path = cfgPath(root, key) @@ -288,13 +288,13 @@ def getConfigurationTree(self, root="", *filters): # get subsections of the root sections = self.getSections(root) if not sections["OK"]: - return S_ERROR("getSections() failed with message: %s" % sections["Message"]) + return S_ERROR(f"getSections() failed with message: {sections['Message']}") # recursively go through subsections and get their subsections for section in sections["Value"]: subtree = self.getConfigurationTree(f"{root}/{section}", *filters) if not subtree["OK"]: - return S_ERROR("getConfigurationTree() failed with message: %s" % sections["Message"]) + return S_ERROR(f"getConfigurationTree() failed with message: {sections['Message']}") result.update(subtree["Value"]) return S_OK(result) diff --git a/src/DIRAC/ConfigurationSystem/private/ConfigurationData.py b/src/DIRAC/ConfigurationSystem/private/ConfigurationData.py index 4f8974afc77..46b2d524d1a 100755 --- a/src/DIRAC/ConfigurationSystem/private/ConfigurationData.py +++ b/src/DIRAC/ConfigurationSystem/private/ConfigurationData.py @@ -7,9 +7,11 @@ import _thread import time import datetime -import DIRAC +import secrets from diraccfg import CFG + +import DIRAC from DIRAC.Core.Utilities.File import mkDir from DIRAC.Core.Utilities import List from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR @@ -38,10 +40,10 @@ def __init__(self, loadDefaultCFG=True): self.remoteServerList = [] if loadDefaultCFG: defaultCFGFile = os.path.join(DIRAC.rootPath, "etc", "dirac.cfg") - gLogger.debug("dirac.cfg should be at", "%s" % defaultCFGFile) + gLogger.debug("dirac.cfg should be at", f"{defaultCFGFile}") retVal = self.loadFile(defaultCFGFile) if not retVal["OK"]: - gLogger.warn("Can't load %s file" % defaultCFGFile) + gLogger.debug(f"Can't load {defaultCFGFile} file") self.sync() def getBackupDir(self): @@ -52,12 +54,12 @@ def sync(self): self.mergedCFG = self.remoteCFG.mergeWith(self.localCFG) self.remoteServerList = [] localServers = self.extractOptionFromCFG( - "%s/Servers" % self.configurationPath, self.localCFG, disableDangerZones=True + f"{self.configurationPath}/Servers", self.localCFG, disableDangerZones=True ) if localServers: self.remoteServerList.extend(List.fromChar(localServers, ",")) remoteServers = self.extractOptionFromCFG( - "%s/Servers" % self.configurationPath, self.remoteCFG, disableDangerZones=True + f"{self.configurationPath}/Servers", self.remoteCFG, disableDangerZones=True ) if remoteServers: self.remoteServerList.extend(List.fromChar(remoteServers, ",")) @@ -70,7 +72,7 @@ def loadFile(self, fileName): fileCFG.loadFromFile(fileName) except OSError: self.localCFG = self.localCFG.mergeWith(fileCFG) - return S_ERROR("Can't load a cfg file '%s'" % fileName) + return S_ERROR(f"Can't load a cfg file '{fileName}'") return self.mergeWithLocal(fileCFG) def mergeWithLocal(self, extraCFG): @@ -81,7 +83,7 @@ def mergeWithLocal(self, extraCFG): gLogger.debug("CFG merged") except Exception as e: self.unlock() - return S_ERROR("Cannot merge with new cfg: %s" % str(e)) + return S_ERROR(f"Cannot merge with new cfg: {str(e)}") self.sync() return S_OK() @@ -102,7 +104,7 @@ def loadConfigurationData(self, fileName=False): self.lock() try: if not fileName: - fileName = "%s.cfg" % name + fileName = f"{name}.cfg" if fileName[0] != "/": fileName = os.path.join(DIRAC.rootPath, "etc", fileName) self.remoteCFG.loadFromFile(fileName) @@ -201,65 +203,59 @@ def deleteOptionInCFG(self, path, cfg=False): def generateNewVersion(self): self.setVersion(str(datetime.datetime.utcnow())) self.sync() - gLogger.info("Generated new version %s" % self.getVersion()) + gLogger.info(f"Generated new version {self.getVersion()}") def setVersion(self, version, cfg=False): if not cfg: cfg = self.remoteCFG - self.setOptionInCFG("%s/Version" % self.configurationPath, version, cfg) + self.setOptionInCFG(f"{self.configurationPath}/Version", version, cfg) def getVersion(self, cfg=False): if not cfg: cfg = self.remoteCFG - value = self.extractOptionFromCFG("%s/Version" % self.configurationPath, cfg) + value = self.extractOptionFromCFG(f"{self.configurationPath}/Version", cfg) if value: return value return "0" def getName(self): - return self.extractOptionFromCFG("%s/Name" % self.configurationPath, self.mergedCFG) + return self.extractOptionFromCFG(f"{self.configurationPath}/Name", self.mergedCFG) def exportName(self): - return self.setOptionInCFG("%s/Name" % self.configurationPath, self.getName(), self.remoteCFG) + return self.setOptionInCFG(f"{self.configurationPath}/Name", self.getName(), self.remoteCFG) def getRefreshTime(self): try: - return int(self.extractOptionFromCFG("%s/RefreshTime" % self.configurationPath, self.mergedCFG)) + return int(self.extractOptionFromCFG(f"{self.configurationPath}/RefreshTime", self.mergedCFG)) except Exception: return 300 def getPropagationTime(self): try: - return int(self.extractOptionFromCFG("%s/PropagationTime" % self.configurationPath, self.mergedCFG)) + return int(self.extractOptionFromCFG(f"{self.configurationPath}/PropagationTime", self.mergedCFG)) except Exception: return 300 def getSlavesGraceTime(self): try: - return int(self.extractOptionFromCFG("%s/SlavesGraceTime" % self.configurationPath, self.mergedCFG)) + return int(self.extractOptionFromCFG(f"{self.configurationPath}/SlavesGraceTime", self.mergedCFG)) except Exception: return 600 def mergingEnabled(self): try: - val = self.extractOptionFromCFG("%s/EnableAutoMerge" % self.configurationPath, self.mergedCFG) + val = self.extractOptionFromCFG(f"{self.configurationPath}/EnableAutoMerge", self.mergedCFG) return val.lower() in ("yes", "true", "y") except Exception: return False def getAutoPublish(self): - value = self.extractOptionFromCFG("%s/AutoPublish" % self.configurationPath, self.localCFG) - if value and value.lower() in ("no", "false", "n"): - return False - else: - return True + value = self.extractOptionFromCFG(f"{self.configurationPath}/AutoPublish", self.localCFG) + return not bool(value and value.lower() in ("no", "false", "n")) def getAutoSlaveSync(self): - value = self.extractOptionFromCFG("%s/AutoSlaveSync" % self.configurationPath, self.localCFG) - if value and value.lower() in ("no", "false", "n"): - return False - else: - return True + value = self.extractOptionFromCFG(f"{self.configurationPath}/AutoSlaveSync", self.localCFG) + return not bool(value and value.lower() in ("no", "false", "n")) def getServers(self): return list(self.remoteServerList) @@ -268,17 +264,17 @@ def getConfigurationGateway(self): return self.extractOptionFromCFG("/DIRAC/Gateway", self.localCFG) def setServers(self, sServers): - self.setOptionInCFG("%s/Servers" % self.configurationPath, sServers, self.remoteCFG) + self.setOptionInCFG(f"{self.configurationPath}/Servers", sServers, self.remoteCFG) self.sync() def deleteLocalOption(self, optionPath): self.deleteOptionInCFG(optionPath, self.localCFG) def getMasterServer(self): - return self.extractOptionFromCFG("%s/MasterServer" % self.configurationPath, self.remoteCFG) + return self.extractOptionFromCFG(f"{self.configurationPath}/MasterServer", self.remoteCFG) def setMasterServer(self, sURL): - self.setOptionInCFG("%s/MasterServer" % self.configurationPath, sURL, self.remoteCFG) + self.setOptionInCFG(f"{self.configurationPath}/MasterServer", sURL, self.remoteCFG) self.sync() def getCompressedData(self): @@ -287,11 +283,8 @@ def getCompressedData(self): return self.__compressedConfigurationData def isMaster(self): - value = self.extractOptionFromCFG("%s/Master" % self.configurationPath, self.localCFG) - if value and value.lower() in ("yes", "true", "y"): - return True - else: - return False + value = self.extractOptionFromCFG(f"{self.configurationPath}/Master", self.localCFG) + return bool(value and value.lower() in ("yes", "true", "y")) def getServicesPath(self): return "/Services" @@ -304,24 +297,20 @@ def isService(self): def useServerCertificate(self): value = self.extractOptionFromCFG("/DIRAC/Security/UseServerCertificate") - if value and value.lower() in ("y", "yes", "true"): - return True - return False + return bool(value and value.lower() in ("yes", "true", "y")) def skipCACheck(self): value = self.extractOptionFromCFG("/DIRAC/Security/SkipCAChecks") - if value and value.lower() in ("y", "yes", "true"): - return True - return False + return bool(value and value.lower() in ("yes", "true", "y")) def dumpLocalCFGToFile(self, fileName): try: with open(fileName, "w") as fd: fd.write(str(self.localCFG)) - gLogger.verbose("Configuration file dumped", "'%s'" % fileName) + gLogger.verbose("Configuration file dumped", f"'{fileName}'") except OSError: - gLogger.error("Can't dump cfg file", "'%s'" % fileName) - return S_ERROR("Can't dump cfg file '%s'" % fileName) + gLogger.error("Can't dump cfg file", f"'{fileName}'") + return S_ERROR(f"Can't dump cfg file '{fileName}'") return S_OK() def getRemoteCFG(self): @@ -335,31 +324,35 @@ def dumpRemoteCFGToFile(self, fileName): fd.write(str(self.remoteCFG)) def __backupCurrentConfiguration(self, backupName): - configurationFilename = "%s.cfg" % self.getName() + configurationFilename = f"{self.getName()}.cfg" configurationFile = os.path.join(DIRAC.rootPath, "etc", configurationFilename) today = datetime.datetime.utcnow().date() backupPath = os.path.join(self.getBackupDir(), str(today.year), "%02d" % today.month) mkDir(backupPath) - backupFile = os.path.join(backupPath, configurationFilename.replace(".cfg", ".%s.zip" % backupName)) + backupFile = os.path.join(backupPath, configurationFilename.replace(".cfg", f".{backupName}.zip")) if os.path.isfile(configurationFile): - gLogger.info("Making a backup of configuration in %s" % backupFile) + gLogger.info(f"Making a backup of configuration in {backupFile}") try: with zipfile.ZipFile(backupFile, "w", zipfile.ZIP_DEFLATED) as zf: zf.write(configurationFile, f"{os.path.split(configurationFile)[1]}.backup.{backupName}") except Exception: gLogger.exception() - gLogger.error("Cannot backup configuration data file", "file %s" % backupFile) + gLogger.error("Cannot backup configuration data file", f"file {backupFile}") else: gLogger.warn("CS data file does not exist", configurationFile) def writeRemoteConfigurationToDisk(self, backupName=False): - configurationFile = os.path.join(DIRAC.rootPath, "etc", "%s.cfg" % self.getName()) + configurationFile = os.path.join(DIRAC.rootPath, "etc", f"{self.getName()}.cfg") + configurationFileTmp = f"{configurationFile}.{secrets.token_hex(8)}" try: - with open(configurationFile, "w") as fd: + with open(configurationFileTmp, "w") as fd: fd.write(str(self.remoteCFG)) + os.rename(configurationFileTmp, configurationFile) except Exception as e: gLogger.fatal("Cannot write new configuration to disk!", f"file {configurationFile} exception {repr(e)}") - return S_ERROR("Can't write cs file {}!: {}".format(configurationFile, repr(e).replace(",)", ")"))) + if os.path.isfile(configurationFileTmp): + os.remove(configurationFileTmp) + return S_ERROR(f"Can't write cs file {configurationFile}!: {repr(e).replace(',)', ')')}") if backupName: self.__backupCurrentConfiguration(backupName) return S_OK() diff --git a/src/DIRAC/ConfigurationSystem/private/Modificator.py b/src/DIRAC/ConfigurationSystem/private/Modificator.py index 7fdb1e90422..04f8afc70e4 100755 --- a/src/DIRAC/ConfigurationSystem/private/Modificator.py +++ b/src/DIRAC/ConfigurationSystem/private/Modificator.py @@ -113,7 +113,7 @@ def __setCommiter(self, entryPath, cfg=False): def setOptionValue(self, optionPath, value): levelList = [level.strip() for level in optionPath.split("/") if level.strip() != ""] - parentPath = "/%s" % "/".join(levelList[:-1]) + parentPath = f"/{'/'.join(levelList[:-1])}" optionName = List.fromChar(optionPath, "/")[-1] self.createSection(parentPath) cfg = self.__getParentCFG(optionPath) @@ -128,7 +128,7 @@ def createSection(self, sectionPath): cfg = self.cfgData createdSection = False for section in levelList: - currentPath += "/%s" % section + currentPath += f"/{section}" if section not in cfg.listSections(): cfg.createNewSection(section) self.__setCommiter(currentPath) @@ -172,7 +172,7 @@ def renameKey(self, path, newName): oldName = pathList[-1] if parentCfg["value"].renameKey(oldName, newName): pathList[-1] = newName - self.__setCommiter("/%s" % "/".join(pathList)) + self.__setCommiter(f"/{'/'.join(pathList)}") return True else: return False @@ -184,7 +184,7 @@ def copyKey(self, originalKeyPath, newKey): pathList = List.fromChar(originalKeyPath, "/") originalKey = pathList[-1] if parentCfg["value"].copyKey(originalKey, newKey): - self.__setCommiter("/{}/{}".format("/".join(pathList[:-1]), newKey)) + self.__setCommiter(f"/{'/'.join(pathList[:-1])}/{newKey}") return True return False @@ -211,7 +211,7 @@ def loadFromFile(self, filename): self.mergeFromFile(filename) def dumpToFile(self, filename): - with open(filename, "wt") as fd: + with open(filename, "w") as fd: fd.write(str(self.cfgData)) def mergeFromFile(self, filename): diff --git a/src/DIRAC/ConfigurationSystem/private/Refresher.py b/src/DIRAC/ConfigurationSystem/private/Refresher.py index 9da1603b50f..ed1fc3d7a58 100755 --- a/src/DIRAC/ConfigurationSystem/private/Refresher.py +++ b/src/DIRAC/ConfigurationSystem/private/Refresher.py @@ -58,6 +58,8 @@ def refreshConfigurationIfNeeded(self): except _thread.error: pass # Launch the refresh + if os.environ.get("DIRAC_DISABLE_GCONFIG_REFRESH", "false").lower() in ("yes", "true"): + raise NotImplementedError("DIRAC_DISABLE_GCONFIG_REFRESH is set") thd = threading.Thread(target=self._refreshInThread) thd.daemon = True thd.start() @@ -70,7 +72,7 @@ def autoRefreshAndPublish(self, sURL): """ gLogger.debug("Setting configuration refresh as automatic") if not gConfigurationData.getAutoPublish(): - gLogger.debug("Slave server won't auto publish itself") + gLogger.debug("Worker server won't auto publish itself") if not gConfigurationData.getName(): import DIRAC diff --git a/src/DIRAC/ConfigurationSystem/private/RefresherBase.py b/src/DIRAC/ConfigurationSystem/private/RefresherBase.py index c70cf4128d7..a61f98a7276 100644 --- a/src/DIRAC/ConfigurationSystem/private/RefresherBase.py +++ b/src/DIRAC/ConfigurationSystem/private/RefresherBase.py @@ -1,29 +1,27 @@ import time -import random - from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData from DIRAC.ConfigurationSystem.Client.PathFinder import getGatewayURLs -from DIRAC.FrameworkSystem.Client.Logger import gLogger from DIRAC.Core.Utilities import List from DIRAC.Core.Utilities.EventDispatcher import gEventDispatcher -from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR +from DIRAC.Core.Utilities.ReturnValues import S_ERROR, S_OK +from DIRAC.FrameworkSystem.Client.Logger import gLogger def _updateFromRemoteLocation(serviceClient): """ Refresh the configuration """ - gLogger.debug("", "Trying to refresh from %s" % serviceClient.serverURL) + gLogger.debug("", f"Trying to refresh from {serviceClient.serverURL}") localVersion = gConfigurationData.getVersion() retVal = serviceClient.getCompressedDataIfNewer(localVersion) if retVal["OK"]: dataDict = retVal["Value"] newestVersion = dataDict["newestVersion"] if localVersion < newestVersion: - gLogger.debug("New version available", "Updating to version %s..." % newestVersion) + gLogger.debug("New version available", f"Updating to version {newestVersion}...") gConfigurationData.loadRemoteCFGFromCompressedMem(dataDict["data"]) - gLogger.debug("Updated to version %s" % gConfigurationData.getVersion()) + gLogger.debug(f"Updated to version {gConfigurationData.getVersion()}") gEventDispatcher.triggerEvent("CSNewVersion", newestVersion, threaded=True) return S_OK() return retVal @@ -90,29 +88,31 @@ def _refreshAndPublish(self): Refresh configuration and publish local updates """ self._lastUpdateTime = time.time() - gLogger.info("Refreshing from master server") - sMasterServer = gConfigurationData.getMasterServer() - if sMasterServer: + gLogger.info("Refreshing from controller server") + sControllerServer = gConfigurationData.getMasterServer() + if sControllerServer: from DIRAC.ConfigurationSystem.Client.ConfigurationClient import ConfigurationClient oClient = ConfigurationClient( - url=sMasterServer, + url=sControllerServer, timeout=self._timeout, useCertificates=gConfigurationData.useServerCertificate(), skipCACheck=gConfigurationData.skipCACheck(), ) dRetVal = _updateFromRemoteLocation(oClient) if not dRetVal["OK"]: - gLogger.error("Can't update from master server", dRetVal["Message"]) + gLogger.error("Can't update from controller server", dRetVal["Message"]) return False if gConfigurationData.getAutoPublish(): - gLogger.info("Publishing to master server...") + gLogger.info("Publishing to controller server...") dRetVal = oClient.publishSlaveServer(self._url) if not dRetVal["OK"]: - gLogger.error("Can't publish to master server", dRetVal["Message"]) + gLogger.error("Can't publish to controller server", dRetVal["Message"]) return True else: - gLogger.warn("No master server is specified in the configuration, trying to get data from other slaves") + gLogger.warn( + "No controller server is specified in the configuration, trying to get data from other Workers" + ) return self._refresh()["OK"] def _refresh(self, fromMaster=False): @@ -127,19 +127,19 @@ def _refresh(self, fromMaster=False): initialServerList = gatewayList gLogger.debug("Using configuration gateway", str(initialServerList[0])) elif fromMaster: - masterServer = gConfigurationData.getMasterServer() - initialServerList = [masterServer] - gLogger.debug("Refreshing from master %s" % masterServer) + controllerServer = gConfigurationData.getMasterServer() + initialServerList = [controllerServer] + gLogger.debug(f"Refreshing from controller {controllerServer}") else: initialServerList = gConfigurationData.getServers() - gLogger.debug("Refreshing from list %s" % str(initialServerList)) + gLogger.debug(f"Refreshing from list {str(initialServerList)}") # If no servers in the initial list, we are supposed to use the local configuration only if not initialServerList: return S_OK() randomServerList = List.randomize(initialServerList) - gLogger.debug("Randomized server list is %s" % ", ".join(randomServerList)) + gLogger.debug(f"Randomized server list is {', '.join(randomServerList)}") for sServer in randomServerList: from DIRAC.ConfigurationSystem.Client.ConfigurationClient import ConfigurationClient @@ -155,9 +155,7 @@ def _refresh(self, fromMaster=False): return dRetVal else: updatingErrorsList.append(dRetVal["Message"]) - gLogger.warn( - "Can't update from server", "Error while updating from {}: {}".format(sServer, dRetVal["Message"]) - ) + gLogger.warn("Can't update from server", f"Error while updating from {sServer}: {dRetVal['Message']}") if dRetVal["Message"].find("Insane environment") > -1: break return S_ERROR("Reason(s):\n\t%s" % "\n\t".join(List.uniqueElements(updatingErrorsList))) diff --git a/src/DIRAC/ConfigurationSystem/private/ServiceInterface.py b/src/DIRAC/ConfigurationSystem/private/ServiceInterface.py index c9a2d09d8ad..ffa48b48c52 100755 --- a/src/DIRAC/ConfigurationSystem/private/ServiceInterface.py +++ b/src/DIRAC/ConfigurationSystem/private/ServiceInterface.py @@ -11,7 +11,7 @@ class ServiceInterface(ServiceInterfaceBase, threading.Thread): """ - Service interface, manage Slave/Master server for CS + Service interface, manage Worker/Controller server for CS Thread components """ @@ -19,11 +19,11 @@ def __init__(self, sURL): threading.Thread.__init__(self) ServiceInterfaceBase.__init__(self, sURL) - def _launchCheckSlaves(self): + def _launchCheckWorkers(self): """ - Start loop which check if slaves are alive + Start loop which check if Workers are alive """ - gLogger.info("Starting purge slaves thread") + gLogger.info("Starting purge Workers thread") self.daemon = True self.start() @@ -31,24 +31,24 @@ def run(self): while True: iWaitTime = gConfigurationData.getSlavesGraceTime() time.sleep(iWaitTime) - self._checkSlavesStatus() + self._checkWorkersStatus() - def _updateServiceConfiguration(self, urlSet, fromMaster=False): + def _updateServiceConfiguration(self, urlSet, fromController=False): """ - Update configuration of a set of slave services in parallel + Update configuration of a set of Worker services in parallel :param set urlSet: a set of service URLs - :param fromMaster: flag to force updating from the master CS + :param fromController: flag to force updating from the master CS :return: Nothing """ if not urlSet: return with ThreadPoolExecutor(max_workers=len(urlSet)) as executor: - futureUpdate = {executor.submit(self._forceServiceUpdate, url, fromMaster): url for url in urlSet} + futureUpdate = {executor.submit(self._forceServiceUpdate, url, fromController): url for url in urlSet} for future in as_completed(futureUpdate): url = futureUpdate[future] result = future.result() if result["OK"]: - gLogger.info("Successfully updated slave configuration", url) + gLogger.info("Successfully updated Worker configuration", url) else: - gLogger.error("Failed to update slave configuration", url) + gLogger.error("Failed to update Worker configuration", url) diff --git a/src/DIRAC/ConfigurationSystem/private/ServiceInterfaceBase.py b/src/DIRAC/ConfigurationSystem/private/ServiceInterfaceBase.py index a9cd56b8e3b..1e068340d97 100644 --- a/src/DIRAC/ConfigurationSystem/private/ServiceInterfaceBase.py +++ b/src/DIRAC/ConfigurationSystem/private/ServiceInterfaceBase.py @@ -1,43 +1,42 @@ -"""Service interface is the service which provide config for client and synchronize Master/Slave servers""" +"""Service interface is the service which provide config for client and synchronize Controller/Worker servers""" import os -import time import re +import time import zipfile import zlib import DIRAC from DIRAC.ConfigurationSystem.Client.ConfigurationClient import ConfigurationClient -from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData, ConfigurationData +from DIRAC.ConfigurationSystem.Client.ConfigurationData import ConfigurationData, gConfigurationData from DIRAC.ConfigurationSystem.private.Refresher import gRefresher from DIRAC.Core.Base.Client import Client from DIRAC.Core.Utilities.File import mkDir -from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR +from DIRAC.Core.Utilities.ReturnValues import S_ERROR, S_OK from DIRAC.FrameworkSystem.Client.Logger import gLogger class ServiceInterfaceBase: - """Service interface is the service which provide config for client and synchronize Master/Slave servers""" + """Service interface is the service which provide config for client and synchronize Controller/Worker servers""" def __init__(self, sURL): self.sURL = sURL - gLogger.info("Initializing Configuration Service", "URL is %s" % sURL) - self.__modificationsIgnoreMask = ["/DIRAC/Configuration/Servers", "/DIRAC/Configuration/Version"] + gLogger.info("Initializing Configuration Service", f"URL is {sURL}") gConfigurationData.setAsService() if not gConfigurationData.isMaster(): - gLogger.info("Starting configuration service as slave") + gLogger.info("Starting configuration service as Worker") gRefresher.autoRefreshAndPublish(self.sURL) else: - gLogger.info("Starting configuration service as master") + gLogger.info("Starting configuration service as controller") gRefresher.disable() self.__loadConfigurationData() - self.dAliveSlaveServers = {} - self._launchCheckSlaves() + self.dAliveWorkerServers = {} + self._launchCheckWorkers() def isMaster(self): return gConfigurationData.isMaster() - def _launchCheckSlaves(self): + def _launchCheckWorkers(self): raise NotImplementedError("Should be implemented by the children class") def __loadConfigurationData(self): @@ -75,91 +74,91 @@ def __generateNewVersion(self): gConfigurationData.generateNewVersion() gConfigurationData.writeRemoteConfigurationToDisk() - def publishSlaveServer(self, sSlaveURL): + def publishSlaveServer(self, sWorkerURL): """ - Called by the slave server via service, it register a new slave server + Called by the Worker server via service, it register a new Worker server - :param sSlaveURL: url of slave server + :param sWorkerURL: url of Worker server """ if not gConfigurationData.isMaster(): return S_ERROR("Configuration modification is not allowed in this server") - gLogger.info("Pinging slave %s" % sSlaveURL) - rpcClient = ConfigurationClient(url=sSlaveURL, timeout=10, useCertificates=True) + gLogger.info(f"Pinging Worker {sWorkerURL}") + rpcClient = ConfigurationClient(url=sWorkerURL, timeout=10, useCertificates=True) retVal = rpcClient.ping() if not retVal["OK"]: - gLogger.info("Slave %s didn't reply" % sSlaveURL) + gLogger.info(f"Worker {sWorkerURL} didn't reply") return if retVal["Value"]["name"] != "Configuration/Server": - gLogger.info("Slave %s is not a CS serveR" % sSlaveURL) + gLogger.info(f"Worker {sWorkerURL} is not a CS serveR") return - bNewSlave = False - if sSlaveURL not in self.dAliveSlaveServers: - bNewSlave = True - gLogger.info("New slave registered", sSlaveURL) - self.dAliveSlaveServers[sSlaveURL] = time.time() - if bNewSlave: - gConfigurationData.setServers(", ".join(self.dAliveSlaveServers)) + bNewWorker = False + if sWorkerURL not in self.dAliveWorkerServers: + bNewWorker = True + gLogger.info("New Worker registered", sWorkerURL) + self.dAliveWorkerServers[sWorkerURL] = time.time() + if bNewWorker: + gConfigurationData.setServers(", ".join(self.dAliveWorkerServers)) self.__generateNewVersion() - def _checkSlavesStatus(self, forceWriteConfiguration=False): + def _checkWorkersStatus(self, forceWriteConfiguration=False): """ - Check if Slaves server are still availlable + Check if Workers server are still availlable - :param forceWriteConfiguration: (default False) Force rewriting configuration after checking slaves + :param forceWriteConfiguration: (default False) Force rewriting configuration after checking workers """ - gLogger.info("Checking status of slave servers") + gLogger.info("Checking status of Worker servers") iGraceTime = gConfigurationData.getSlavesGraceTime() - bModifiedSlaveServers = False - for sSlaveURL in list(self.dAliveSlaveServers): - if time.time() - self.dAliveSlaveServers[sSlaveURL] > iGraceTime: - gLogger.warn("Found dead slave", sSlaveURL) - del self.dAliveSlaveServers[sSlaveURL] - bModifiedSlaveServers = True - if bModifiedSlaveServers or forceWriteConfiguration: - gConfigurationData.setServers(", ".join(self.dAliveSlaveServers)) + bModifiedWorkerServers = False + for sWorkerURL in list(self.dAliveWorkerServers): + if time.time() - self.dAliveWorkerServers[sWorkerURL] > iGraceTime: + gLogger.warn("Found dead Worker", sWorkerURL) + del self.dAliveWorkerServers[sWorkerURL] + bModifiedWorkerServers = True + if bModifiedWorkerServers or forceWriteConfiguration: + gConfigurationData.setServers(", ".join(self.dAliveWorkerServers)) self.__generateNewVersion() @staticmethod - def _forceServiceUpdate(url, fromMaster): + def _forceServiceUpdate(url, fromController): """ Force updating configuration on a given service This should be called by _updateServiceConfiguration :param str url: service URL - :param bool fromMaster: flag to force updating from the master CS + :param bool fromController: flag to force updating from the controller CS :return: S_OK/S_ERROR """ gLogger.info("Updating service configuration on", url) - result = Client(url=url).refreshConfiguration(fromMaster) + result = Client(url=url).refreshConfiguration(fromController) result["URL"] = url return result - def _updateServiceConfiguration(self, urlSet, fromMaster=False): + def _updateServiceConfiguration(self, urlSet, fromController=False): """ Update configuration in a set of service in parallel :param set urlSet: a set of service URLs - :param fromMaster: flag to force updating from the master CS + :param fromController: flag to force updating from the controller CS :return: Nothing """ raise NotImplementedError("Should be implemented by the children class") - def forceSlavesUpdate(self): + def forceWorkersUpdate(self): """ - Force updating configuration on all the slave configuration servers + Force updating configuration on all the Worker configuration servers :return: Nothing """ - gLogger.info("Updating configuration on slave servers") + gLogger.info("Updating configuration on Worker servers") iGraceTime = gConfigurationData.getSlavesGraceTime() urlSet = set() - for slaveURL in self.dAliveSlaveServers: - if time.time() - self.dAliveSlaveServers[slaveURL] <= iGraceTime: - urlSet.add(slaveURL) - self._updateServiceConfiguration(urlSet, fromMaster=True) + for workerURL in self.dAliveWorkerServers: + if time.time() - self.dAliveWorkerServers[workerURL] <= iGraceTime: + urlSet.add(workerURL) + self._updateServiceConfiguration(urlSet, fromController=True) def forceGlobalUpdate(self): """ @@ -186,7 +185,7 @@ def forceGlobalUpdate(self): def updateConfiguration(self, sBuffer, committer="", updateVersionOption=False): """ - Update the master configuration with the newly received changes + Update the controller configuration with the newly received changes :param str sBuffer: newly received configuration data :param str committer: the user name of the committer @@ -216,7 +215,7 @@ def updateConfiguration(self, sBuffer, committer="", updateVersionOption=False): result = self.__mergeIndependentUpdates(oRemoteConfData) if not result["OK"]: gLogger.warn("Could not AutoMerge!", result["Message"]) - return S_ERROR("AutoMerge failed: %s" % result["Message"]) + return S_ERROR(f"AutoMerge failed: {result['Message']}") requestedRemoteCFG = result["Value"] gLogger.info("AutoMerge successful!") oRemoteConfData.setRemoteCFG(requestedRemoteCFG) @@ -233,14 +232,14 @@ def updateConfiguration(self, sBuffer, committer="", updateVersionOption=False): gConfigurationData.unlock() gLogger.info("Generating new version") gConfigurationData.generateNewVersion() - # self.__checkSlavesStatus( forceWriteConfiguration = True ) + # self.__checkWorkersStatus( forceWriteConfiguration = True ) gLogger.info("Writing new version to disk") retVal = gConfigurationData.writeRemoteConfigurationToDisk(f"{committer}@{gConfigurationData.getVersion()}") gLogger.info("New version", gConfigurationData.getVersion()) - # Attempt to update the configuration on currently registered slave services + # Attempt to update the configuration on currently registered Worker services if gConfigurationData.getAutoSlaveSync(): - self.forceSlavesUpdate() + self.forceWorkersUpdate() return retVal @@ -259,11 +258,11 @@ def getVersionContents(self, date): backupDir = gConfigurationData.getBackupDir() files = self.__getCfgBackups(backupDir, date) for fileName in files: - with zipfile.ZipFile(f"{backupDir}/{fileName}", "rb") as zFile: + with zipfile.ZipFile(f"{backupDir}/{fileName}", "r") as zFile: cfgName = zFile.namelist()[0] retVal = S_OK(zlib.compress(zFile.read(cfgName), 9)) return retVal - return S_ERROR("Version %s does not exist" % date) + return S_ERROR(f"Version {date} does not exist") def __getCfgBackups(self, basePath, date="", subPath=""): rs = re.compile(rf"^{gConfigurationData.getName()}\..*{date}.*\.zip$") @@ -290,8 +289,8 @@ def __getPreviousCFG(self, oRemoteConfData): try: prevRemoteConfData.loadConfigurationData(backFile) except Exception as e: - return S_ERROR("Could not load original committer's version: %s" % str(e)) - gLogger.info("Loaded client original version %s" % prevRemoteConfData.getVersion()) + return S_ERROR(f"Could not load original committer's version: {str(e)}") + gLogger.info(f"Loaded client original version {prevRemoteConfData.getVersion()}") return S_OK(prevRemoteConfData.getRemoteCFG()) def _checkConflictsInModifications(self, realModList, reqModList, parentSection=""): @@ -343,7 +342,7 @@ def __mergeIndependentUpdates(self, oRemoteConfData): prevCliToCurSrvModList = prevCliCFG.getModifications(curSrvCFG) result = self._checkConflictsInModifications(prevCliToCurSrvModList, prevCliToCurCliModList) if not result["OK"]: - return S_ERROR("Cannot AutoMerge: %s" % result["Message"]) + return S_ERROR(f"Cannot AutoMerge: {result['Message']}") # Merge! result = curSrvCFG.applyModifications(prevCliToCurCliModList) if not result["OK"]: diff --git a/src/DIRAC/ConfigurationSystem/private/ServiceInterfaceTornado.py b/src/DIRAC/ConfigurationSystem/private/ServiceInterfaceTornado.py index 8152be6449e..a665d2bc03e 100644 --- a/src/DIRAC/ConfigurationSystem/private/ServiceInterfaceTornado.py +++ b/src/DIRAC/ConfigurationSystem/private/ServiceInterfaceTornado.py @@ -17,29 +17,29 @@ class ServiceInterfaceTornado(ServiceInterfaceBase): def __init__(self, sURL): ServiceInterfaceBase.__init__(self, sURL) - def _launchCheckSlaves(self): + def _launchCheckWorkers(self): """ - Start loop to check if slaves are alive + Start loop to check if workers are alive """ IOLoop.current().spawn_callback(self.run) - gLogger.info("Starting purge slaves thread") + gLogger.info("Starting purge workers thread") def run(self): """ - Check if slaves are alive + Check if workers are alive """ while True: yield gen.sleep(gConfigurationData.getSlavesGraceTime()) - self._checkSlavesStatus() + self._checkWorkersStatus() - def _updateServiceConfiguration(self, urlSet, fromMaster=False): + def _updateServiceConfiguration(self, urlSet, fromController=False): """ Update configuration in a set of service in parallel :param set urlSet: a set of service URLs - :param fromMaster: flag to force updating from the master CS + :param fromController: flag to force updating from the controller CS :return: Nothing """ for url in urlSet: - IOLoop.current().spawn_callback(self._forceServiceUpdate, [url, fromMaster]) + IOLoop.current().spawn_callback(self._forceServiceUpdate, [url, fromController]) diff --git a/src/DIRAC/ConfigurationSystem/private/TornadoRefresher.py b/src/DIRAC/ConfigurationSystem/private/TornadoRefresher.py index d4d0c620fc2..a90215d76e5 100644 --- a/src/DIRAC/ConfigurationSystem/private/TornadoRefresher.py +++ b/src/DIRAC/ConfigurationSystem/private/TornadoRefresher.py @@ -42,7 +42,7 @@ def autoRefreshAndPublish(self, sURL): """ gLogger.debug("Setting configuration refresh as automatic") if not gConfigurationData.getAutoPublish(): - gLogger.debug("Slave server won't auto publish itself") + gLogger.debug("Worker server won't auto publish itself") if not gConfigurationData.getName(): import DIRAC @@ -66,12 +66,11 @@ def __refreshLoop(self): for official documentation about this type of method. """ while self._automaticUpdate: - # This is the sleep from Tornado, like a sleep it wait some time # But this version is non-blocking, so IOLoop can continue execution yield gen.sleep(gConfigurationData.getPropagationTime()) # Publish step is blocking so we have to run it in executor - # If we are not doing it, when master try to ping we block the IOLoop + # If we are not doing it, when controller try to ping we block the IOLoop yield _IOLoop.current().run_in_executor(None, self.__AutoRefresh) diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_resources.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_resources.py index 8cfa74d79c7..b3e863433fa 100755 --- a/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_resources.py +++ b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_resources.py @@ -20,7 +20,6 @@ def processScriptSwitches(): - global vo, dry, doCEs, hostURL, onecore Script.registerSwitch("V:", "vo=", "Virtual Organization") @@ -55,7 +54,6 @@ def processScriptSwitches(): def checkUnusedCEs(): - global vo, dry, ceBdiiDict, hostURL gLogger.notice("looking for new computing resources in the BDII database...") @@ -77,7 +75,7 @@ def checkUnusedCEs(): unknownCEs = result["UnknownCEs"] if unknownCEs: - gLogger.notice("There is no (longer) information about the following CEs for the %s VO:" % vo) + gLogger.notice(f"There is no (longer) information about the following CEs for the {vo} VO:") gLogger.notice("\n".join(sorted(unknownCEs))) siteDict = result["Value"] @@ -122,28 +120,28 @@ def checkUnusedCEs(): country = "xx" result = getDIRACSiteName(site) if not result["OK"]: - gLogger.notice("\nThe site %s is not yet in the CS, give it a name" % site) - diracSite = input("[help|skip|..%s]: " % country) + gLogger.notice(f"\nThe site {site} is not yet in the CS, give it a name") + diracSite = input(f"[help|skip|..{country}]: ") if diracSite.lower() == "skip": continue if diracSite.lower() == "help": - gLogger.notice("%s site details:" % site) + gLogger.notice(f"{site} site details:") for k, v in ceBdiiDict[site].items(): if k != "CEs": gLogger.notice(f"{k}\t{v}") - gLogger.notice("\nEnter DIRAC site name in the form ..%s\n" % country) - diracSite = input("[..%s]: " % country) + gLogger.notice(f"\nEnter DIRAC site name in the form ..{country}\n") + diracSite = input(f"[..{country}]: ") try: _, _, _ = diracSite.split(".") except ValueError: - gLogger.error("ERROR: DIRAC site name does not follow convention: %s" % diracSite) + gLogger.error(f"ERROR: DIRAC site name does not follow convention: {diracSite}") continue diracSites = [diracSite] else: diracSites = result["Value"] if len(diracSites) > 1: - gLogger.notice("Attention! GOC site %s corresponds to more than one DIRAC sites:" % site) + gLogger.notice(f"Attention! GOC site {site} corresponds to more than one DIRAC sites:") gLogger.notice(str(diracSites)) gLogger.notice("Please, pay attention which DIRAC site the new CEs will join\n") @@ -162,8 +160,8 @@ def checkUnusedCEs(): for diracSite in diracSites: if diracSite in newCEs: - cmd = "dirac-admin-add-site {} {} {}".format(diracSite, site, " ".join(newCEs[diracSite])) - gLogger.notice("\nNew site/CEs will be added with command:\n%s" % cmd) + cmd = f"dirac-admin-add-site {diracSite} {site} {' '.join(newCEs[diracSite])}" + gLogger.notice(f"\nNew site/CEs will be added with command:\n{cmd}") yn = input("Add it ? [default yes] [yes|no]: ") if not (yn == "" or yn.lower().startswith("y")): continue @@ -207,7 +205,6 @@ def checkUnusedCEs(): def updateCS(changeSet): - global vo, dry, ceBdiiDict changeList = sorted(changeSet) @@ -237,11 +234,10 @@ def updateCS(changeSet): if not result["OK"]: gLogger.error("Error while commit to CS", result["Message"]) else: - gLogger.notice("Successfully committed %d changes to CS" % len(changeSet)) + gLogger.notice(f"Successfully committed {len(changeSet)} changes to CS") def updateSites(): - global vo, dry, ceBdiiDict, onecore result = getSiteUpdates(vo, bdiiInfo=ceBdiiDict, onecore=onecore) diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_site.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_site.py index 53f01626811..50ce5c18462 100755 --- a/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_site.py +++ b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_add_site.py @@ -39,7 +39,7 @@ def main(): newSite = True if result["OK"] and result["Value"]: if len(result["Value"]) > 1: - gLogger.notice("%s GOC site name is associated with several DIRAC sites:" % gridSiteName) + gLogger.notice(f"{gridSiteName} GOC site name is associated with several DIRAC sites:") for i, dSite in enumerate(result["Value"]): gLogger.notice("%d: %s" % (i, dSite)) inp = input("Enter your choice number: ") @@ -62,12 +62,12 @@ def main(): gLogger.error(f"ERROR: Site with GOC name {gridSiteName} is already defined as {diracCSSite}") DIRACExit(-1) else: - gLogger.error("ERROR getting DIRAC site name of %s" % gridSiteName, result.get("Message")) + gLogger.error(f"ERROR getting DIRAC site name of {gridSiteName}", result.get("Message")) csAPI = CSAPI() if newSite: - gLogger.notice("Site to CS: %s" % diracSiteName) + gLogger.notice(f"Site to CS: {diracSiteName}") res = csAPI.addSite(diracSiteName, {"Name": gridSiteName}) if not res["OK"]: gLogger.error("Failed adding site to CS", res["Message"]) @@ -78,10 +78,10 @@ def main(): DIRACExit(3) for ce in ces: - gLogger.notice("Adding CE %s" % ce) + gLogger.notice(f"Adding CE {ce}") res = csAPI.addCEtoSite(diracSiteName, ce) if not res["OK"]: - gLogger.error("Failed adding CE %s to CS" % ce, res["Message"]) + gLogger.error(f"Failed adding CE {ce} to CS", res["Message"]) DIRACExit(2) res = csAPI.commit() if not res["OK"]: diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_check_config_options.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_check_config_options.py index a0b1f214ccd..2c95ffdcbf7 100755 --- a/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_check_config_options.py +++ b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_check_config_options.py @@ -92,9 +92,9 @@ def _check(self): diff = currentCfg.getModifications(cfg, ignoreOrder=True, ignoreComments=True) LOG.debug("*" * 80) - LOG.debug("Default Configuration: %s" % str(cfg)) + LOG.debug(f"Default Configuration: {str(cfg)}") LOG.debug("*" * 80) - LOG.debug("Current Configuration: %s " % str(currentCfg)) + LOG.debug(f"Current Configuration: {str(currentCfg)} ") for entry in diff: self._printDiff(entry) @@ -116,13 +116,13 @@ def _parseConfigTemplate(self, templatePath, cfg=None): templatePath = os.path.join(templatePath, "ConfigTemplate.cfg") if not os.path.exists(templatePath): - return S_ERROR("File not found: %s" % templatePath) + return S_ERROR(f"File not found: {templatePath}") loadCfg = CFG() loadCfg.loadFromFile(templatePath) newCfg = CFG() - newCfg.createNewSection("/%s" % system, contents=loadCfg) + newCfg.createNewSection(f"/{system}", contents=loadCfg) cfg = cfg.mergeWith(newCfg) @@ -145,26 +145,19 @@ def _getCurrentConfig(self): gConfig.forceRefresh() fullCfg = CFG() - setup = gConfig.getValue("/DIRAC/Setup", "") - setupList = gConfig.getSections("/DIRAC/Setups", []) - if not setupList["OK"]: - return S_ERROR("Could not get /DIRAC/Setups sections") - setupList = setupList["Value"] - if setup not in setupList: - return S_ERROR("Setup {} is not in allowed list: {}".format(setup, ", ".join(setupList))) - serviceSetups = gConfig.getOptionsDict("/DIRAC/Setups/%s" % setup) - if not serviceSetups["OK"]: - return S_ERROR("Could not get /DIRAC/Setups/%s options" % setup) - serviceSetups = serviceSetups["Value"] # dict - for system, setup in serviceSetups.items(): + res = gConfig.getSections("/Systems") + if not res["OK"]: + return res + systems = res["Value"] + for system in systems: if self.systems and system not in self.systems: continue - systemCfg = gConfigurationData.remoteCFG.getAsCFG(f"/Systems/{system}/{setup}") + systemCfg = gConfigurationData.remoteCFG.getAsCFG(f"/Systems/{system}") for section in systemCfg.listSections(): if section not in ("Agents", "Services", "Executors"): systemCfg.deleteKey(section) - fullCfg.createNewSection("/%s" % system, contents=systemCfg) + fullCfg.createNewSection(f"/{system}", contents=systemCfg) return S_OK(fullCfg) @@ -174,6 +167,8 @@ def _printDiff(self, entry, level=""): diffType, entryName, _value, changes, _comment = entry elif len(entry) == 4: diffType, entryName, _value, changes = entry + else: + raise ValueError(f"Invalid entry {entry}") fullPath = os.path.join(level, entryName) @@ -185,10 +180,10 @@ def _printDiff(self, entry, level=""): LOG.notice(f"Changed option {fullPath!r} from {changes!r}") elif diffType == "delOpt": if self.showAdded: - LOG.notice("Option %r does not exist in template" % fullPath) + LOG.notice(f"Option {fullPath!r} does not exist in template") elif diffType == "delSec": if self.showAdded: - LOG.notice("Section %r does not exist in template" % fullPath) + LOG.notice(f"Section {fullPath!r} does not exist in template") elif diffType == "addSec": if self.showMissingSections: LOG.notice(f"Section {fullPath!r} not found in current configuration: {pformat(changes)}") diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_sort_cs_sites.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_sort_cs_sites.py index e31e788706d..4b04d53596b 100755 --- a/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_sort_cs_sites.py +++ b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_sort_cs_sites.py @@ -93,8 +93,8 @@ def main(): hasRun = False isDirty = False for i in resultList: - if not cfg.isSection("Resources/Sites/%s" % i): - gLogger.error("Subsection /Resources/Sites/%s does not exists" % i) + if not cfg.isSection(f"Resources/Sites/{i}"): + gLogger.error(f"Subsection /Resources/Sites/{i} does not exists") continue hasRun = True if SORTBYNAME: diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_voms_sync.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_voms_sync.py index 2f703c23b0c..56929a15a3d 100755 --- a/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_voms_sync.py +++ b/src/DIRAC/ConfigurationSystem/scripts/dirac_admin_voms_sync.py @@ -11,10 +11,12 @@ from DIRAC.ConfigurationSystem.Client.VOMS2CSSynchronizer import VOMS2CSSynchronizer from DIRAC.Core.Utilities.Proxy import executeWithUserProxy from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOOption - +from DIRAC.FrameworkSystem.Client.TokenManagerClient import gTokenManager dryRun = False voName = None +compareWithIAM = False +useIAM = False def setDryRun(value): @@ -29,10 +31,25 @@ def setVO(value): return S_OK() +def setCompareWithIAM(value): + global compareWithIAM + compareWithIAM = True + return S_OK() + + +def setUseIAM(value): + global useIAM + useIAM = True + return S_OK() + + @Script() def main(): Script.registerSwitch("V:", "vo=", "VO name", setVO) Script.registerSwitch("D", "dryRun", "Dry run", setDryRun) + Script.registerSwitch("C", "compareWithIAM", "Compare user list with IAM", setCompareWithIAM) + Script.registerSwitch("I", "useIAM", "Use IAM as authoritative source", setUseIAM) + Script.parseCommandLine(ignoreErrors=True) @executeWithUserProxy @@ -41,8 +58,19 @@ def syncCSWithVOMS(vomsSync): voAdminUser = getVOOption(voName, "VOAdmin") voAdminGroup = getVOOption(voName, "VOAdminGroup", getVOOption(voName, "DefaultGroup")) - - vomsSync = VOMS2CSSynchronizer(voName) + accessToken = None + if compareWithIAM or useIAM: + res = gTokenManager.getToken( + userGroup=voAdminGroup, + requiredTimeLeft=3600, + scope=["scim:read"], + ) + if not res["OK"]: + return res + + accessToken = res["Value"]["access_token"] + + vomsSync = VOMS2CSSynchronizer(voName, compareWithIAM=compareWithIAM, useIAM=useIAM, accessToken=accessToken) result = syncCSWithVOMS( # pylint: disable=unexpected-keyword-arg vomsSync, proxyUserName=voAdminUser, proxyUserGroup=voAdminGroup ) @@ -67,6 +95,7 @@ def syncCSWithVOMS(vomsSync): if csapi and csapi.csModified: if dryRun: gLogger.notice("There are changes to Registry ready to commit, skipped because of dry run") + csapi.showDiff() else: yn = input("There are changes to Registry ready to commit, do you want to proceed ? [Y|n]:") if yn == "" or yn[0].lower() == "y": @@ -74,11 +103,11 @@ def syncCSWithVOMS(vomsSync): if not result["OK"]: gLogger.error("Could not commit configuration changes", result["Message"]) else: - gLogger.notice("Registry changes committed for VO %s" % voName) + gLogger.notice(f"Registry changes committed for VO {voName}") else: gLogger.notice("Registry changes are not committed") else: - gLogger.notice("No changes to Registry for VO %s" % voName) + gLogger.notice(f"No changes to Registry for VO {voName}") result = vomsSync.getVOUserReport() if not result["OK"]: diff --git a/src/DIRAC/ConfigurationSystem/scripts/dirac_configuration_dump_local_cache.py b/src/DIRAC/ConfigurationSystem/scripts/dirac_configuration_dump_local_cache.py index a70c5b8d0f1..5ee35df533e 100755 --- a/src/DIRAC/ConfigurationSystem/scripts/dirac_configuration_dump_local_cache.py +++ b/src/DIRAC/ConfigurationSystem/scripts/dirac_configuration_dump_local_cache.py @@ -18,14 +18,14 @@ def main(): fileName = "" def setFilename(args): - global fileName + nonlocal fileName fileName = args return DIRAC.S_OK() raw = False def setRaw(args): - global raw + nonlocal raw raw = True return DIRAC.S_OK() @@ -37,7 +37,7 @@ def setRaw(args): result = gConfig.dumpCFGAsLocalCache(fileName, raw) if not result["OK"]: - print("Error: %s" % result["Message"]) + print(f"Error: {result['Message']}") sys.exit(1) if not fileName: diff --git a/src/DIRAC/ConfigurationSystem/test/Test_agentOptions.py b/src/DIRAC/ConfigurationSystem/test/Test_agentOptions.py index 4cf44743f8c..172ce3fab91 100644 --- a/src/DIRAC/ConfigurationSystem/test/Test_agentOptions.py +++ b/src/DIRAC/ConfigurationSystem/test/Test_agentOptions.py @@ -15,7 +15,6 @@ ("DIRAC.ConfigurationSystem.Agent.RucioSynchronizerAgent", {}), ("DIRAC.ConfigurationSystem.Agent.VOMS2CSAgent", {"IgnoreOptions": ["VO"]}), ("DIRAC.DataManagementSystem.Agent.FTS3Agent", {}), - ("DIRAC.FrameworkSystem.Agent.CAUpdateAgent", {}), ("DIRAC.FrameworkSystem.Agent.ComponentSupervisionAgent", {}), ("DIRAC.FrameworkSystem.Agent.ProxyRenewalAgent", {}), ("DIRAC.RequestManagementSystem.Agent.CleanReqDBAgent", {}), diff --git a/src/DIRAC/Core/Base/API.py b/src/DIRAC/Core/Base/API.py index 19759628fed..ee89081a2ab 100644 --- a/src/DIRAC/Core/Base/API.py +++ b/src/DIRAC/Core/Base/API.py @@ -3,13 +3,11 @@ import pprint import sys -from DIRAC import gLogger, gConfig, S_OK, S_ERROR -from DIRAC.Core.Security.ProxyInfo import getProxyInfo, formatProxyInfoAsString -from DIRAC.Core.Utilities.Version import getCurrentVersion +from DIRAC import S_ERROR, S_OK, gLogger from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getDNForUsername from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSites - -COMPONENT_NAME = "API" +from DIRAC.Core.Security.ProxyInfo import formatProxyInfoAsString, getProxyInfo +from DIRAC.Core.Utilities.Version import getCurrentVersion def _printFormattedDictList(dictList, fields, uniqueField, orderBy): @@ -35,7 +33,7 @@ def _printFormattedDictList(dictList, fields, uniqueField, orderBy): orderDict[orderValue] = [] orderDict[orderValue].append(myDict[uniqueField]) dictFields[myDict[uniqueField]] = myDict - headString = "%s" % fields[0].ljust(fieldWidths[fields[0]] + 5) + headString = f"{fields[0].ljust(fieldWidths[fields[0]] + 5)}" for field in fields[1:]: headString = f"{headString} {field.ljust(fieldWidths[field] + 5)}" print(headString) @@ -43,7 +41,7 @@ def _printFormattedDictList(dictList, fields, uniqueField, orderBy): uniqueFields = orderDict[orderValue] for uniqueField in sorted(uniqueFields): myDict = dictFields[uniqueField] - outStr = "%s" % str(myDict[fields[0]]).ljust(fieldWidths[fields[0]] + 5) + outStr = f"{str(myDict[fields[0]]).ljust(fieldWidths[fields[0]] + 5)}" for field in fields[1:]: outStr = f"{outStr} {str(myDict[field]).ljust(fieldWidths[field] + 5)}" print(outStr) @@ -60,12 +58,10 @@ class API: def __init__(self): """c'tor""" self._printFormattedDictList = _printFormattedDictList - self.log = gLogger.getSubLogger(COMPONENT_NAME) - self.section = COMPONENT_NAME + self.log = gLogger.getSubLogger(self.__class__.__name__) self.pPrint = pprint.PrettyPrinter() # Global error dictionary self.errorDict = {} - self.setup = gConfig.getValue("/DIRAC/Setup", "Unknown") self.diracInfo = getCurrentVersion()["Value"] self._siteSet = set(getSites().get("Value", [])) @@ -136,7 +132,7 @@ def _checkSiteIsValid(self, site): return S_OK() elif site in self._siteSet: return S_OK() - return self._reportError("Specified site %s is not in list of defined sites" % str(site)) + return self._reportError(f"Specified site {str(site)} is not in list of defined sites") ############################################################################# @@ -177,15 +173,10 @@ def _reportError(self, message, name="", **kwargs): for key in kwargs: if kwargs[key]: arguments.append(f"{key} = {kwargs[key]} ( {type(kwargs[key])} )") - finalReport = """Problem with {}.{}() call: -Arguments: {} -Message: {} -""".format( - className, - methodName, - "/".join(arguments), - message, - ) + finalReport = f"""Problem with {className}.{methodName}() call: +Arguments: {'/'.join(arguments)} +Message: {message} +""" if methodName in self.errorDict: tmp = self.errorDict[methodName] tmp.append(finalReport) diff --git a/src/DIRAC/Core/Base/AgentModule.py b/src/DIRAC/Core/Base/AgentModule.py index e36aebc8a5c..18e0f08187a 100644 --- a/src/DIRAC/Core/Base/AgentModule.py +++ b/src/DIRAC/Core/Base/AgentModule.py @@ -1,24 +1,25 @@ """ Base class for all agent modules """ +import datetime +import importlib.metadata +import inspect import os +import signal import threading import time -import signal -import importlib.metadata -import inspect -import datetime + import psutil import DIRAC -from DIRAC import S_OK, S_ERROR, gConfig, gLogger, rootPath -from DIRAC.Core.Utilities.File import mkDir +from DIRAC import S_ERROR, S_OK, gConfig, gLogger, rootPath +from DIRAC.ConfigurationSystem.Client import PathFinder +from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.Core.Utilities import Network, TimeUtilities -from DIRAC.Core.Utilities.Shifter import setupShifterProxyInEnv +from DIRAC.Core.Utilities.File import mkDir from DIRAC.Core.Utilities.ReturnValues import isReturnStructure -from DIRAC.ConfigurationSystem.Client import PathFinder +from DIRAC.Core.Utilities.Shifter import setupShifterProxyInEnv from DIRAC.Core.Utilities.ThreadScheduler import gThreadScheduler -from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations class AgentModule: @@ -68,7 +69,6 @@ def __init__(self, agentName, loadName, baseAgentName=False, properties={}): They are used to populate __codeProperties The following Options are used from the Configuration: - - /DIRAC/Setup - Status - Enabled - PollingTime default = 120 @@ -113,13 +113,12 @@ def __init__(self, agentName, loadName, baseAgentName=False, properties={}): "loadSection": PathFinder.getAgentSection(loadName), "cyclesDone": 0, "totalElapsedTime": 0, - "setup": gConfig.getValue("/DIRAC/Setup", "Unknown"), "alive": True, } self.__moduleProperties["system"], self.__moduleProperties["agentName"] = agentName.split("/") self.__configDefaults = {} self.__configDefaults["MonitoringEnabled"] = self.am_getOption("MonitoringEnabled", True) - self.__configDefaults["Enabled"] = self.am_getOption("Status", "Active").lower() in ("active") + self.__configDefaults["Enabled"] = self.am_getOption("Status", "Active").lower() == "active" self.__configDefaults["PollingTime"] = self.am_getOption("PollingTime", 120) self.__configDefaults["MaxCycles"] = self.am_getOption("MaxCycles", 500) self.__configDefaults["WatchdogTime"] = self.am_getOption("WatchdogTime", 0) @@ -147,7 +146,6 @@ def __init__(self, agentName, loadName, baseAgentName=False, properties={}): self.activityMonitoring = True def __getCodeInfo(self): - try: self.__codeProperties["version"] = importlib.metadata.version( inspect.getmodule(self).__package__.split(".")[0] @@ -155,15 +153,21 @@ def __getCodeInfo(self): except Exception: self.log.exception(f"Failed to find version for {self!r}") self.__codeProperties["version"] = "unset" + try: - self.__agentModule = __import__(self.__class__.__module__, globals(), locals(), "__doc__") - except Exception as excp: - self.log.exception("Cannot load agent module", lException=excp) - try: - self.__codeProperties["description"] = getattr(self.__agentModule, "__doc__") - except Exception: - self.log.error("Missing property __doc__") + try: + self.__agentModule = importlib.import_module(self.__class__.__module__) + except ImportError as e: + self.log.exception("Cannot load agent module", lException=e) + raise e + try: + self.__codeProperties["description"] = getattr(self.__agentModule, "__doc__") + except AttributeError as e: + self.log.error("Missing property __doc__", lException=e) + raise e + except (ImportError, AttributeError): self.__codeProperties["description"] = "unset" + self.__codeProperties["DIRACVersion"] = DIRAC.version self.__codeProperties["platform"] = DIRAC.getPlatform() @@ -181,7 +185,7 @@ def am_initialize(self, *initArgs): if not isReturnStructure(result): return S_ERROR("initialize must return S_OK/S_ERROR") if not result["OK"]: - return S_ERROR("Error while initializing {}: {}".format(agentName, result["Message"])) + return S_ERROR(f"Error while initializing {agentName}: {result['Message']}") mkDir(self.am_getControlDirectory()) workDirectory = self.am_getWorkDirectory() mkDir(workDirectory) @@ -194,25 +198,24 @@ def am_initialize(self, *initArgs): if not self.am_Enabled(): return S_ERROR("Agent is disabled via the configuration") self.log.notice("=" * 40) - self.log.notice("Loaded agent module %s" % self.__moduleProperties["fullName"]) - self.log.notice(" Site: %s" % DIRAC.siteName()) - self.log.notice(" Setup: %s" % gConfig.getValue("/DIRAC/Setup")) - self.log.notice(" Agent version: %s" % self.__codeProperties["version"]) - self.log.notice(" DIRAC version: %s" % DIRAC.version) - self.log.notice(" DIRAC platform: %s" % DIRAC.getPlatform()) + self.log.notice(f"Loaded agent module {self.__moduleProperties['fullName']}") + self.log.notice(f" Site: {DIRAC.siteName()}") + self.log.notice(f" Agent version: {self.__codeProperties['version']}") + self.log.notice(f" DIRAC version: {DIRAC.version}") + self.log.notice(f" DIRAC platform: {DIRAC.getPlatform()}") pollingTime = int(self.am_getOption("PollingTime")) if pollingTime > 3600: - self.log.notice(" Polling time: %s hours" % (pollingTime / 3600.0)) + self.log.notice(f" Polling time: {pollingTime / 3600.0} hours") else: - self.log.notice(" Polling time: %s seconds" % self.am_getOption("PollingTime")) - self.log.notice(" Control dir: %s" % self.am_getControlDirectory()) - self.log.notice(" Work dir: %s" % self.am_getWorkDirectory()) + self.log.notice(f" Polling time: {self.am_getOption('PollingTime')} seconds") + self.log.notice(f" Control dir: {self.am_getControlDirectory()}") + self.log.notice(f" Work dir: {self.am_getWorkDirectory()}") if self.am_getOption("MaxCycles") > 0: - self.log.notice(" Cycles: %s" % self.am_getMaxCycles()) + self.log.notice(f" Cycles: {self.am_getMaxCycles()}") else: self.log.notice(" Cycles: unlimited") if self.am_getWatchdogTime() > 0: - self.log.notice(" Watchdog interval: %s" % self.am_getWatchdogTime()) + self.log.notice(f" Watchdog interval: {self.am_getWatchdogTime()}") else: self.log.notice(" Watchdog interval: disabled ") self.log.notice("=" * 40) @@ -230,7 +233,7 @@ def am_checkStopAgentFile(self): def am_createStopAgentFile(self): try: with open(self.am_getStopAgentFile(), "w") as fd: - fd.write("Dirac site agent Stopped at %s" % str(datetime.datetime.utcnow())) + fd.write(f"Dirac site agent Stopped at {str(datetime.datetime.utcnow())}") except Exception: pass @@ -317,7 +320,7 @@ def am_secureCall(self, functor, args=(), name=False): ) return result except Exception as e: - self.log.exception("Agent exception while calling method %s" % name, lException=e) + self.log.exception(f"Agent exception while calling method {name}", lException=e) return S_ERROR(f"Exception while calling {name} method: {str(e)}") def _setShifterProxy(self): @@ -334,7 +337,7 @@ def am_go(self): if not result["OK"]: return result self.log.notice("-" * 40) - self.log.notice("Starting cycle for module %s" % self.__moduleProperties["fullName"]) + self.log.notice(f"Starting cycle for module {self.__moduleProperties['fullName']}") mD = self.am_getMaxCycles() if mD > 0: cD = self.__moduleProperties["cyclesDone"] @@ -348,23 +351,24 @@ def am_go(self): elapsedTime = time.time() if self.activityMonitoring: initialWallTime, initialCPUTime, mem = self._startReportToMonitoring() - cycleResult = self.__executeModuleCycle() - if self.activityMonitoring and initialWallTime and initialCPUTime: + cycleResult = self.__executeModuleCycle() cpuPercentage = self._endReportToMonitoring(initialWallTime, initialCPUTime) + else: + cycleResult = self.__executeModuleCycle() # Increment counters self.__moduleProperties["cyclesDone"] += 1 # Show status elapsedTime = time.time() - elapsedTime self.__moduleProperties["totalElapsedTime"] += elapsedTime self.log.notice("-" * 40) - self.log.notice("Agent module %s run summary" % self.__moduleProperties["fullName"]) - self.log.notice(" Executed %s times previously" % self.__moduleProperties["cyclesDone"]) - self.log.notice(" Cycle took %.2f seconds" % elapsedTime) + self.log.notice(f"Agent module {self.__moduleProperties['fullName']} run summary") + self.log.notice(f" Executed {self.__moduleProperties['cyclesDone']} times previously") + self.log.notice(f" Cycle took {elapsedTime:.2f} seconds") averageElapsedTime = self.__moduleProperties["totalElapsedTime"] / self.__moduleProperties["cyclesDone"] - self.log.notice(" Average execution time: %.2f seconds" % (averageElapsedTime)) + self.log.notice(f" Average execution time: {averageElapsedTime:.2f} seconds") elapsedPollingRate = averageElapsedTime * 100 / self.am_getOption("PollingTime") - self.log.notice(" Polling time: %s seconds" % self.am_getOption("PollingTime")) - self.log.notice(" Average execution/polling time: %.2f%%" % elapsedPollingRate) + self.log.notice(f" Polling time: {self.am_getOption('PollingTime')} seconds") + self.log.notice(f" Average execution/polling time: {elapsedPollingRate:.2f}%") if cycleResult["OK"]: self.log.notice(" Cycle was successful") if self.activityMonitoring: diff --git a/src/DIRAC/Core/Base/AgentReactor.py b/src/DIRAC/Core/Base/AgentReactor.py index bfa076636b8..48f9e8bf13e 100644 --- a/src/DIRAC/Core/Base/AgentReactor.py +++ b/src/DIRAC/Core/Base/AgentReactor.py @@ -30,7 +30,6 @@ from DIRAC.ConfigurationSystem.Client import PathFinder from DIRAC.Core.Base.private.ModuleLoader import ModuleLoader from DIRAC.Core.Utilities import ThreadScheduler -from DIRAC.Core.Base.AgentModule import AgentModule class AgentReactor: @@ -58,7 +57,7 @@ class AgentReactor: def __init__(self, baseAgentName): self.__agentModules = {} - self.__loader = ModuleLoader("Agent", PathFinder.getAgentSection, AgentModule) + self.__loader = ModuleLoader("Agent", PathFinder.getAgentSection) self.__tasks = {} self.__baseAgentName = baseAgentName self.__scheduler = ThreadScheduler.ThreadScheduler(enableReactorThread=False, minPeriod=10) @@ -80,13 +79,11 @@ def loadAgentModules(self, modulesList, hideExceptions=False): instanceObj = agentData["classObj"](agentName, agentData["loadName"], self.__baseAgentName) result = instanceObj.am_initialize() if not result["OK"]: - return S_ERROR( - "Error while calling initialize method of {}: {}".format(agentName, result["Message"]) - ) + return S_ERROR(f"Error while calling initialize method of {agentName}: {result['Message']}") agentData["instanceObj"] = instanceObj except Exception as excp: if not hideExceptions: - gLogger.exception("Can't load agent %s" % agentName, lException=excp) + gLogger.exception(f"Can't load agent {agentName}", lException=excp) return S_ERROR(f"Can't load agent {agentName}: \n {excp}") agentPeriod = instanceObj.am_getPollingTime() result = self.__scheduler.addPeriodicTask( @@ -116,7 +113,7 @@ def runNumCycles(self, agentName=None, numCycles=1): result = self.setAgentModuleCyclesToExecute(aName, numCycles) if not result["OK"]: error = "Failed to set cycles to execute" - gLogger.error("%s:" % error, aName) + gLogger.error(f"{error}:", aName) break if error: return S_ERROR(error) @@ -131,7 +128,7 @@ def __finalize(self): try: self.__agentModules[agentName]["instanceObj"].finalize() except Exception as excp: - gLogger.exception("Failed to execute finalize for Agent: %s" % agentName, lException=excp) + gLogger.exception(f"Failed to execute finalize for Agent: {agentName}", lException=excp) def go(self): """ @@ -157,7 +154,7 @@ def setAgentModuleCyclesToExecute(self, agentName, maxCycles=1): Set number of cycles to execute for a given agent (previously defined) """ if agentName not in self.__agentModules: - return S_ERROR("%s has not been loaded" % agentName) + return S_ERROR(f"{agentName} has not been loaded") if maxCycles: try: maxCycles += self.__agentModules[agentName]["instanceObj"].am_getCyclesDone() @@ -181,11 +178,11 @@ def __checkControlDir(self): alive = agent.am_getModuleParam("alive") if alive: if agent.am_checkStopAgentFile(): - gLogger.info("Found StopAgent file for agent %s" % agentName) + gLogger.info(f"Found StopAgent file for agent {agentName}") alive = False if not alive: - gLogger.info("Stopping agent module %s" % (agentName)) + gLogger.info(f"Stopping agent module {agentName}") self.__scheduler.removeTask(self.__agentModules[agentName]["taskId"]) del self.__tasks[self.__agentModules[agentName]["taskId"]] self.__agentModules[agentName]["running"] = False diff --git a/src/DIRAC/Core/Base/CLI.py b/src/DIRAC/Core/Base/CLI.py index 9361b30d807..fd09243ea5c 100644 --- a/src/DIRAC/Core/Base/CLI.py +++ b/src/DIRAC/Core/Base/CLI.py @@ -38,13 +38,11 @@ def colorize(text, color): class CLI(cmd.Cmd): def __init__(self): - cmd.Cmd.__init__(self) self.indentSpace = 20 self._initSignals() def _handleSignal(self, sig, frame): - print("\nReceived signal", sig, ", exiting ...") self.do_quit(self) @@ -65,7 +63,7 @@ def _errMsg(self, errMsg): :param str errMsg: error message string :return: nothing """ - gLogger.error("{} {}".format(colorize("[ERROR]", "red"), errMsg)) + gLogger.error(f"{colorize('[ERROR]', 'red')} {errMsg}") def emptyline(self): pass @@ -100,13 +98,13 @@ def do_execfile(self, args): argss = args.split() fname = argss[0] if not os.path.exists(fname): - print("Error: File not found %s" % fname) + print(f"Error: File not found {fname}") return with open(fname) as input_cmd: contents = input_cmd.readlines() for line in contents: try: - gLogger.notice("\n--> Executing %s\n" % line) + gLogger.notice(f"\n--> Executing {line}\n") self.onecmd(line) except Exception as error: self._errMsg(str(error)) @@ -115,9 +113,9 @@ def do_execfile(self, args): def printPair(self, key, value, separator=":"): valueList = value.split("\n") - print("{}{}{} {}".format(key, " " * (self.indentSpace - len(key)), separator, valueList[0].strip())) + print(f"{key}{' ' * (self.indentSpace - len(key))}{separator} {valueList[0].strip()}") for valueLine in valueList[1:-1]: - print("{} {}".format(" " * self.indentSpace, valueLine.strip())) + print(f"{' ' * self.indentSpace} {valueLine.strip()}") def do_help(self, args): """ @@ -135,8 +133,8 @@ def do_help(self, args): else: command = args.split()[0].strip() try: - obj = getattr(self, "do_%s" % command) + obj = getattr(self, f"do_{command}") except Exception: - print("There's no such %s command" % command) + print(f"There's no such {command} command") return self.printPair(command, obj.__doc__[1:]) diff --git a/src/DIRAC/Core/Base/Client.py b/src/DIRAC/Core/Base/Client.py index 6ce57def625..8cb064e74f6 100644 --- a/src/DIRAC/Core/Base/Client.py +++ b/src/DIRAC/Core/Base/Client.py @@ -109,7 +109,10 @@ def _getRPC(self, rpc=None, url="", timeout=None): timeout = self.timeout self.__kwargs["timeout"] = timeout - rpc = RPCClientSelector(url, httpsClient=self.httpsClient, **self.__kwargs) + + rpc = RPCClientSelector( + url, httpsClient=self.httpsClient, diracxClient=getattr(self, "diracxClient", None), **self.__kwargs + ) return rpc @@ -127,7 +130,7 @@ def createClient(serviceName): def genFunc(funcName, arguments, handlerClassPath, doc): """Create a function with *funcName* taking *arguments*.""" doc = "" if doc is None else doc - funcDocString = "{}({}, **kwargs)\n".format(funcName, ", ".join(arguments)) + funcDocString = f"{funcName}({', '.join(arguments)}, **kwargs)\n" # do not describe self or cls in the parameter description if arguments and arguments[0] in ("self", "cls"): arguments = arguments[1:] @@ -156,8 +159,8 @@ def addFunctions(clientCls): # loop over all the nodes (classes, functions, imports) in the handlerModule for node in ast.iter_child_nodes(handlerAst): - # find only a class with the name of the handlerClass - if not (isinstance(node, ast.ClassDef) and node.name == handlerClassName): + # find only a class that starts with the name of the handlerClass + if not (isinstance(node, ast.ClassDef) and node.name.startswith(handlerClassName)): continue for member in ast.iter_child_nodes(node): # only look at functions diff --git a/src/DIRAC/Core/Base/DB.py b/src/DIRAC/Core/Base/DB.py index 55a91f592c2..6eee6c84197 100755 --- a/src/DIRAC/Core/Base/DB.py +++ b/src/DIRAC/Core/Base/DB.py @@ -10,12 +10,11 @@ class DB(DIRACDB, MySQL): """All DIRAC DB classes should inherit from this one (unless using sqlalchemy)""" def __init__(self, dbname, fullname, debug=False, parentLogger=None): - self.fullname = fullname result = getDBParameters(fullname) if not result["OK"]: - raise RuntimeError("Cannot get database parameters: %s" % result["Message"]) + raise RuntimeError(f"Cannot get database parameters: {result['Message']}") dbParameters = result["Value"] self.dbHost = dbParameters["Host"] @@ -35,7 +34,7 @@ def __init__(self, dbname, fullname, debug=False, parentLogger=None): ) if not self._connected: - raise RuntimeError("Can not connect to DB '%s', exiting..." % self.dbName) + raise RuntimeError(f"Can not connect to DB '{self.dbName}', exiting...") self.log.info("===================== MySQL ======================") self.log.info("User: " + self.dbUser) diff --git a/src/DIRAC/Core/Base/ElasticDB.py b/src/DIRAC/Core/Base/ElasticDB.py index bfebff523d8..7f73806bbbc 100644 --- a/src/DIRAC/Core/Base/ElasticDB.py +++ b/src/DIRAC/Core/Base/ElasticDB.py @@ -1,15 +1,15 @@ -""" ElasticDB is a base class used to connect an Elasticsearch database and manages queries. +""" ElasticDB is a base class used to connect an Opensearch database and manages queries. """ +from DIRAC.ConfigurationSystem.Client.Utilities import getElasticDBParameters from DIRAC.Core.Base.DIRACDB import DIRACDB from DIRAC.Core.Utilities.ElasticSearchDB import ElasticSearchDB -from DIRAC.ConfigurationSystem.Client.Utilities import getElasticDBParameters class ElasticDB(DIRACDB, ElasticSearchDB): """Class for interfacing DIRAC ES DB definitions to ES clusters""" ######################################################################## - def __init__(self, dbname, fullName, indexPrefix="", parentLogger=None): + def __init__(self, fullName, indexPrefix="", parentLogger=None): """c'tor :param self: self reference @@ -22,14 +22,13 @@ def __init__(self, dbname, fullName, indexPrefix="", parentLogger=None): result = getElasticDBParameters(fullName) if not result["OK"]: - raise RuntimeError("Cannot get database parameters: %s" % result["Message"]) + raise RuntimeError(f"Cannot get database parameters: {result['Message']}") dbParameters = result["Value"] self._dbHost = dbParameters["Host"] self._dbPort = dbParameters["Port"] - # we can have db which does not have any authentication... - self.__user = dbParameters.get("User", "") - self.__dbPassword = dbParameters.get("Password", "") + self.__user = dbParameters["User"] + self.__dbPassword = dbParameters["Password"] self.__useSSL = dbParameters.get("SSL", True) self.__useCRT = dbParameters.get("CRT", True) self.__ca_certs = dbParameters.get("ca_certs", None) @@ -51,15 +50,15 @@ def __init__(self, dbname, fullName, indexPrefix="", parentLogger=None): ) if not self._connected: - raise RuntimeError("Can not connect to ES cluster %s, exiting..." % self.clusterName) + raise RuntimeError(f"Can not connect to ES cluster {self.clusterName}, exiting...") - self.log.info("================= ElasticSearch ==================") - self.log.info("Host: %s " % self._dbHost) + self.log.debug("================= OpenSearch ==================") + self.log.debug(f"Host: {self._dbHost} ") if self._dbPort: - self.log.info("Port: %d " % self._dbPort) + self.log.debug("Port: %d " % self._dbPort) else: - self.log.info("Port: Not specified, assuming URL points to right location") - self.log.info( + self.log.debug("Port: Not specified, assuming URL points to right location") + self.log.debug( "Connecting with %s, %s:%s" % ( "SSL" if self.__useSSL else "no SSL", @@ -67,5 +66,5 @@ def __init__(self, dbname, fullName, indexPrefix="", parentLogger=None): "with password" if self.__dbPassword else "no password", ) ) - self.log.info("ClusterName: %s " % self.clusterName) - self.log.info("==================================================") + self.log.debug(f"ClusterName: {self.clusterName} ") + self.log.debug("==================================================") diff --git a/src/DIRAC/Core/Base/ExecutorMindHandler.py b/src/DIRAC/Core/Base/ExecutorMindHandler.py index 90369afbb35..cf8560b98f3 100644 --- a/src/DIRAC/Core/Base/ExecutorMindHandler.py +++ b/src/DIRAC/Core/Base/ExecutorMindHandler.py @@ -10,7 +10,6 @@ class ExecutorMindHandler(RequestHandler): - MSG_DEFINITIONS = { "ProcessTask": {"taskId": int, "taskStub": str, "eType": str}, "TaskDone": {"taskId": int, "taskStub": str}, @@ -80,7 +79,7 @@ def initializeHandler(cls, serviceInfoDict): 10, lambda: cls.log.verbose( "== Internal state ==", - "\n%s\n===========" % pprint.pformat(cls.__eDispatch._internals()), + f"\n{pprint.pformat(cls.__eDispatch._internals())}\n===========", ), ) return S_OK() @@ -172,7 +171,7 @@ def msg_TaskDone(self, msgObj): taskObj = result["Value"] result = self.__eDispatch.taskProcessed(self.srv_getTransportID(), msgObj.taskId, taskObj) if not result["OK"]: - gLogger.error("There was a problem processing task", "{}: {}".format(taskId, result["Message"])) + gLogger.error("There was a problem processing task", f"{taskId}: {result['Message']}") return S_OK() auth_msg_TaskFreeze = ["all"] @@ -191,7 +190,7 @@ def msg_TaskFreeze(self, msgObj): taskObj = result["Value"] result = self.__eDispatch.freezeTask(self.srv_getTransportID(), msgObj.taskId, msgObj.freezeTime, taskObj) if not result["OK"]: - gLogger.error("There was a problem freezing task", "{}: {}".format(taskId, result["Message"])) + gLogger.error("There was a problem freezing task", f"{taskId}: {result['Message']}") return S_OK() auth_msg_TaskError = ["all"] diff --git a/src/DIRAC/Core/Base/ExecutorModule.py b/src/DIRAC/Core/Base/ExecutorModule.py index 3bc10dbe9a6..d418e80860f 100644 --- a/src/DIRAC/Core/Base/ExecutorModule.py +++ b/src/DIRAC/Core/Base/ExecutorModule.py @@ -3,10 +3,11 @@ Just provides a number of functions used by all executors """ import os -from DIRAC import S_OK, S_ERROR, gConfig, gLogger, rootPath + +from DIRAC import S_ERROR, S_OK, gConfig, gLogger, rootPath from DIRAC.ConfigurationSystem.Client import PathFinder -from DIRAC.Core.Utilities.Shifter import setupShifterProxyInEnv from DIRAC.Core.Utilities.ReturnValues import isReturnStructure +from DIRAC.Core.Utilities.Shifter import setupShifterProxyInEnv class ExecutorModule: @@ -21,7 +22,6 @@ def _ex_initialize(cls, exeName, loadName): "loadSection": PathFinder.getExecutorSection(loadName), "messagesProcessed": 0, "reconnects": 0, - "setup": gConfig.getValue("/DIRAC/Setup", "Unknown"), } cls.__defaults = {} cls.__defaults["MonitoringEnabled"] = True @@ -43,20 +43,20 @@ def _ex_initialize(cls, exeName, loadName): try: result = cls.initialize() # pylint: disable=no-member except Exception as excp: - gLogger.exception("Exception while initializing %s" % loadName, lException=excp) - return S_ERROR("Exception while initializing: %s" % str(excp)) + gLogger.exception(f"Exception while initializing {loadName}", lException=excp) + return S_ERROR(f"Exception while initializing: {str(excp)}") if not isReturnStructure(result): - return S_ERROR("Executor %s does not return an S_OK/S_ERROR after initialization" % loadName) + return S_ERROR(f"Executor {loadName} does not return an S_OK/S_ERROR after initialization") return result def __installShifterProxy(self): shifterProxy = self.ex_getProperty("shifterProxy") if not shifterProxy: return S_OK() - location = "{}-{}".format(self.ex_getProperty("shifterProxyLocation"), shifterProxy) + location = f"{self.ex_getProperty('shifterProxyLocation')}-{shifterProxy}" result = setupShifterProxyInEnv(shifterProxy, location) if not result["OK"]: - self.log.error("Cannot set shifter proxy: %s" % result["Message"]) + self.log.error(f"Cannot set shifter proxy: {result['Message']}") return result @classmethod @@ -105,7 +105,7 @@ def __serialize(self, taskId, taskObj): try: result = self.serializeTask(taskObj) except Exception as excp: - gLogger.exception("Exception while serializing task %s" % taskId, lException=excp) + gLogger.exception(f"Exception while serializing task {taskId}", lException=excp) return S_ERROR(f"Cannot serialize task {taskId}: {str(excp)}") if not isReturnStructure(result): raise Exception("serializeTask does not return a return structure") @@ -115,7 +115,7 @@ def __deserialize(self, taskId, taskStub): try: result = self.deserializeTask(taskStub) except Exception as excp: - gLogger.exception("Exception while deserializing task %s" % taskId, lException=excp) + gLogger.exception(f"Exception while deserializing task {taskId}", lException=excp) return S_ERROR(f"Cannot deserialize task {taskId}: {str(excp)}") if not isReturnStructure(result): raise Exception("deserializeTask does not return a return structure") @@ -125,10 +125,10 @@ def _ex_processTask(self, taskId, taskStub): self.__properties["shifterProxy"] = self.ex_getOption("shifterProxy") self.__freezeTime = 0 self.__fastTrackEnabled = True - self.log.verbose("Task %s: Received" % str(taskId)) + self.log.verbose(f"Task {str(taskId)}: Received") result = self.__deserialize(taskId, taskStub) if not result["OK"]: - self.log.error("Can not deserialize task", "Task {}: {}".format(str(taskId), result["Message"])) + self.log.error("Can not deserialize task", f"Task {str(taskId)}: {result['Message']}") return result taskObj = result["Value"] # Shifter proxy? @@ -147,7 +147,7 @@ def _ex_processTask(self, taskId, taskStub): # Serialize again result = self.__serialize(taskId, taskObj) if not result["OK"]: - self.log.verbose("Task {}: Cannot serialize: {}".format(str(taskId), result["Message"])) + self.log.verbose(f"Task {str(taskId)}: Cannot serialize: {result['Message']}") return result taskStub = result["Value"] # Try fast track @@ -155,7 +155,7 @@ def _ex_processTask(self, taskId, taskStub): if not self.__freezeTime and self.__fastTrackEnabled: result = self.fastTrackDispatch(taskId, taskObj) if not result["OK"]: - self.log.error("FastTrackDispatch failed for job", "{}: {}".format(taskId, result["Message"])) + self.log.error("FastTrackDispatch failed for job", f"{taskId}: {result['Message']}") else: fastTrackType = result["Value"] diff --git a/src/DIRAC/Core/Base/ExecutorReactor.py b/src/DIRAC/Core/Base/ExecutorReactor.py index a4c1605c0e3..41aa470bc57 100644 --- a/src/DIRAC/Core/Base/ExecutorReactor.py +++ b/src/DIRAC/Core/Base/ExecutorReactor.py @@ -26,7 +26,6 @@ from DIRAC.Core.DISET.MessageClient import MessageClient from DIRAC.ConfigurationSystem.Client import PathFinder from DIRAC.Core.Base.private.ModuleLoader import ModuleLoader -from DIRAC.Core.Base.ExecutorModule import ExecutorModule class ExecutorReactor: @@ -85,13 +84,13 @@ def connect(self): ) if result["OK"]: self.__aliveLock.alive() - gLogger.info("Connected to %s" % self.__mindName) + gLogger.info(f"Connected to {self.__mindName}") return result def __disconnected(self, msgClient): retryCount = 0 while True: - gLogger.notice("Trying to reconnect to %s" % self.__mindName) + gLogger.notice(f"Trying to reconnect to {self.__mindName}") result = self.__msgClient.connect( executorTypes=list(self.__modules), maxTasks=self.__maxTasks, extraArgs=self.__extraArgs ) @@ -99,12 +98,12 @@ def __disconnected(self, msgClient): if result["OK"]: if retryCount >= self.__reconnectRetries: self.__aliveLock.alive() - gLogger.notice("Reconnected to %s" % self.__mindName) + gLogger.notice(f"Reconnected to {self.__mindName}") return S_OK() retryCount += 1 if retryCount == self.__reconnectRetries: self.__aliveLock.alive() - gLogger.info("Connect error failed: %s" % result["Message"]) + gLogger.info(f"Connect error failed: {result['Message']}") gLogger.notice("Failed to reconnect. Sleeping for %d seconds" % self.__reconnectSleep) time.sleep(self.__reconnectSleep) @@ -155,9 +154,7 @@ def __processTask(self, msgObj): result = self.__msgClient.createMessage(msgName) if not result["OK"]: - return self.__sendExecutorError( - eType, taskId, "Can't generate {} message: {}".format(msgName, result["Message"]) - ) + return self.__sendExecutorError(eType, taskId, f"Can't generate {msgName} message: {result['Message']}") gLogger.verbose(f"Task {str(taskId)}: Sending {msgName}") msgObj = result["Value"] msgObj.taskId = taskId @@ -177,13 +174,13 @@ def __moduleProcess(self, eType, taskId, taskStub, fastTrackLevel=0): try: result = modInstance._ex_processTask(taskId, taskStub) except Exception as excp: - gLogger.exception("Error while processing task %s" % taskId, lException=excp) + gLogger.exception(f"Error while processing task {taskId}", lException=excp) return S_ERROR(f"Error processing task {taskId}: {excp}") self.__storeInstance(eType, modInstance) if not result["OK"]: - return S_OK(("TaskError", taskStub, "Error: %s" % result["Message"])) + return S_OK(("TaskError", taskStub, f"Error: {result['Message']}")) taskStub, freezeTime, fastTrackType = result["Value"] if freezeTime: return S_OK(("TaskFreeze", taskStub, freezeTime)) @@ -192,7 +189,7 @@ def __moduleProcess(self, eType, taskId, taskStub, fastTrackLevel=0): gLogger.notice(f"Fast tracking task {taskId} to {fastTrackType}") return self.__moduleProcess(fastTrackType, taskId, taskStub, fastTrackLevel + 1) else: - gLogger.notice("Stopping %s fast track. Sending back to the mind" % (taskId)) + gLogger.notice(f"Stopping {taskId} fast track. Sending back to the mind") return S_OK(("TaskDone", taskStub, True)) @@ -203,9 +200,8 @@ def __moduleProcess(self, eType, taskId, taskStub, fastTrackLevel=0): def __init__(self): self.__aliveLock = self.AliveLock() self.__executorModules = {} - self.__codeModules = {} self.__minds = {} - self.__loader = ModuleLoader("Executor", PathFinder.getExecutorSection, ExecutorModule) + self.__loader = ModuleLoader("Executor", PathFinder.getExecutorSection) def loadModules(self, modulesList, hideExceptions=False): """ @@ -230,7 +226,7 @@ def go(self): mc = self.__minds[mind] mc.addModule(name, exeClass) for mindName in self.__minds: - gLogger.info("Trying to connect to %s" % mindName) + gLogger.info(f"Trying to connect to {mindName}") result = self.__minds[mindName].connect() if not result["OK"]: return result diff --git a/src/DIRAC/Core/Base/SQLAlchemyDB.py b/src/DIRAC/Core/Base/SQLAlchemyDB.py index 0c940ebbee6..9400168665e 100644 --- a/src/DIRAC/Core/Base/SQLAlchemyDB.py +++ b/src/DIRAC/Core/Base/SQLAlchemyDB.py @@ -3,7 +3,9 @@ Uses sqlalchemy """ + import datetime +from urllib import parse as urlparse from sqlalchemy import create_engine, desc, exc from sqlalchemy.engine.reflection import Inspector from sqlalchemy.orm import sessionmaker @@ -12,6 +14,7 @@ from DIRAC import gConfig, gLogger, S_OK, S_ERROR from DIRAC.Core.Base.DIRACDB import DIRACDB from DIRAC.ConfigurationSystem.Client.Utilities import getDBParameters +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader class SQLAlchemyDB(DIRACDB): @@ -30,6 +33,8 @@ def __init__(self, *args, **kwargs): self.extensions = gConfig.getValue("DIRAC/Extensions", []) self.tablesList = [] + self.objectLoader = ObjectLoader() + def _initializeConnection(self, dbPath): """ Collect from the CS all the info needed to connect to the DB. @@ -37,14 +42,16 @@ def _initializeConnection(self, dbPath): result = getDBParameters(dbPath) if not result["OK"]: - raise Exception("Cannot get database parameters: %s" % result["Message"]) + raise Exception(f"Cannot get database parameters: {result['Message']}") dbParameters = result["Value"] - self.log.debug("db parameters: %s" % dbParameters) + db_param_redacted = dbParameters.copy() + db_param_redacted["Password"] = "**REDACTED**" + self.log.debug(f"db parameters: {db_param_redacted}") self.host = dbParameters["Host"] self.port = dbParameters["Port"] self.user = dbParameters["User"] - self.password = dbParameters["Password"] + self.password = urlparse.quote_plus(dbParameters["Password"]) self.dbName = dbParameters["DBName"] self.engine = create_engine( @@ -64,28 +71,12 @@ def _createTablesIfNotThere(self, tablesList): for table in tablesList: if table not in tablesInDB: - found = False - # is it in the extension? (fully or extended) - for ext in self.extensions: - try: - getattr( - __import__(ext + self.__class__.__module__, globals(), locals(), [table]), table - ).__table__.create( - self.engine - ) # pylint: disable=no-member - found = True - break - except (ImportError, AttributeError): - continue - # If not found in extensions, import it from DIRAC base. - if not found: - getattr( - __import__(self.__class__.__module__, globals(), locals(), [table]), table - ).__table__.create( - self.engine - ) # pylint: disable=no-member + result = self.objectLoader.loadObject(self.__class__.__module__, table) + if not result["OK"]: + return result + result["Value"].__table__.create(self.engine) else: - gLogger.debug("Table %s already exists" % table) + gLogger.debug(f"Table {table} already exists") def insert(self, table, params): """ @@ -100,18 +91,10 @@ def insert(self, table, params): # expire_on_commit is set to False so that we can still use the object after we close the session session = self.sessionMaker_o(expire_on_commit=False) # FIXME: should we use this flag elsewhere? - found = False - for ext in self.extensions: - try: - tableRow_o = getattr(__import__(ext + self.__class__.__module__, globals(), locals(), [table]), table)() - found = True - break - except (ImportError, AttributeError): - continue - # If not found in extensions, import it from DIRAC base (this same module). - if not found: - tableRow_o = getattr(__import__(self.__class__.__module__, globals(), locals(), [table]), table)() - + result = self.objectLoader.loadObject(self.__class__.__module__, table) + if not result["OK"]: + return result + tableRow_o = result["Value"]() tableRow_o.fromDict(params) try: @@ -119,15 +102,17 @@ def insert(self, table, params): session.commit() return S_OK() except exc.IntegrityError as err: - self.log.warn("insert: trying to insert a duplicate key? %s" % err) + self.log.warn(f"insert: trying to insert a duplicate key? {err}") session.rollback() except exc.SQLAlchemyError as e: session.rollback() self.log.exception("insert: unexpected exception", lException=e) - return S_ERROR("insert: unexpected exception %s" % e) + return S_ERROR(f"insert: unexpected exception {e}") finally: session.close() + return S_OK() + def select(self, table, params): """ Uses params to build conditional SQL statement ( WHERE ... ). @@ -141,18 +126,10 @@ def select(self, table, params): session = self.sessionMaker_o() - # finding the table - found = False - for ext in self.extensions: - try: - table_c = getattr(__import__(ext + self.__class__.__module__, globals(), locals(), [table]), table) - found = True - break - except (ImportError, AttributeError): - continue - # If not found in extensions, import it from DIRAC base (this same module). - if not found: - table_c = getattr(__import__(self.__class__.__module__, globals(), locals(), [table]), table) + result = self.objectLoader.loadObject(self.__class__.__module__, table) + if not result["OK"]: + return result + table_c = result["Value"] # handling query conditions found in 'Meta' columnNames = [column.lower() for column in params.get("Meta", {}).get("columns", [])] @@ -187,7 +164,7 @@ def select(self, table, params): elif isinstance(columnValue, (str, datetime.datetime, bool)): select = select.filter(column_a == columnValue) else: - self.log.error("type(columnValue) == %s" % type(columnValue)) + self.log.error(f"type(columnValue) == {type(columnValue)}") if older: column_a = getattr(table_c, older[0].lower()) select = select.filter(column_a < older[1]) @@ -221,7 +198,7 @@ def select(self, table, params): except exc.SQLAlchemyError as e: session.rollback() self.log.exception("select: unexpected exception", lException=e) - return S_ERROR("select: unexpected exception %s" % e) + return S_ERROR(f"select: unexpected exception {e}") finally: session.close() @@ -236,17 +213,10 @@ def delete(self, table, params): """ session = self.sessionMaker_o() - found = False - for ext in self.extensions: - try: - table_c = getattr(__import__(ext + self.__class__.__module__, globals(), locals(), [table]), table) - found = True - break - except (ImportError, AttributeError): - continue - # If not found in extensions, import it from DIRAC base (this same module). - if not found: - table_c = getattr(__import__(self.__class__.__module__, globals(), locals(), [table]), table) + result = self.objectLoader.loadObject(self.__class__.__module__, table) + if not result["OK"]: + return result + table_c = result["Value"] # handling query conditions found in 'Meta' older = params.get("Meta", {}).get("older", None) @@ -266,7 +236,7 @@ def delete(self, table, params): elif isinstance(columnValue, (str, datetime.datetime, bool)): deleteQuery = deleteQuery.filter(column_a == columnValue) else: - self.log.error("type(columnValue) == %s" % type(columnValue)) + self.log.error(f"type(columnValue) == {type(columnValue)}") if older: column_a = getattr(table_c, older[0].lower()) deleteQuery = deleteQuery.filter(column_a < older[1]) @@ -290,6 +260,6 @@ def delete(self, table, params): except exc.SQLAlchemyError as e: session.rollback() self.log.exception("delete: unexpected exception", lException=e) - return S_ERROR("delete: unexpected exception %s" % e) + return S_ERROR(f"delete: unexpected exception {e}") finally: session.close() diff --git a/src/DIRAC/Core/Base/Script.py b/src/DIRAC/Core/Base/Script.py index 2f28b93405f..bc37d258953 100755 --- a/src/DIRAC/Core/Base/Script.py +++ b/src/DIRAC/Core/Base/Script.py @@ -38,8 +38,7 @@ def __call__(self, func=None): """Set the wrapped function or call the script This function is either called with a decorator or directly to call the - underlying function. When running with Python 2 the raw function will always - be called however in Python 3 the priorities will be applied from the + underlying function. The priorities will be applied from the dirac.extension_metadata entry_point. """ # If func is provided then the decorator is being applied to a function @@ -51,7 +50,7 @@ def __call__(self, func=None): return functools.wraps(func)(self) # Iterate through all known entry_points looking for self.scriptName - matches = [ep for ep in metadata.entry_points()["console_scripts"] if ep.name == self.scriptName] + matches = [ep for ep in metadata.entry_points(group="console_scripts") if ep.name == self.scriptName] if not matches: raise NotImplementedError("Something is very wrong") @@ -69,7 +68,7 @@ def __call__(self, func=None): "Invalid dirac- console_scripts entry_point: " + repr(entrypoint) + "\n" - + "All dirac- console_scripts should be wrapped in the DiracScript " + + "All dirac- console_scripts should be wrapped in the Script " + "decorator to ensure extension overlays are applied correctly." ) return entrypointFunc._func() @@ -113,8 +112,6 @@ def initialize(cls, script=False, ignoreErrors=False, initializeMonitor=False, e cls.scriptName = script cls.localCfg.setConfigurationForScript(cls.scriptName) - if not ignoreErrors: - cls.localCfg.addMandatoryEntry("/DIRAC/Setup") resultDict = cls.localCfg.loadUserData() if not ignoreErrors and not resultDict["OK"]: gLogger.error("There were errors when loading configuration", resultDict["Message"]) diff --git a/src/DIRAC/Core/Base/private/ModuleLoader.py b/src/DIRAC/Core/Base/private/ModuleLoader.py index 34ce2efa6ec..c1139c6ee38 100644 --- a/src/DIRAC/Core/Base/private/ModuleLoader.py +++ b/src/DIRAC/Core/Base/private/ModuleLoader.py @@ -3,22 +3,20 @@ import os from DIRAC.Core.Utilities import List from DIRAC import gConfig, S_ERROR, S_OK, gLogger -from DIRAC.ConfigurationSystem.Client import PathFinder from DIRAC.Core.Utilities.Extensions import extensionsByPriority, recurseImport class ModuleLoader: - def __init__(self, importLocation, sectionFinder, superClass, csSuffix=False, moduleSuffix=False): + def __init__(self, importLocation, sectionFinder, csSuffix=False, moduleSuffix=False): self.__modules = {} self.__loadedModules = {} - self.__superClass = superClass # Function to find the self.__sectionFinder = sectionFinder # Import from where? .System.. self.__importLocation = importLocation # Where to look in the CS for the module? /Systems/// if not csSuffix: - csSuffix = "%ss" % importLocation + csSuffix = f"{importLocation}s" self.__csSuffix = csSuffix # Module suffix (for Handlers) self.__modSuffix = moduleSuffix @@ -34,10 +32,10 @@ def loadModules(self, modulesList, hideExceptions=False): Load all modules required in moduleList """ for modName in modulesList: - gLogger.verbose("Checking %s" % modName) + gLogger.verbose(f"Checking {modName}") # if it's a executor modName name just load it and be done with it if "/" in modName: - gLogger.verbose("Module %s seems to be a valid name. Try to load it!" % modName) + gLogger.verbose(f"Module {modName} seems to be a valid name. Try to load it!") result = self.loadModule(modName, hideExceptions=hideExceptions) if not result["OK"]: return result @@ -45,9 +43,8 @@ def loadModules(self, modulesList, hideExceptions=False): # Check if it's a system name # Look in the CS system = modName - # Can this be generated with sectionFinder? - csPath = "%s/Executors" % PathFinder.getSystemSection(system) - gLogger.verbose("Exploring %s to discover modules" % csPath) + csPath = f"/Systems/{system}/Executors" + gLogger.verbose(f"Exploring {csPath} to discover modules") result = gConfig.getSections(csPath) if result["OK"]: # Add all modules in the CS :P @@ -71,14 +68,14 @@ def loadModules(self, modulesList, hideExceptions=False): if not parentModule: continue parentPath = parentModule.__path__[0] - gLogger.notice("Found modules path at %s" % parentImport) + gLogger.notice(f"Found modules path at {parentImport}") for entry in os.listdir(parentPath): if entry == "__init__.py" or not entry.endswith(".py"): continue if not os.path.isfile(os.path.join(parentPath, entry)): continue modName = f"{system}/{entry[:-3]}" - gLogger.verbose("Trying to import %s" % modName) + gLogger.verbose(f"Trying to import {modName}") result = self.loadModule(modName, hideExceptions=hideExceptions, parentModule=parentModule) return S_OK() @@ -94,12 +91,12 @@ def loadModule(self, modName, hideExceptions=False, parentModule=False): return S_OK() modList = modName.split("/") if len(modList) != 2: - return S_ERROR("Can't load %s: Invalid module name" % (modName)) + return S_ERROR(f"Can't load {modName}: Invalid module name") csSection = self.__sectionFinder(modName) - loadGroup = gConfig.getValue("%s/Load" % csSection, []) + loadGroup = gConfig.getValue(f"{csSection}/Load", []) # Check if it's a load group if loadGroup: - gLogger.info("Found load group {}. Will load {}".format(modName, ", ".join(loadGroup))) + gLogger.info(f"Found load group {modName}. Will load {', '.join(loadGroup)}") for loadModName in loadGroup: if "/" not in loadModName: loadModName = f"{modList[0]}/{loadModName}" @@ -108,10 +105,10 @@ def loadModule(self, modName, hideExceptions=False, parentModule=False): return result return S_OK() # Normal load - loadName = gConfig.getValue("%s/Module" % csSection, "") + loadName = gConfig.getValue(f"{csSection}/Module", "") if not loadName: loadName = modName - gLogger.info("Loading %s" % (modName)) + gLogger.info(f"Loading {modName}") else: if "/" not in loadName: loadName = f"{modList[0]}/{loadName}" @@ -119,7 +116,7 @@ def loadModule(self, modName, hideExceptions=False, parentModule=False): # If already loaded, skip loadList = loadName.split("/") if len(loadList) != 2: - return S_ERROR("Can't load %s: Invalid module name" % (loadName)) + return S_ERROR(f"Can't load {loadName}: Invalid module name") system, module = loadList # Load className = module @@ -128,20 +125,19 @@ def loadModule(self, modName, hideExceptions=False, parentModule=False): if loadName not in self.__loadedModules: # Check if handler is defined loadCSSection = self.__sectionFinder(loadName) - handlerPath = gConfig.getValue("%s/HandlerPath" % loadCSSection, "") + handlerPath = gConfig.getValue(f"{loadCSSection}/HandlerPath", "") if handlerPath: - gLogger.info(f"Trying to {loadName} from CS defined path {handlerPath}") - gLogger.verbose(f"Found handler for {loadName}: {handlerPath}") + gLogger.info(f"Trying to load handler for {loadName} from CS defined path {handlerPath}") handlerPath = handlerPath.replace("/", ".") if handlerPath.endswith(".py"): handlerPath = handlerPath[:-3] className = List.fromChar(handlerPath, ".")[-1] result = recurseImport(handlerPath) if not result["OK"]: - return S_ERROR("Cannot load user defined handler {}: {}".format(handlerPath, result["Message"])) - gLogger.verbose("Loading %s" % handlerPath) + return S_ERROR(f"Cannot load user defined handler {handlerPath}: {result['Message']}") + gLogger.verbose(f"Loading {handlerPath}") elif parentModule: - gLogger.info("Trying to autodiscover %s from parent" % loadName) + gLogger.info(f"Trying to autodiscover {loadName} from parent") # If we've got a parent module, load from there. modImport = module if self.__modSuffix: @@ -149,23 +145,23 @@ def loadModule(self, modName, hideExceptions=False, parentModule=False): result = recurseImport(modImport, parentModule, hideExceptions=hideExceptions) else: # Check to see if the module exists in any of the root modules - gLogger.info("Trying to autodiscover %s" % loadName) + gLogger.info(f"Trying to autodiscover {loadName}") for rootModule in extensionsByPriority(): importString = f"{rootModule}.{system}System.{self.__importLocation}.{module}" if self.__modSuffix: importString = f"{importString}{self.__modSuffix}" - gLogger.verbose("Trying to load %s" % importString) + gLogger.verbose(f"Trying to load {importString}") result = recurseImport(importString, hideExceptions=hideExceptions) # Error while loading if not result["OK"]: return result # Something has been found! break :) if result["Value"]: - gLogger.verbose("Found %s" % importString) + gLogger.verbose(f"Found {importString}") break # Nothing found if not result["Value"]: - return S_ERROR("Could not find %s" % loadName) + return S_ERROR(f"Could not find {loadName}") modObj = result["Value"] try: # Try to get the class from the module @@ -176,10 +172,8 @@ def loadModule(self, modName, hideExceptions=False, parentModule=False): else: location = modObj.__path__ gLogger.exception(f"{location} module does not have a {module} class!") - return S_ERROR("Cannot load %s" % module) - # Check if it's subclass - if not issubclass(modClass, self.__superClass): - return S_ERROR(f"{loadName} has to inherit from {self.__superClass.__name__}") + return S_ERROR(f"Cannot load {module}") + self.__loadedModules[loadName] = {"classObj": modClass, "moduleObj": modObj} # End of loading of 'loadName' module @@ -188,6 +182,6 @@ def loadModule(self, modName, hideExceptions=False, parentModule=False): # keep the name of the real code module self.__modules[modName]["modName"] = modName self.__modules[modName]["loadName"] = loadName - gLogger.notice("Loaded module %s" % modName) + gLogger.notice(f"Loaded module {modName}") return S_OK() diff --git a/src/DIRAC/Core/DISET/AuthManager.py b/src/DIRAC/Core/DISET/AuthManager.py index 9f4b46abc19..f78cbe08e8d 100755 --- a/src/DIRAC/Core/DISET/AuthManager.py +++ b/src/DIRAC/Core/DISET/AuthManager.py @@ -1,5 +1,9 @@ """ Module that holds DISET Authorization class for services """ +from threading import Lock + +from cachetools import TTLCache + from DIRAC.ConfigurationSystem.Client.Config import gConfig from DIRAC.ConfigurationSystem.Client.Helpers import Registry from DIRAC.Core.Security import Properties @@ -26,6 +30,10 @@ def __init__(self, authSection): :param authSection: Section containing the authorization rules """ self.authSection = authSection + self._cache_getUsersInGroup = TTLCache(maxsize=1000, ttl=60) + self._cache_getUsersInGroupLock = Lock() + self._cache_getUsernameForDN = TTLCache(maxsize=1000, ttl=60) + self._cache_getUsernameForDNLock = Lock() def authQuery(self, methodQuery, credDict, defaultProperties=False): """ @@ -40,12 +48,12 @@ def authQuery(self, methodQuery, credDict, defaultProperties=False): """ userString = "" if self.KW_DN in credDict: - userString += "DN=%s" % credDict[self.KW_DN] + userString += f"DN={credDict[self.KW_DN]}" if self.KW_GROUP in credDict: - userString += " group=%s" % credDict[self.KW_GROUP] + userString += f" group={credDict[self.KW_GROUP]}" if self.KW_EXTRA_CREDENTIALS in credDict: - userString += " extraCredentials=%s" % str(credDict[self.KW_EXTRA_CREDENTIALS]) - self.__authLogger.debug("Trying to authenticate %s" % userString) + userString += f" extraCredentials={str(credDict[self.KW_EXTRA_CREDENTIALS])}" + self.__authLogger.debug(f"Trying to authenticate {userString}") # Get properties requiredProperties = self.getValidPropertiesForMethod(methodQuery, defaultProperties) # Extract valid groups @@ -159,7 +167,7 @@ def getHostNickName(self, credDict): return False retVal = Registry.getHostnameForDN(credDict[self.KW_DN]) if not retVal["OK"]: - gLogger.warn("Cannot find hostname for DN {}: {}".format(credDict[self.KW_DN], retVal["Message"])) + gLogger.warn(f"Cannot find hostname for DN {credDict[self.KW_DN]}: {retVal['Message']}") return False credDict[self.KW_USERNAME] = retVal["Value"] credDict[self.KW_PROPERTIES] = Registry.getPropertiesForHost(credDict[self.KW_USERNAME], []) @@ -181,12 +189,12 @@ def getValidPropertiesForMethod(self, method, defaultProperties=False): if not isinstance(defaultProperties, (list, tuple)): return List.fromChar(defaultProperties) return defaultProperties - defaultPath = "%s/Default" % "/".join(method.split("/")[:-1]) + defaultPath = f"{'/'.join(method.split('/')[:-1])}/Default" authProps = gConfig.getValue(f"{self.authSection}/{defaultPath}", []) if authProps: self.__authLogger.debug(f"Method {method} has no properties defined using {defaultPath}") return authProps - self.__authLogger.debug("Method %s has no authorization rules defined. Allowing no properties" % method) + self.__authLogger.debug(f"Method {method} has no authorization rules defined. Allowing no properties") return [] def getValidGroups(self, rawProperties): @@ -257,10 +265,20 @@ def getUsername(self, credDict): return False credDict[self.KW_GROUP] = result["Value"] credDict[self.KW_PROPERTIES] = Registry.getPropertiesForGroup(credDict[self.KW_GROUP], []) - usersInGroup = Registry.getUsersInGroup(credDict[self.KW_GROUP], []) + + with self._cache_getUsersInGroupLock: + usersInGroup = self._cache_getUsersInGroup.get(credDict[self.KW_GROUP]) + if usersInGroup is None: + usersInGroup = Registry.getUsersInGroup(credDict[self.KW_GROUP], []) + self._cache_getUsersInGroup[credDict[self.KW_GROUP]] = usersInGroup if not usersInGroup: return False - retVal = Registry.getUsernameForDN(credDict[self.KW_DN], usersInGroup) + + with self._cache_getUsernameForDNLock: + retVal = self._cache_getUsernameForDN.get(credDict[self.KW_DN]) + if retVal is None: + retVal = Registry.getUsernameForDN(credDict[self.KW_DN], usersInGroup) + self._cache_getUsernameForDN[credDict[self.KW_DN]] = retVal if retVal["OK"]: credDict[self.KW_USERNAME] = retVal["Value"] return True diff --git a/src/DIRAC/Core/DISET/MessageClient.py b/src/DIRAC/Core/DISET/MessageClient.py index 1aa47937f2e..59e8a32db5d 100755 --- a/src/DIRAC/Core/DISET/MessageClient.py +++ b/src/DIRAC/Core/DISET/MessageClient.py @@ -137,19 +137,19 @@ def sendMessage(self, msgObj): def subscribeToAllMessages(self, cbFunction): if not callable(cbFunction): - return S_ERROR("%s is not callable" % cbFunction) + return S_ERROR(f"{cbFunction} is not callable") self.__specialCallbacks["msg"].append(cbFunction) return S_OK() def subscribeToMessage(self, msgName, cbFunction): if not callable(cbFunction): - return S_ERROR("%s is not callable" % cbFunction) + return S_ERROR(f"{cbFunction} is not callable") self.__callbacks[msgName] = cbFunction return S_OK() def subscribeToDisconnect(self, cbFunction): if not callable(cbFunction): - return S_ERROR("%s is not callable" % cbFunction) + return S_ERROR(f"{cbFunction} is not callable") self.__specialCallbacks["drop"].append(cbFunction) return S_OK() diff --git a/src/DIRAC/Core/DISET/RPCClient.py b/src/DIRAC/Core/DISET/RPCClient.py index c3921390d9b..d47c8ef6195 100755 --- a/src/DIRAC/Core/DISET/RPCClient.py +++ b/src/DIRAC/Core/DISET/RPCClient.py @@ -36,7 +36,7 @@ def __call__(self, *args, **kwargs): return self.__doRPCFunc(self.__remoteFuncName, args, **kwargs) def __str__(self): - return "" % self.__remoteFuncName + return f"" class RPCClient: diff --git a/src/DIRAC/Core/DISET/RequestHandler.py b/src/DIRAC/Core/DISET/RequestHandler.py index 2df8e48ebdc..309fa8036f6 100755 --- a/src/DIRAC/Core/DISET/RequestHandler.py +++ b/src/DIRAC/Core/DISET/RequestHandler.py @@ -14,16 +14,12 @@ def getServiceOption(serviceInfo, optionName, defaultValue): - """Get service option resolving default values from the master service""" + """Get service option resolving default values from the controller service""" if optionName[0] == "/": return gConfig.getValue(optionName, defaultValue) for csPath in serviceInfo["csPaths"]: result = gConfig.getOption( - "%s/%s" - % ( - csPath, - optionName, - ), + f"{csPath}/{optionName}", defaultValue, ) if result["OK"]: @@ -36,7 +32,7 @@ def __init__(self, msg): self.__msg = msg def __str__(self): - return "ConnectionError: %s" % self.__msg + return f"ConnectionError: {self.__msg}" class RequestHandler: @@ -127,7 +123,7 @@ def _rh_executeAction(self, proposalTuple): elif actionType == "Connection": retVal = self.__doConnection(actionTuple[1]) else: - return S_ERROR("Unknown action %s" % actionType) + return S_ERROR(f"Unknown action {actionType}") except ConnectionError as excp: gLogger.error("ConnectionError", str(excp)) return S_ERROR(excp) @@ -173,15 +169,15 @@ def __doFileTransfer(self, sDirection): # Reconvert to tuple fileInfo = tuple(retVal["Value"]) sDirection = f"{sDirection[0].lower()}{sDirection[1:]}" - if "transfer_%s" % sDirection not in dir(self): - self.__trPool.send(self.__trid, S_ERROR("Service can't transfer files %s" % sDirection)) + if f"transfer_{sDirection}" not in dir(self): + self.__trPool.send(self.__trid, S_ERROR(f"Service can't transfer files {sDirection}")) return retVal = self.__trPool.send(self.__trid, S_OK("Accepted")) if not retVal["OK"]: return retVal - self.__logRemoteQuery("FileTransfer/%s" % sDirection, fileInfo) + self.__logRemoteQuery(f"FileTransfer/{sDirection}", fileInfo) - self.__lockManager.lock("FileTransfer/%s" % sDirection) + self.__lockManager.lock(f"FileTransfer/{sDirection}") try: try: fileHelper = FileHelper(self.__trPool.get(self.__trid)) @@ -201,17 +197,17 @@ def __doFileTransfer(self, sDirection): fileHelper.setDirection("toClient") uRetVal = self.transfer_listBulk(fileInfo[0], fileInfo[1], fileHelper) else: - return S_ERROR("Direction %s does not exist!!!" % sDirection) + return S_ERROR(f"Direction {sDirection} does not exist!!!") if uRetVal["OK"] and not fileHelper.finishedTransmission(): gLogger.error("You haven't finished receiving/sending the file", str(fileInfo)) return S_ERROR("Incomplete transfer") del fileHelper return uRetVal finally: - self.__lockManager.unlock("FileTransfer/%s" % sDirection) + self.__lockManager.unlock(f"FileTransfer/{sDirection}") except Exception as e: # pylint: disable=broad-except - gLogger.exception("Uncaught exception when serving Transfer", "%s" % sDirection, lException=e) + gLogger.exception("Uncaught exception when serving Transfer", f"{sDirection}", lException=e) return S_ERROR(f"Server error while serving {sDirection}: {repr(e)}") def transfer_fromClient(self, fileId, token, fileSize, fileHelper): # pylint: disable=unused-argument @@ -251,7 +247,7 @@ def __doRPC(self, method): ) ) args = retVal["Value"] - self.__logRemoteQuery("RPC/%s" % method, args) + self.__logRemoteQuery(f"RPC/{method}", args) return self.__RPCCallFunction(method, args) def __RPCCallFunction(self, method, args): @@ -263,19 +259,19 @@ def __RPCCallFunction(self, method, args): :return: S_OK/S_ERROR """ - realMethod = "export_%s" % method - gLogger.debug("RPC to %s" % realMethod) + realMethod = f"export_{method}" + gLogger.debug(f"RPC to {realMethod}") try: # Get the method we are trying to call oMethod = getattr(self, realMethod) except Exception: - return S_ERROR("Unknown method %s" % method) + return S_ERROR(f"Unknown method {method}") # Check if the client sends correct arguments dRetVal = self.__checkExpectedArgumentTypes(method, args) if not dRetVal["OK"]: return dRetVal # Lock the method with Semaphore to avoid too many calls at the same time - self.__lockManager.lock("RPC/%s" % method) + self.__lockManager.lock(f"RPC/{method}") # 18.02.19 WARNING CHRIS # The line below adds the current transportID to the message broker # First of all, I do not see why it is doing so. @@ -297,12 +293,12 @@ def __RPCCallFunction(self, method, args): return uReturnValue finally: # Unlock method - self.__lockManager.unlock("RPC/%s" % method) + self.__lockManager.unlock(f"RPC/{method}") # 18.02.19 WARNING CHRIS # See comment above # self.__msgBroker.removeTransport(self.__trid, closeTransport=False) except Exception as e: - gLogger.exception("Uncaught exception when serving RPC", "Function %s" % method, lException=e) + gLogger.exception("Uncaught exception when serving RPC", f"Function {method}", lException=e) return S_ERROR(f"Server error while serving {method}: {str(e)}") def __checkExpectedArgumentTypes(self, method, args): @@ -315,11 +311,11 @@ def __checkExpectedArgumentTypes(self, method, args): :param args: Arguments to check :return: S_OK/S_ERROR """ - sListName = "types_%s" % method + sListName = f"types_{method}" try: oTypesList = getattr(self, sListName) except Exception: - gLogger.error("There's no types info for method", "export_%s" % method) + gLogger.error("There's no types info for method", f"export_{method}") return S_ERROR( "Handler error for server {} while processing method {}".format( self.serviceInfoDict["serviceName"], method @@ -349,7 +345,7 @@ def __checkExpectedArgumentTypes(self, method, args): if len(args) < len(oTypesList): return S_ERROR(f"Function {method} expects at least {len(oTypesList)} arguments") except Exception as v: - sError = "Error in parameter check: %s" % str(v) + sError = f"Error in parameter check: {str(v)}" gLogger.exception(sError) return S_ERROR(sError) return S_OK() @@ -377,23 +373,23 @@ def __doConnection(self, methodName): return self._rh_executeConnectionCallback(methodName, args) def _rh_executeConnectionCallback(self, methodName, args=False): - self.__logRemoteQuery("Connection/%s" % methodName, args) + self.__logRemoteQuery(f"Connection/{methodName}", args) if methodName not in RequestHandler.__connectionCallbackTypes: - return S_ERROR("Invalid connection method %s" % methodName) + return S_ERROR(f"Invalid connection method {methodName}") cbTypes = RequestHandler.__connectionCallbackTypes[methodName] if args: if len(args) != len(cbTypes): - return S_ERROR("Expected %s arguments" % len(cbTypes)) + return S_ERROR(f"Expected {len(cbTypes)} arguments") for i in range(len(cbTypes)): if not isinstance(args[i], cbTypes[i]): - return S_ERROR("Invalid type for argument %s" % i) + return S_ERROR(f"Invalid type for argument {i}") self.__trPool.associateData(self.__trid, "connectData", args) if not args: args = self.__trPool.getAssociatedData(self.__trid, "connectData") - realMethod = "conn_%s" % methodName - gLogger.debug("Callback to %s" % realMethod) + realMethod = f"conn_{methodName}" + gLogger.debug(f"Callback to {realMethod}") try: oMethod = getattr(self, realMethod) except Exception: @@ -406,20 +402,20 @@ def _rh_executeConnectionCallback(self, methodName, args=False): uReturnValue = oMethod(self.__trid) return uReturnValue except Exception as e: - gLogger.exception("Uncaught exception when serving Connect", "Function %s" % realMethod, lException=e) + gLogger.exception("Uncaught exception when serving Connect", f"Function {realMethod}", lException=e) return S_ERROR(f"Server error while serving {methodName}: {str(e)}") def _rh_executeMessageCallback(self, msgObj): msgName = msgObj.getName() if not self.__msgBroker.getMsgFactory().messageExists(self.__svcName, msgName): - return S_ERROR("Unknown message %s" % msgName) - methodName = "msg_%s" % msgName - self.__logRemoteQuery("Message/%s" % methodName, msgObj.dumpAttrs()) + return S_ERROR(f"Unknown message {msgName}") + methodName = f"msg_{msgName}" + self.__logRemoteQuery(f"Message/{methodName}", msgObj.dumpAttrs()) startTime = time.time() try: oMethod = getattr(self, methodName) except Exception: - return S_ERROR("Handler function for message %s does not exist!" % msgName) + return S_ERROR(f"Handler function for message {msgName} does not exist!") self.__lockManager.lock(methodName) try: try: @@ -431,7 +427,7 @@ def _rh_executeMessageCallback(self, msgObj): self.__lockManager.unlock(methodName) if not isReturnStructure(uReturnValue): gLogger.error("Message does not return a S_OK/S_ERROR", msgName) - uReturnValue = S_ERROR("Message %s does not return a S_OK/S_ERROR" % msgName) + uReturnValue = S_ERROR(f"Message {msgName} does not return a S_OK/S_ERROR") elapsedTime = time.time() - startTime self.__logRemoteQueryResponse(uReturnValue, elapsedTime) return S_OK([uReturnValue, elapsedTime]) @@ -478,7 +474,7 @@ def __logRemoteQueryResponse(self, retVal, elapsedTime): if retVal["OK"]: argsString = "OK" else: - argsString = "ERROR: %s" % retVal["Message"] + argsString = f"ERROR: {retVal['Message']}" if "CallStack" in retVal: argsString += "\n" + "".join(retVal["CallStack"]) gLogger.notice( @@ -504,7 +500,7 @@ def export_ping(self): startTime = self.serviceInfoDict["serviceStartTime"] dInfo["service start time"] = self.serviceInfoDict["serviceStartTime"] serviceUptime = datetime.datetime.utcnow() - startTime - dInfo["service uptime"] = serviceUptime.days * 3600 + serviceUptime.seconds + dInfo["service uptime"] = int(serviceUptime.total_seconds()) # Load average dInfo["load"] = " ".join([str(lx) for lx in os.getloadavg()]) dInfo["name"] = self.serviceInfoDict["serviceName"] @@ -552,7 +548,7 @@ def export_refreshConfiguration(fromMaster): """ Force refreshing the configuration data - :param bool fromMaster: flag to refresh from the master configuration service + :param bool fromMaster: flag to refresh from the controller configuration service """ return gConfig.forceRefresh(fromMaster=fromMaster) @@ -595,11 +591,7 @@ def srv_getCSOption(cls, optionName, defaultValue=False): return gConfig.getValue(optionName, defaultValue) for csPath in cls.__srvInfoDict["csPaths"]: result = gConfig.getOption( - "%s/%s" - % ( - csPath, - optionName, - ), + f"{csPath}/{optionName}", defaultValue, ) if result["OK"]: diff --git a/src/DIRAC/Core/DISET/ServiceReactor.py b/src/DIRAC/Core/DISET/ServiceReactor.py index 76aead02d43..2f3bd76ced1 100644 --- a/src/DIRAC/Core/DISET/ServiceReactor.py +++ b/src/DIRAC/Core/DISET/ServiceReactor.py @@ -14,23 +14,18 @@ must inherit from the base class RequestHandler """ + import time import datetime -import socket +import selectors import signal import os import sys import multiprocessing -try: - import selectors -except ImportError: - import selectors2 as selectors - from DIRAC import gLogger, S_OK, S_ERROR from DIRAC.Core.DISET.private.Service import Service from DIRAC.Core.DISET.private.GatewayService import GatewayService -from DIRAC.Core.DISET.RequestHandler import RequestHandler from DIRAC.Core.Base.private.ModuleLoader import ModuleLoader from DIRAC.Core.DISET.private.Protocols import gProtocolDict from DIRAC.ConfigurationSystem.Client.Helpers import Registry @@ -42,7 +37,6 @@ class ServiceReactor: - __transportExtraKeywords = { "SSLSessionTimeout": False, "IgnoreCRLs": False, @@ -53,7 +47,7 @@ class ServiceReactor: def __init__(self): self.__services = {} self.__alive = True - self.__loader = ModuleLoader("Service", PathFinder.getServiceSection, RequestHandler, moduleSuffix="Handler") + self.__loader = ModuleLoader("Service", PathFinder.getServiceSection, moduleSuffix="Handler") self.__maxFD = 0 self.__listeningConnections = {} self.__stats = ReactorStats() @@ -77,7 +71,7 @@ def initialize(self, servicesList): # Loop again to include the GW in case there is one (included in the __init__) for serviceName in self.__services: - gLogger.info("Initializing %s" % serviceName) + gLogger.info(f"Initializing {serviceName}") result = self.__services[serviceName].initialize() if not result["OK"]: return result @@ -100,7 +94,7 @@ def __createListeners(self): protocol = svcCfg.getProtocol() port = svcCfg.getPort() if not port: - return S_ERROR("No port defined for service %s" % serviceName) + return S_ERROR(f"No port defined for service {serviceName}") if protocol not in gProtocolDict: return S_ERROR(f"Protocol {protocol} is not known for service {serviceName}") self.__listeningConnections[serviceName] = {"port": port, "protocol": protocol} @@ -114,13 +108,11 @@ def __createListeners(self): if kw == "timeout": value = int(value) transportArgs[kw] = value - gLogger.verbose("Initializing %s transport" % protocol, svcCfg.getURL()) + gLogger.verbose(f"Initializing {protocol} transport", svcCfg.getURL()) transport = gProtocolDict[protocol]["transport"](("", port), bServerMode=True, **transportArgs) retVal = transport.initAsServer() if not retVal["OK"]: - return S_ERROR( - "Cannot start listening connection for service {}: {}".format(serviceName, retVal["Message"]) - ) + return S_ERROR(f"Cannot start listening connection for service {serviceName}: {retVal['Message']}") self.__listeningConnections[serviceName]["transport"] = transport self.__listeningConnections[serviceName]["socket"] = transport.getSocket() return S_OK() @@ -150,7 +142,7 @@ def serve(self): self.__closeListeningConnections() return result for svcName in self.__listeningConnections: - gLogger.always("Listening at %s" % self.__services[svcName].getConfig().getURL()) + gLogger.always(f"Listening at {self.__services[svcName].getConfig().getURL()}") isMultiProcessingAllowed = False for svcName in self.__listeningConnections: @@ -209,6 +201,7 @@ def __acceptIncomingConnection(self, svcName=False): """ sel = self.__getListeningSelector(svcName) while self.__alive: + clientTransport = None try: events = sel.select(timeout=10) if len(events) == 0: @@ -227,7 +220,7 @@ def __acceptIncomingConnection(self, svcName=False): # Is it banned? clientIP = clientTransport.getRemoteAddress()[0] if clientIP in Registry.getBannedIPs(): - gLogger.warn("Client connected from banned ip %s" % clientIP) + gLogger.warn(f"Client connected from banned ip {clientIP}") clientTransport.close() continue # Handle connection diff --git a/src/DIRAC/Core/DISET/TransferClient.py b/src/DIRAC/Core/DISET/TransferClient.py index 64a928c47a5..b9a37623950 100755 --- a/src/DIRAC/Core/DISET/TransferClient.py +++ b/src/DIRAC/Core/DISET/TransferClient.py @@ -38,7 +38,7 @@ def _sendTransferHeader(self, actionName, fileInfo): return S_OK((trid, transport)) except Exception as e: self._disconnect(trid) - return S_ERROR("Cound not request transfer: %s" % str(e)) + return S_ERROR(f"Cound not request transfer: {str(e)}") def sendFile(self, filename, fileId, token=""): """ @@ -71,6 +71,7 @@ def sendFile(self, filename, fileId, token=""): retVal = transport.receiveData() return retVal finally: + fileHelper.oFile.close() self._disconnect(trid) def receiveFile(self, filename, fileId, token=""): @@ -136,9 +137,9 @@ def sendBulk(self, fileList, bulkId, token="", compress=True, bulkSize=-1, onthe if bogusEntries: return S_ERROR("Some files or directories don't exist :\n\t%s" % "\n\t".join(bogusEntries)) if compress: - bulkId = "%s.tar.bz2" % bulkId + bulkId = f"{bulkId}.tar.bz2" else: - bulkId = "%s.tar" % bulkId + bulkId = f"{bulkId}.tar" retVal = self._sendTransferHeader("BulkFromClient", (bulkId, token, bulkSize)) if not retVal["OK"]: return retVal @@ -168,11 +169,11 @@ def receiveBulk(self, destDir, bulkId, token="", compress=True): :return: S_OK/S_ERROR """ if not os.path.isdir(destDir): - return S_ERROR("%s is not a directory for bulk receival" % destDir) + return S_ERROR(f"{destDir} is not a directory for bulk receival") if compress: - bulkId = "%s.tar.bz2" % bulkId + bulkId = f"{bulkId}.tar.bz2" else: - bulkId = "%s.tar" % bulkId + bulkId = f"{bulkId}.tar" retVal = self._sendTransferHeader("BulkToClient", (bulkId, token)) if not retVal["OK"]: return retVal @@ -200,9 +201,9 @@ def listBulk(self, bulkId, token="", compress=True): :return: S_OK/S_ERROR """ if compress: - bulkId = "%s.tar.bz2" % bulkId + bulkId = f"{bulkId}.tar.bz2" else: - bulkId = "%s.tar" % bulkId + bulkId = f"{bulkId}.tar" trid = None retVal = self._sendTransferHeader("ListBulk", (bulkId, token)) if not retVal["OK"]: diff --git a/src/DIRAC/Core/DISET/private/BaseClient.py b/src/DIRAC/Core/DISET/private/BaseClient.py index fbb693906ef..c1f2da0147d 100755 --- a/src/DIRAC/Core/DISET/private/BaseClient.py +++ b/src/DIRAC/Core/DISET/private/BaseClient.py @@ -84,7 +84,6 @@ def __init__(self, serviceName, **kwargs): self.__retryCounter = 1 self.__bannedUrls = [] for initFunc in ( - self.__discoverSetup, self.__discoverVO, self.__discoverTimeout, self.__discoverURL, @@ -117,24 +116,6 @@ def getServiceName(self): """ return self._serviceName - def __discoverSetup(self): - """Discover which setup to use and stores it in self.setup - The setup is looked for: - * kwargs of the constructor (see KW_SETUP) - * the ThreadConfig - * in the CS /DIRAC/Setup - * default to 'Test' - - :return: S_OK()/S_ERROR() - """ - if self.KW_SETUP in self.kwargs and self.kwargs[self.KW_SETUP]: - self.setup = str(self.kwargs[self.KW_SETUP]) - else: - self.setup = self.__threadConfig.getSetup() - if not self.setup: - self.setup = gConfig.getValue("/DIRAC/Setup", "Test") - return S_OK() - def __discoverVO(self): """Discover which VO to use and stores it in self.vo The VO is looked for: @@ -253,7 +234,11 @@ def __discoverExtraCredentials(self): :return: S_OK()/S_ERROR() """ # which extra credentials to use? - self.__extraCredentials = self.VAL_EXTRA_CREDENTIALS_HOST if self.__useCertificates else "" + self.__extraCredentials = ( + self.VAL_EXTRA_CREDENTIALS_HOST + if (self.__useCertificates and not self.kwargs.get(self.KW_PROXY_LOCATION)) + else "" + ) if self.KW_EXTRA_CREDENTIALS in self.kwargs: self.__extraCredentials = self.kwargs[self.KW_EXTRA_CREDENTIALS] @@ -297,7 +282,7 @@ def __findServiceURL(self): # Load the Gateways URLs for the current site Name gatewayURL = False if not self.kwargs.get(self.KW_IGNORE_GATEWAYS): - dRetVal = gConfig.getOption("/DIRAC/Gateways/%s" % DIRAC.siteName()) + dRetVal = gConfig.getOption(f"/DIRAC/Gateways/{DIRAC.siteName()}") if dRetVal["OK"]: rawGatewayURL = List.randomize(List.fromChar(dRetVal["Value"], ","))[0] gatewayURL = "/".join(rawGatewayURL.split("/")[:3]) @@ -306,7 +291,7 @@ def __findServiceURL(self): # we just return this one. # If we have to use a gateway, we just replace the server name in it for protocol in gProtocolDict: - if self._destinationSrv.find("%s://" % protocol) == 0: + if self._destinationSrv.find(f"{protocol}://") == 0: gLogger.debug("Already given a valid url", self._destinationSrv) if not gatewayURL: return S_OK(self._destinationSrv) @@ -322,16 +307,16 @@ def __findServiceURL(self): # We extract the list of URLs from the CS (System/URLs/Component) try: - urls = getServiceURL(self._destinationSrv, setup=self.setup) + urls = getServiceURL(self._destinationSrv) except Exception as e: - return S_ERROR(f"Cannot get URL for {self._destinationSrv} in setup {self.setup}: {repr(e)}") + return S_ERROR(f"Cannot get URL for {self._destinationSrv} : {repr(e)}") if not urls: - return S_ERROR("URL for service %s not found" % self._destinationSrv) + return S_ERROR(f"URL for service {self._destinationSrv} not found") failoverUrls = [] # Try if there are some failover URLs to use as last resort try: - failoverUrlsStr = getServiceFailoverURL(self._destinationSrv, setup=self.setup) + failoverUrlsStr = getServiceFailoverURL(self._destinationSrv) if failoverUrlsStr: failoverUrls = failoverUrlsStr.split(",") except Exception: @@ -351,7 +336,7 @@ def __findServiceURL(self): # we have host which is not accessible. We remove that host from the list. # We only remove if we have more than one instance for i in self.__bannedUrls: - gLogger.debug("Removing banned URL", "%s" % i) + gLogger.debug("Removing banned URL", f"{i}") urlsList.remove(i) # Take the first URL from the list @@ -423,16 +408,12 @@ def __checkThreadID(self): if not self.__allowedThreadID: self.__allowedThreadID = cThID elif cThID != self.__allowedThreadID: - msgTxt = """ + msgTxt = f""" =======DISET client thread safety error======================== -Client {} -can only run on thread {} -and this is thread {} -===============================================================""".format( - str(self), - self.__allowedThreadID, - cThID, - ) +Client {str(self)} +can only run on thread {self.__allowedThreadID} +and this is thread {cThID} +===============================================================""" gLogger.error("DISET client thread safety error", msgTxt) # raise Exception( msgTxt ) @@ -465,7 +446,7 @@ def _connect(self): if self.__enableThreadCheck: self.__checkThreadID() - gLogger.debug("Trying to connect to: %s" % self.serviceURL) + gLogger.debug(f"Trying to connect to: {self.serviceURL}") try: # Calls the transport method of the apropriate protocol. # self.__URLTuple[1:3] = [server name, port, System/Component] @@ -475,9 +456,7 @@ def _connect(self): retVal = transport.initAsClient() # We try at most __nbOfRetry each URLs if not retVal["OK"]: - gLogger.warn( - "Issue getting socket:", "{} : {} : {}".format(transport, self.__URLTuple, retVal["Message"]) - ) + gLogger.warn("Issue getting socket:", f"{transport} : {self.__URLTuple} : {retVal['Message']}") # We try at most __nbOfRetry each URLs if self.__retry < self.__nbOfRetry * self.__nbOfUrls - 1: # Recompose the URL (why not using self.serviceURL ? ) @@ -489,7 +468,7 @@ def _connect(self): ) # Add the url to the list of banned URLs if it is not already there. (Can it happen ? I don't think so) if url not in self.__bannedUrls: - gLogger.warn("Non-responding URL temporarily banned", "%s" % url) + gLogger.warn("Non-responding URL temporarily banned", f"{url}") self.__bannedUrls += [url] # Increment the retry counter self.__retry += 1 @@ -509,7 +488,7 @@ def _connect(self): self.__retryCounter += 1 # we run only one service! In that case we increase the retry delay. self.__retryDelay = 3.0 / self.__nbOfUrls if self.__nbOfUrls > 1 else 2 - gLogger.info("Waiting %f seconds before retry all service(s)" % self.__retryDelay) + gLogger.info(f"Waiting {self.__retryDelay:f} seconds before retry all service(s)") time.sleep(self.__retryDelay) # rediscover the URL self.__discoverURL() @@ -521,7 +500,7 @@ def _connect(self): gLogger.exception(lException=True, lExcInfo=True) return S_ERROR(f"Can't connect to {self.serviceURL}: {repr(e)}") # We add the connection to the transport pool - gLogger.debug("Connected to: %s" % self.serviceURL) + gLogger.debug(f"Connected to: {self.serviceURL}") trid = getGlobalTransportPool().add(transport) return S_OK((trid, transport)) @@ -566,7 +545,7 @@ def _proposeAction(self, transport, action): """ if not self.__initStatus["OK"]: return self.__initStatus - stConnectionInfo = ((self.__URLTuple[3], self.setup, self.vo), action, self.__extraCredentials, DIRAC.version) + stConnectionInfo = ((self.__URLTuple[3], "None", self.vo), action, self.__extraCredentials, DIRAC.version) # Send the connection info and get the answer back retVal = transport.sendData(S_OK(BaseClient._serializeStConnectionInfo(stConnectionInfo))) @@ -658,8 +637,5 @@ def _getBaseStub(self): def __bool__(self): return True - # For Python 2 compatibility - __nonzero__ = __bool__ - def __str__(self): return f"" diff --git a/src/DIRAC/Core/DISET/private/FileHelper.py b/src/DIRAC/Core/DISET/private/FileHelper.py index fba2f7922b8..6258b141600 100755 --- a/src/DIRAC/Core/DISET/private/FileHelper.py +++ b/src/DIRAC/Core/DISET/private/FileHelper.py @@ -5,7 +5,7 @@ import tempfile import threading -from io import StringIO +from io import StringIO, BytesIO from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR from DIRAC.FrameworkSystem.Client.Logger import gLogger @@ -14,7 +14,6 @@ class FileHelper: - __validDirections = ("toClient", "fromClient", "receive", "send") __directionsMapping = {"toClient": "send", "fromClient": "receive"} @@ -131,14 +130,14 @@ def errorInTransmission(self): def networkToString(self, maxFileSize=0): """Receive the input from a DISET client and return it as a string""" - stringIO = StringIO() - result = self.networkToDataSink(stringIO, maxFileSize=maxFileSize) + bytesIO = BytesIO() + result = self.networkToDataSink(bytesIO, maxFileSize=maxFileSize) if not result["OK"]: return result - return S_OK(stringIO.getvalue()) + return S_OK(bytesIO.getvalue()) def networkToFD(self, iFD, maxFileSize=0): - dataSink = os.fdopen(iFD, "w") + dataSink = os.fdopen(iFD, "wb") try: return self.networkToDataSink(dataSink, maxFileSize=maxFileSize) finally: @@ -149,7 +148,7 @@ def networkToFD(self, iFD, maxFileSize=0): def networkToDataSink(self, dataSink, maxFileSize=0): if "write" not in dir(dataSink): - return S_ERROR("%s data sink object does not have a write method" % str(dataSink)) + return S_ERROR(f"{str(dataSink)} data sink object does not have a write method") self.__oMD5 = hashlib.md5() self.bReceivedEOF = False self.bErrorInMD5 = False @@ -165,7 +164,7 @@ def networkToDataSink(self, dataSink, maxFileSize=0): while not self.receivedEOF(): if maxFileSize > 0 and receivedBytes > maxFileSize: self.sendError("Exceeded maximum file size") - return S_ERROR("Received file exceeded maximum size of %s bytes" % (maxFileSize)) + return S_ERROR(f"Received file exceeded maximum size of {maxFileSize} bytes") dataSink.write(strBuffer) result = self.receiveData(maxBufferSize=(maxFileSize - len(strBuffer))) if not result["OK"]: @@ -205,7 +204,7 @@ def stringToNetwork(self, stringVal): ioffset += iPacketSize self.sendEOF() except Exception as e: - return S_ERROR("Error while sending string: %s" % str(e)) + return S_ERROR(f"Error while sending string: {str(e)}") try: stringIO.close() except Exception: @@ -231,7 +230,7 @@ def FDToNetwork(self, iFD): self.sendEOF() except Exception as e: gLogger.exception("Error while sending file") - return S_ERROR("Error while sending file: %s" % str(e)) + return S_ERROR(f"Error while sending file: {str(e)}") self.__fileBytes = sentBytes return S_OK() @@ -244,7 +243,7 @@ def BufferToNetwork(self, stringToSend): def DataSourceToNetwork(self, dataSource): if "read" not in dir(dataSource): - return S_ERROR("%s data source object does not have a read method" % str(dataSource)) + return S_ERROR(f"{str(dataSource)} data source object does not have a read method") self.__oMD5 = hashlib.md5() iPacketSize = self.packetSize self.__fileBytes = 0 @@ -263,7 +262,7 @@ def DataSourceToNetwork(self, dataSource): self.sendEOF() except Exception as e: gLogger.exception("Error while sending file") - return S_ERROR("Error while sending file: %s" % str(e)) + return S_ERROR(f"Error while sending file: {str(e)}") self.__fileBytes = sentBytes return S_OK() @@ -273,7 +272,7 @@ def getFileDescriptor(self, uFile, sFileMode): try: self.oFile = open(uFile, sFileMode) except OSError: - return S_ERROR("%s can't be opened" % uFile) + return S_ERROR(f"{uFile} can't be opened") iFD = self.oFile.fileno() elif isinstance(uFile, file_types): iFD = uFile.fileno() @@ -281,7 +280,7 @@ def getFileDescriptor(self, uFile, sFileMode): iFD = uFile closeAfter = False else: - return S_ERROR("%s is not a valid file." % uFile) + return S_ERROR(f"{uFile} is not a valid file.") result = S_OK(iFD) result["closeAfterUse"] = closeAfter return result @@ -292,7 +291,7 @@ def getDataSink(self, uFile): try: oFile = open(uFile, "wb") except OSError: - return S_ERROR("%s can't be opened" % uFile) + return S_ERROR(f"{uFile} can't be opened") elif isinstance(uFile, file_types): oFile = uFile closeAfter = False @@ -303,7 +302,7 @@ def getDataSink(self, uFile): oFile = uFile closeAfter = False else: - return S_ERROR("%s is not a valid file." % uFile) + return S_ERROR(f"{uFile} is not a valid file.") result = S_OK(oFile) result["closeAfterUse"] = closeAfter return result @@ -312,7 +311,7 @@ def __createTar(self, fileList, wPipe, compress, autoClose=True): if "write" in dir(wPipe): filePipe = wPipe else: - filePipe = os.fdopen(wPipe, "w") + filePipe = os.fdopen(wPipe, "wb") tarMode = "w|" if compress: tarMode = "w|bz2" @@ -331,12 +330,12 @@ def bulkToNetwork(self, fileList, compress=True, onthefly=True): try: filePipe, filePath = tempfile.mkstemp() except Exception as e: - return S_ERROR("Can't create temporary file to pregenerate the bulk: %s" % str(e)) + return S_ERROR(f"Can't create temporary file to pregenerate the bulk: {str(e)}") self.__createTar(fileList, filePipe, compress) try: fo = open(filePath, "rb") except Exception as e: - return S_ERROR("Can't read pregenerated bulk: %s" % str(e)) + return S_ERROR(f"Can't read pregenerated bulk: {str(e)}") result = self.DataSourceToNetwork(fo) try: fo.close() @@ -356,7 +355,7 @@ def bulkToNetwork(self, fileList, compress=True, onthefly=True): return response def __extractTar(self, destDir, rPipe, compress): - filePipe = os.fdopen(rPipe, "r") + filePipe = os.fdopen(rPipe, "rb") tarMode = "r|*" if compress: tarMode = "r|bz2" @@ -383,12 +382,12 @@ def networkToBulk(self, destDir, compress=True, maxFileSize=0): try: self.__extractTar(destDir, rPipe, compress) except Exception as e: - return S_ERROR("Error while extracting bulk: %s" % e) + return S_ERROR(f"Error while extracting bulk: {e}") thrd.join() return retList[0] def bulkListToNetwork(self, iFD, compress=True): - filePipe = os.fdopen(iFD, "r") + filePipe = os.fdopen(iFD, "rb") try: tarMode = "r|" if compress: @@ -400,8 +399,8 @@ def bulkListToNetwork(self, iFD, compress=True): filePipe.close() return S_OK(entries) except tarfile.ReadError as v: - return S_ERROR("Error reading bulk: %s" % str(v)) + return S_ERROR(f"Error reading bulk: {str(v)}") except tarfile.CompressionError as v: - return S_ERROR("Error in bulk compression setting: %s" % str(v)) + return S_ERROR(f"Error in bulk compression setting: {str(v)}") except Exception as v: - return S_ERROR("Error in listing bulk: %s" % str(v)) + return S_ERROR(f"Error in listing bulk: {str(v)}") diff --git a/src/DIRAC/Core/DISET/private/GatewayService.py b/src/DIRAC/Core/DISET/private/GatewayService.py index 4299919b295..de200a32645 100644 --- a/src/DIRAC/Core/DISET/private/GatewayService.py +++ b/src/DIRAC/Core/DISET/private/GatewayService.py @@ -66,8 +66,8 @@ def initialize(self): # Build the URLs self._url = self._cfg.getURL() if not self._url: - return S_ERROR("Could not build service URL for %s" % GatewayService.GATEWAY_NAME) - gLogger.verbose("Service URL is %s" % self._url) + return S_ERROR(f"Could not build service URL for {GatewayService.GATEWAY_NAME}") + gLogger.verbose(f"Service URL is {self._url}") # Load handler result = self._loadHandlerInit() if not result["OK"]: @@ -76,7 +76,7 @@ def initialize(self): # Discover Handler self._threadPool = ThreadPoolExecutor(max(0, self._cfg.getMaxThreads())) - self._msgBroker = MessageBroker("%sMSB" % GatewayService.GATEWAY_NAME, threadPool=self._threadPool) + self._msgBroker = MessageBroker(f"{GatewayService.GATEWAY_NAME}MSB", threadPool=self._threadPool) self._msgBroker.useMessageObjects(False) getGlobalMessageBroker().useMessageObjects(False) self._msgForwarder = MessageForwarder(self._msgBroker) @@ -121,7 +121,7 @@ def _receiveAndCheckProposal(self, trid): if not retVal["OK"]: gLogger.error( "Invalid action proposal", - "{} {}".format(self._createIdentityString(credDict, clientTransport), retVal["Message"]), + f"{self._createIdentityString(credDict, clientTransport)} {retVal['Message']}", ) return S_ERROR("Invalid action proposal") proposalTuple = retVal["Value"] @@ -146,11 +146,11 @@ def __getClientInitArgs(self, trid, proposalTuple): dP = self.__delegatedCredentials.get(cKey, 3600) idString = self._createIdentityString(credDict, clientTransport) if dP: - gLogger.verbose("Proxy for %s is cached" % idString) + gLogger.verbose(f"Proxy for {idString} is cached") return S_OK(dP) result = self.__requestDelegation(clientTransport, credDict) if not result["OK"]: - gLogger.warn("Could not get proxy for {}: {}".format(idString, result["Message"])) + gLogger.warn(f"Could not get proxy for {idString}: {result['Message']}") return result delChain = result["Value"] delegatedChain = delChain.dumpAllToString()["Value"] @@ -179,13 +179,13 @@ def __requestDelegation(self, clientTransport, credDict): retVal = S_ERROR("Server Error: Can't generate delegation request") clientTransport.sendData(retVal) return retVal - gLogger.info("Sending delegation request for %s" % delegationRequest.getSubjectDN()["Value"]) + gLogger.info(f"Sending delegation request for {delegationRequest.getSubjectDN()['Value']}") clientTransport.sendData(S_OK({"delegate": retVal["Value"]})) delegatedCertChain = clientTransport.receiveData() delegatedChain = X509Chain(keyObj=delegationRequest.getPKey()) retVal = delegatedChain.loadChainFromString(delegatedCertChain) if not retVal["OK"]: - retVal = S_ERROR("Error in receiving delegated proxy: %s" % retVal["Message"]) + retVal = S_ERROR(f"Error in receiving delegated proxy: {retVal['Message']}") clientTransport.sendData(retVal) return retVal return S_OK(delegatedChain) @@ -214,10 +214,10 @@ def _executeAction(self, trid, proposalTuple, clientInitArgs): retVal = clientTransport.receiveData() if not retVal["OK"]: gLogger.error("Error while receiving file description", retVal["Message"]) - clientTransport.sendData(S_ERROR("Error while receiving file description: %s" % retVal["Message"])) + clientTransport.sendData(S_ERROR(f"Error while receiving file description: {retVal['Message']}")) return if actionType == "FileTransfer": - gLogger.warn("Received a file transfer action from %s" % idString) + gLogger.warn(f"Received a file transfer action from {idString}") clientTransport.sendData(S_OK("Accepted")) retVal = self.__forwardFileTransferCall( targetService, clientInitArgs, actionMethod, retVal["Value"], clientTransport @@ -230,7 +230,7 @@ def _executeAction(self, trid, proposalTuple, clientInitArgs): retVal = self._msgForwarder.addClient(trid, targetService, clientInitArgs, retVal["Value"]) else: gLogger.warn(f"Received an invalid {actionType}/{actionMethod} action from {idString}") - retVal = S_ERROR("Unknown type of action (%s)" % actionType) + retVal = S_ERROR(f"Unknown type of action ({actionType})") # TODO: Send back the data? if "rpcStub" in retVal: retVal.pop("rpcStub") @@ -266,9 +266,9 @@ def __forwardFileTransferCall(self, targetService, clientInitArgs, method, param return S_ERROR("Transfer size is too big") # Forward queries try: - relayMethodObject = getattr(transferRelay, "forward%s" % method) + relayMethodObject = getattr(transferRelay, f"forward{method}") except Exception: - return S_ERROR("Cannot forward unknown method %s" % method) + return S_ERROR(f"Cannot forward unknown method {method}") result = relayMethodObject(cliFH, params) return result @@ -303,7 +303,7 @@ def getDataFromClient(self, clientFileHelper): return result data = sIO.getvalue() sIO.close() - self.infoMsg("Got %s bytes from client" % len(data)) + self.infoMsg(f"Got {len(data)} bytes from client") return S_OK(data) def sendDataToClient(self, clientFileHelper, dataToSend): @@ -312,11 +312,11 @@ def sendDataToClient(self, clientFileHelper, dataToSend): if not result["OK"]: self.errMsg("Could not send data to client", result["Message"]) return result - self.infoMsg("Sent %s bytes from client" % len(dataToSend)) + self.infoMsg(f"Sent {len(dataToSend)} bytes from client") return S_OK() def sendDataToService(self, srvMethod, params, data): - self.infoMsg("Sending header request to %s" % self.getDestinationService(), str(params)) + self.infoMsg(f"Sending header request to {self.getDestinationService()}", str(params)) result = self._sendTransferHeader(srvMethod, params) if not result["OK"]: self.errMsg("Could not send header", result["Message"]) @@ -330,13 +330,13 @@ def sendDataToService(self, srvMethod, params, data): self.errMsg("Could send data to server", result["Message"]) srvTransport.close() return result - self.infoMsg("Data sent to service (%s bytes)" % len(data)) + self.infoMsg(f"Data sent to service ({len(data)} bytes)") retVal = srvTransport.receiveData() srvTransport.close() return retVal def getDataFromService(self, srvMethod, params): - self.infoMsg("Sending header request to %s" % self.getDestinationService(), str(params)) + self.infoMsg(f"Sending header request to {self.getDestinationService()}", str(params)) result = self._sendTransferHeader(srvMethod, params) if not result["OK"]: self.errMsg("Could not send header", result["Message"]) @@ -354,7 +354,7 @@ def getDataFromService(self, srvMethod, params): return result dataReceived = sIO.getvalue() sIO.close() - self.infoMsg("Received %s bytes from service" % len(dataReceived)) + self.infoMsg(f"Received {len(dataReceived)} bytes from service") retVal = srvTransport.receiveData() srvTransport.close() if not retVal["OK"]: @@ -409,7 +409,7 @@ def forwardBulkToClient(self, clientFileHelper, params): def forwardListBulk(self, clientFileHelper, params): self.__currentMethod = "ListBulk" - self.infoMsg("Sending header request to %s" % self.getDestinationService(), str(params)) + self.infoMsg(f"Sending header request to {self.getDestinationService()}", str(params)) result = self._sendTransferHeader("ListBulk", params) if not result["OK"]: self.errMsg("Could not send header", result["Message"]) @@ -452,7 +452,7 @@ def __srvDisconnect(self, srvEndCli): cliTrid = self.__srvToCliTrid[srvEndCli.getTrid()] except IndexError: gLogger.exception("This shouldn't happen!") - gLogger.info("Service %s disconnected messaging connection" % self.__byClient[cliTrid]["srvName"]) + gLogger.info(f"Service {self.__byClient[cliTrid]['srvName']} disconnected messaging connection") self.__msgBroker.removeTransport(cliTrid) self.__removeClient(cliTrid) @@ -460,7 +460,7 @@ def cliDisconnect(self, cliTrid): if cliTrid not in self.__byClient: gLogger.fatal("This shouldn't happen!") return - gLogger.info("Client to %s disconnected messaging connection" % self.__byClient[cliTrid]["srvName"]) + gLogger.info(f"Client to {self.__byClient[cliTrid]['srvName']} disconnected messaging connection") self.__byClient[cliTrid]["srvEnd"].disconnect() self.__removeClient(cliTrid) @@ -477,7 +477,7 @@ def __removeClient(self, cliTrid): self.__inOutLock.release() def msgFromClient(self, cliTrid, msgObj): - gLogger.info("Message {} to {} service".format(msgObj.getName(), self.__byClient[cliTrid]["srvName"])) + gLogger.info(f"Message {msgObj.getName()} to {self.__byClient[cliTrid]['srvName']} service") result = self.__byClient[cliTrid]["srvEnd"].sendMessage(msgObj) return result @@ -487,5 +487,5 @@ def msgFromSrv(self, srvEndCli, msgObj): except Exception: gLogger.exception("This shouldn't happen") return S_ERROR("MsgFromSrv -> Mismatched srv2cli trid") - gLogger.info("Message {} from {} service".format(msgObj.getName(), self.__byClient[cliTrid]["srvName"])) + gLogger.info(f"Message {msgObj.getName()} from {self.__byClient[cliTrid]['srvName']} service") return self.__msgBroker.sendMessage(cliTrid, msgObj) diff --git a/src/DIRAC/Core/DISET/private/LockManager.py b/src/DIRAC/Core/DISET/private/LockManager.py index a3ab53f65e7..499d9c3255f 100755 --- a/src/DIRAC/Core/DISET/private/LockManager.py +++ b/src/DIRAC/Core/DISET/private/LockManager.py @@ -13,7 +13,7 @@ def __init__(self, iMaxThreads=None): def createLock(self, sLockName, iMaxThreads): if sLockName in self.dLocks: - raise RuntimeError("%s lock already exists" % sLockName) + raise RuntimeError(f"{sLockName} lock already exists") if iMaxThreads < 1: return self.dLocks[sLockName] = threading.Semaphore(iMaxThreads) diff --git a/src/DIRAC/Core/DISET/private/MessageBroker.py b/src/DIRAC/Core/DISET/private/MessageBroker.py index 5aca6ad33ae..23fd80a5a57 100644 --- a/src/DIRAC/Core/DISET/private/MessageBroker.py +++ b/src/DIRAC/Core/DISET/private/MessageBroker.py @@ -2,16 +2,10 @@ """ import threading import time -import select -import socket +import selectors from concurrent.futures import ThreadPoolExecutor -try: - import selectors -except ImportError: - import selectors2 as selectors - from DIRAC import gLogger, S_OK, S_ERROR from DIRAC.Core.DISET.private.TransportPool import getGlobalTransportPool from DIRAC.Core.Utilities.ReturnValues import isReturnStructure @@ -93,7 +87,7 @@ def addTransportId( return S_OK() tr = self.__trPool.get(trid) if not tr: - return S_ERROR("No transport with id %s registered" % trid) + return S_ERROR(f"No transport with id {trid} registered") self.__messageTransports[trid] = { "transport": tr, "svcName": svcName, @@ -140,6 +134,7 @@ def __listenAutoReceiveConnections(self): continue sel.register(mt["transport"].getSocket(), selectors.EVENT_READ, trid) if not sel.get_map(): + sel.close() self.__listeningForMessages = False return finally: @@ -154,6 +149,8 @@ def __listenAutoReceiveConnections(self): except Exception as e: gLogger.exception("Exception while selecting persistent connections", lException=e) continue + finally: + sel.close() for key, event in events: if event & selectors.EVENT_READ: @@ -173,13 +170,20 @@ def __receiveMsgDataAndQueue(self, trid): self.__log.debug(f"[trid {trid}] Received data: {str(result)}") # If error close transport and exit if not result["OK"]: - self.__log.debug("[trid {}] ERROR RCV DATA {}".format(trid, result["Message"])) + self.__log.debug(f"[trid {trid}] ERROR RCV DATA {result['Message']}") gLogger.warn( "Error while receiving message", - "from {} : {}".format(self.__trPool.get(trid).getFormattedCredentials(), result["Message"]), + f"from {self.__trPool.get(trid).getFormattedCredentials()} : {result['Message']}", ) return self.removeTransport(trid) - self.__threadPool.submit(self.__processIncomingData, trid, result) + + def err_handler(res): + err = res.exception() + if err: + self.__log.exception("Exception in receiveMsgDataAndQueue thread", lException=err) + + future = self.__threadPool.submit(self.__processIncomingData, trid, result) + future.add_done_callback(err_handler) return S_OK() def __processIncomingData(self, trid, receivedResult): @@ -191,7 +195,7 @@ def __processIncomingData(self, trid, receivedResult): try: idleRead = self.__messageTransports[trid]["idleRead"] except KeyError: - return S_ERROR("Transport %s unknown" % trid) + return S_ERROR(f"Transport {trid} unknown") finally: self.__trInOutLock.release() if idleRead: @@ -199,7 +203,7 @@ def __processIncomingData(self, trid, receivedResult): gLogger.fatal("OOOops. Idle read has returned data!") return S_OK() if not receivedResult["Value"]: - self.__log.debug("Transport %s closed connection" % trid) + self.__log.debug(f"Transport {trid} closed connection") return self.removeTransport(trid) # This is a message req/resp msg = receivedResult["Value"] @@ -227,22 +231,22 @@ def __processIncomingRequest(self, trid, msg): try: rcvCB = self.__messageTransports[trid]["cbReceiveMessage"] except KeyError: - return S_ERROR("Transport %s unknown" % trid) + return S_ERROR(f"Transport {trid} unknown") finally: self.__trInOutLock.release() if not rcvCB: - gLogger.fatal("Transport %s does not have a callback defined and a message arrived!" % trid) + gLogger.fatal(f"Transport {trid} does not have a callback defined and a message arrived!") return S_ERROR("No message was expected in for this transport") # Check message has id and name for requiredField in ["name"]: if requiredField not in msg: gLogger.error("Message does not have required field", requiredField) - return S_ERROR("Message does not have %s" % requiredField) + return S_ERROR(f"Message does not have {requiredField}") # Load message if "attrs" in msg: attrs = msg["attrs"] if not isinstance(attrs, (tuple, list)): - return S_ERROR("Message args has to be a tuple or a list, not %s" % type(attrs)) + return S_ERROR(f"Message args has to be a tuple or a list, not {type(attrs)}") else: attrs = None # Do we "unpack" or do we send the raw data to the callback? @@ -264,15 +268,15 @@ def __processIncomingRequest(self, trid, msg): return result except Exception as e: # Whoops. Show exception and return - gLogger.exception("Exception while processing message %s" % msg["name"], lException=e) - return S_ERROR("Exception while processing message {}: {}".format(msg["name"], str(e))) + gLogger.exception(f"Exception while processing message {msg['name']}", lException=e) + return S_ERROR(f"Exception while processing message {msg['name']}: {str(e)}") def __processIncomingResponse(self, trid, msg): # This is a message response for requiredField in ("id", "result"): if requiredField not in msg: gLogger.error("Message does not have required field", requiredField) - return S_ERROR("Message does not have %s" % requiredField) + return S_ERROR(f"Message does not have {requiredField}") if not isReturnStructure(msg["result"]): return S_ERROR("Message response did not return a result structure") return self.__notifyCallback(msg["id"], msg["result"]) @@ -293,7 +297,7 @@ def sendMessage(self, trid, msgObj): def __sendMessage(self, trid, msgObj): if not self.__trPool.exists(trid): - return S_ERROR("Not transport with id %s defined for messaging" % trid) + return S_ERROR(f"Not transport with id {trid} defined for messaging") msg = {"request": True, "name": msgObj.getName()} attrs = msgObj.dumpAttrs()["Value"] diff --git a/src/DIRAC/Core/DISET/private/MessageFactory.py b/src/DIRAC/Core/DISET/private/MessageFactory.py index dafddbbd5e9..f3f0ddfa3d4 100644 --- a/src/DIRAC/Core/DISET/private/MessageFactory.py +++ b/src/DIRAC/Core/DISET/private/MessageFactory.py @@ -42,12 +42,12 @@ def __loadHandler(self, serviceName): # TODO: Load handlers as the Service does (1. CS 2. SysNameSystem/Service/servNameHandler.py) sL = List.fromChar(serviceName, "/") if len(sL) != 2: - return S_ERROR("Service name is not valid: %s" % serviceName) + return S_ERROR(f"Service name is not valid: {serviceName}") sysName = sL[0] - svcHandlerName = "%sHandler" % sL[1] - loadedObjs = loadObjects("%sSystem/Service" % sysName, reFilter=re.compile(r"^%s\.py$" % svcHandlerName)) + svcHandlerName = f"{sL[1]}Handler" + loadedObjs = loadObjects(f"{sysName}System/Service", reFilter=re.compile(r"^%s\.py$" % svcHandlerName)) if svcHandlerName not in loadedObjs: - return S_ERROR("Could not find %s for getting messages definition" % serviceName) + return S_ERROR(f"Could not find {serviceName} for getting messages definition") return S_OK(loadedObjs[svcHandlerName]) def __loadMessagesFromService(self, serviceName): @@ -66,7 +66,7 @@ def __loadMessagesFromService(self, serviceName): return result msgDefs = result["Value"] if not msgDefs: - return S_ERROR("%s does not have messages defined" % serviceName) + return S_ERROR(f"{serviceName} does not have messages defined") self.__definitions[serviceName] = msgDefs return S_OK() @@ -83,7 +83,7 @@ def __loadMessagesForAncestry(self, handlerClass): return S_OK(finalDefs) msgDefs = getattr(handlerClass, "MSG_DEFINITIONS") if not isinstance(msgDefs, dict): - return S_ERROR("Message definitions for service %s is not a dict" % handlerClass.__name__) + return S_ERROR(f"Message definitions for service {handlerClass.__name__} is not a dict") for msgName in msgDefs: msgDefDict = msgDefs[msgName] if not isinstance(msgDefDict, dict): @@ -93,7 +93,6 @@ def __loadMessagesForAncestry(self, handlerClass): class Message: - DEFAULTWAITFORACK = False def __init__(self, msgName, msgDefDict): @@ -155,7 +154,7 @@ def dumpAttrs(self): try: return S_OK([self.__waitForAck, [self.__values[k] for k in self.__order]]) except Exception as e: - return S_ERROR("Could not dump message: %s doesn't have a value" % e) + return S_ERROR(f"Could not dump message: {e} doesn't have a value") def loadAttrs(self, data): if not isinstance(data, (list, tuple)) and len(data) != 2: @@ -177,15 +176,15 @@ def loadAttrs(self, data): return S_OK() def __str__(self): - msgStr = [" 5: - v = "%s..." % v[:5] + v = f"{v[:5]}..." msgStr.append(f"{k}->{v}") else: - msgStr.append("%s" % k) + msgStr.append(f"{k}") if self.isOK(): msgStr.append(") OK>") else: @@ -214,7 +213,7 @@ def __getattr__(self, k): if k not in self.__values: if k not in self.__fDef: raise AttributeError(f"No {k} attribute for message {self.__name}") - raise AttributeError("%s has no value" % k) + raise AttributeError(f"{k} has no value") return self.__values[k] diff --git a/src/DIRAC/Core/DISET/private/Service.py b/src/DIRAC/Core/DISET/private/Service.py index ea2d0064708..25b2058900c 100644 --- a/src/DIRAC/Core/DISET/private/Service.py +++ b/src/DIRAC/Core/DISET/private/Service.py @@ -33,7 +33,6 @@ class Service: - SVC_VALID_ACTIONS = {"RPC": "export", "FileTransfer": "transfer", "Message": "msg", "Connection": "Message"} SVC_SECLOG_CLIENT = SecurityLogClient() @@ -59,7 +58,7 @@ def __init__(self, serviceData): self._standalone = serviceData["standalone"] self.__monitorLastStatsUpdate = time.time() self._stats = {"queries": 0, "connections": 0} - self._authMgr = AuthManager("%s/Authorization" % PathFinder.getServiceSection(serviceData["loadName"])) + self._authMgr = AuthManager(f"{PathFinder.getServiceSection(serviceData['loadName'])}/Authorization") self._transportPool = getGlobalTransportPool() self.__cloneId = 0 self.__maxFD = 0 @@ -81,8 +80,8 @@ def initialize(self): # Build the URLs self._url = self._cfg.getURL() if not self._url: - return S_ERROR("Could not build service URL for %s" % self._name) - gLogger.verbose("Service URL is %s" % self._url) + return S_ERROR(f"Could not build service URL for {self._name}") + gLogger.verbose(f"Service URL is {self._url}") # Load handler result = self._loadHandlerInit() if not result["OK"]: @@ -91,7 +90,7 @@ def initialize(self): # Initialize lock manager self._lockManager = LockManager(self._cfg.getMaxWaitingPetitions()) self._threadPool = ThreadPoolExecutor(max(0, self._cfg.getMaxThreads())) - self._msgBroker = MessageBroker("%sMSB" % self._name, threadPool=self._threadPool) + self._msgBroker = MessageBroker(f"{self._name}MSB", threadPool=self._threadPool) # Create static dict self._serviceInfoDict = { "serviceName": self._name, @@ -101,9 +100,7 @@ def initialize(self): "validNames": self._validNames, "csPaths": [PathFinder.getServiceSection(svcName) for svcName in self._validNames], } - self.securityLogging = Operations().getValue("EnableSecurityLogging", True) and getServiceOption( - self._serviceInfoDict, "EnableSecurityLogging", True - ) + self.securityLogging = Operations().getValue("EnableSecurityLogging", False) # Initialize Monitoring # The import needs to be here because of the CS must be initialized before importing @@ -124,13 +121,13 @@ def initialize(self): result = initFunc(dict(self._serviceInfoDict)) except Exception as excp: gLogger.exception("Exception while calling initialization function", lException=excp) - return S_ERROR("Exception while calling initialization function: %s" % str(excp)) + return S_ERROR(f"Exception while calling initialization function: {str(excp)}") if not isReturnStructure(result): - return S_ERROR("Service initialization function %s must return S_OK/S_ERROR" % initFunc) + return S_ERROR(f"Service initialization function {initFunc} must return S_OK/S_ERROR") if not result["OK"]: - return S_ERROR("Error while initializing {}: {}".format(self._name, result["Message"])) + return S_ERROR(f"Error while initializing {self._name}: {result['Message']}") except Exception as e: - errMsg = "Exception while initializing %s" % self._name + errMsg = f"Exception while initializing {self._name}" gLogger.exception(e) gLogger.exception(errMsg) return S_ERROR(errMsg) @@ -166,12 +163,12 @@ def _loadHandlerInit(self): handlerName = handlerClass.__name__ handlerInitMethods = self.__searchInitFunctions(handlerClass) try: - handlerInitMethods.append(getattr(self._svcData["moduleObj"], "initialize%s" % handlerName)) + handlerInitMethods.append(getattr(self._svcData["moduleObj"], f"initialize{handlerName}")) except AttributeError: gLogger.verbose("Not found global initialization function for service") if handlerInitMethods: - gLogger.info("Found %s initialization methods" % len(handlerInitMethods)) + gLogger.info(f"Found {len(handlerInitMethods)} initialization methods") handlerInfo = {} handlerInfo["name"] = handlerName @@ -182,7 +179,6 @@ def _loadHandlerInit(self): return S_OK(handlerInfo) def _loadActions(self): - handlerClass = self._handler["class"] authRules = {} @@ -198,7 +194,7 @@ def _loadActions(self): for actionType in Service.SVC_VALID_ACTIONS: if self._isMetaAction(actionType): continue - methodPrefix = "%s_" % Service.SVC_VALID_ACTIONS[actionType] + methodPrefix = f"{Service.SVC_VALID_ACTIONS[actionType]}_" for attribute in handlerAttributeList: if attribute.find(methodPrefix) != 0: continue @@ -211,8 +207,8 @@ def _loadActions(self): ) # Look for type and auth rules if actionType == "RPC": - typeAttr = "types_%s" % exportedName - authAttr = "auth_%s" % exportedName + typeAttr = f"types_{exportedName}" + authAttr = f"auth_{exportedName}" else: typeAttr = f"types_{Service.SVC_VALID_ACTIONS[actionType]}_{exportedName}" authAttr = f"auth_{Service.SVC_VALID_ACTIONS[actionType]}_{exportedName}" @@ -275,7 +271,14 @@ def handleConnection(self, clientTransport): """ if not self.activityMonitoring: self._stats["connections"] += 1 - self._threadPool.submit(self._processInThread, clientTransport) + + def err_handler(result): + err = result.exception() + if err: + gLogger.exception("Exception in handleConnection thread", lException=err) + + future = self._threadPool.submit(self._processInThread, clientTransport) + future.add_done_callback(err_handler) @property def wantsThrottle(self): @@ -357,9 +360,9 @@ def _processInThread(self, clientTransport): def _createIdentityString(credDict, clientTransport=None): if "username" in credDict: if "group" in credDict: - identity = "[{}:{}]".format(credDict["username"], credDict["group"]) + identity = f"[{credDict['username']}:{credDict['group']}]" else: - identity = "[%s:unknown]" % credDict["username"] + identity = f"[{credDict['username']}:unknown]" else: identity = "unknown" if clientTransport: @@ -367,7 +370,7 @@ def _createIdentityString(credDict, clientTransport=None): if addr: addr = f"{{{addr[0]}:{addr[1]}}}" if "DN" in credDict: - identity += "(%s)" % credDict["DN"] + identity += f"({credDict['DN']})" return identity @staticmethod @@ -387,7 +390,7 @@ def _receiveAndCheckProposal(self, trid): if not retVal["OK"]: gLogger.error( "Invalid action proposal", - "{} {}".format(self._createIdentityString(credDict, clientTransport), retVal["Message"]), + f"{self._createIdentityString(credDict, clientTransport)} {retVal['Message']}", ) return S_ERROR("Invalid action proposal") proposalTuple = Service._deserializeProposalTuple(retVal["Value"]) @@ -398,11 +401,11 @@ def _receiveAndCheckProposal(self, trid): # Check if this is the requested service requestedService = proposalTuple[0][0] if requestedService not in self._validNames: - return S_ERROR("%s is not up in this server" % requestedService) + return S_ERROR(f"{requestedService} is not up in this server") # Check if the action is valid requestedActionType = proposalTuple[1][0] if requestedActionType not in Service.SVC_VALID_ACTIONS: - return S_ERROR("%s is not a known action type" % requestedActionType) + return S_ERROR(f"{requestedActionType} is not a known action type") # Check if it's authorized result = self._authorizeProposal(proposalTuple[1], trid, credDict) if not result["OK"]: @@ -414,7 +417,7 @@ def _authorizeProposal(self, actionTuple, trid, credDict): # Find CS path for the Auth rules referedAction = self._isMetaAction(actionTuple[0]) if referedAction: - csAuthPath = "%s/Default" % actionTuple[0] + csAuthPath = f"{actionTuple[0]}/Default" hardcodedMethodAuth = self._actions["auth"][actionTuple[0]] else: if actionTuple[0] == "RPC": @@ -442,7 +445,7 @@ def _authorizeProposal(self, actionTuple, trid, credDict): fromHost = "/".join([str(item) for item in tr.getRemoteAddress()]) gLogger.warn( "Unauthorized query", - "to {}:{} by {} from {}".format(self._name, "/".join(actionTuple), identity, fromHost), + f"to {self._name}:{'/'.join(actionTuple)} by {identity} from {fromHost}", ) result = S_ERROR(ENOAUTH, "Unauthorized query") else: @@ -498,7 +501,7 @@ def _instantiateHandler(self, trid, proposalTuple=None): handlerInstance = self._handler["class"](handlerInitDict, trid) handlerInstance.initialize() except Exception as e: - gLogger.exception("Server error while loading handler: %s" % str(e)) + gLogger.exception(f"Server error while loading handler: {str(e)}") return S_ERROR("Server error while loading handler") return S_OK(handlerInstance) @@ -506,6 +509,7 @@ def _processProposal(self, trid, proposalTuple, handlerObj): # Notify the client we're ready to execute the action retVal = self._transportPool.send(trid, S_OK()) if not retVal["OK"]: + retVal["closeTransport"] = True return retVal messageConnection = False @@ -513,7 +517,6 @@ def _processProposal(self, trid, proposalTuple, handlerObj): messageConnection = True if messageConnection: - if self._msgBroker.getNumConnections() > self._cfg.getMaxMessagingConnections(): result = S_ERROR("Maximum number of connections reached. Try later") result["closeTransport"] = True @@ -551,7 +554,15 @@ def _executeAction(self, trid, proposalTuple, handlerObj): response = handlerObj._rh_executeAction(proposalTuple) if not response["OK"]: return response + retVal = response["Value"][0] if self.activityMonitoring: + _actionType, actionName = proposalTuple[1] + retStatus = "Unknown" + if isReturnStructure(retVal): + if retVal["OK"]: + retStatus = "OK" + else: + retStatus = "ERROR" self.activityMonitoringReporter.addRecord( { "timestamp": int(TimeUtilities.toEpochMilliSeconds()), @@ -559,12 +570,15 @@ def _executeAction(self, trid, proposalTuple, handlerObj): "ServiceName": "_".join(self._name.split("/")), "Location": self._cfg.getURL(), "ResponseTime": response["Value"][1], + "MethodName": actionName, + "Protocol": "dips", + "Status": retStatus, } ) - return response["Value"][0] + return retVal except Exception as e: gLogger.exception("Exception while executing handler action") - return S_ERROR("Server error while executing action: %s" % str(e)) + return S_ERROR(f"Server error while executing action: {str(e)}") def _mbReceivedMsg(self, trid, msgObj): result = self._authorizeProposal( diff --git a/src/DIRAC/Core/DISET/private/TransportPool.py b/src/DIRAC/Core/DISET/private/TransportPool.py index 84c33b98090..46bd7ec23ba 100644 --- a/src/DIRAC/Core/DISET/private/TransportPool.py +++ b/src/DIRAC/Core/DISET/private/TransportPool.py @@ -98,14 +98,14 @@ def receive(self, trid, maxBufferSize=0, blockAfterKeepAlive=True, idleReceive=F received = self.__transports[trid][0].receiveData(maxBufferSize, blockAfterKeepAlive, idleReceive) return received except KeyError: - return S_ERROR("No transport with id %s defined" % trid) + return S_ERROR(f"No transport with id {trid} defined") # Send def send(self, trid, msg): try: transport = self.__transports[trid][0] except KeyError: - return S_ERROR("No transport with id %s defined" % trid) + return S_ERROR(f"No transport with id {trid} defined") return transport.sendData(msg) # Send And Close @@ -116,7 +116,7 @@ def sendErrorAndClose(self, trid, msg): if not result["OK"]: return result except KeyError: - return S_ERROR("No transport with id %s defined" % trid) + return S_ERROR(f"No transport with id {trid} defined") finally: self.close(trid) @@ -126,7 +126,7 @@ def sendAndClose(self, trid, msg): if not result["OK"]: return result except KeyError: - return S_ERROR("No transport with id %s defined" % trid) + return S_ERROR(f"No transport with id {trid} defined") finally: self.close(trid) @@ -134,7 +134,7 @@ def sendKeepAlive(self, trid, responseId=None): try: return self.__transports[trid][0].sendKeepAlive(responseId) except KeyError: - return S_ERROR("No transport with id %s defined" % trid) + return S_ERROR(f"No transport with id {trid} defined") # Close @@ -142,7 +142,7 @@ def close(self, trid): try: self.__transports[trid][0].close() except KeyError: - return S_ERROR("No transport with id %s defined" % trid) + return S_ERROR(f"No transport with id {trid} defined") self.remove(trid) def remove(self, trid): diff --git a/src/DIRAC/Core/DISET/private/Transports/BaseTransport.py b/src/DIRAC/Core/DISET/private/Transports/BaseTransport.py index 7c03242ffaa..37b0858bbf7 100755 --- a/src/DIRAC/Core/DISET/private/Transports/BaseTransport.py +++ b/src/DIRAC/Core/DISET/private/Transports/BaseTransport.py @@ -17,6 +17,7 @@ Client <- Service : Close """ + import time from io import BytesIO from hashlib import md5 @@ -27,6 +28,9 @@ from DIRAC.FrameworkSystem.Client.Logger import gLogger from DIRAC.Core.Utilities import MixedEncode +# https://datatracker.ietf.org/doc/html/rfc8446#section-5.1 +TLS_PAYLOAD_SIZE = 16384 + class BaseTransport: """Invokes MixedEncode for marshaling/unmarshaling of data calls in transit""" @@ -42,7 +46,7 @@ class BaseTransport: def __init__(self, stServerAddress, bServerMode=False, **kwargs): self.bServerMode = bServerMode self.extraArgsDict = kwargs - self.byteStream = b"" + self.byteStream = bytearray() self.packetSize = 1048576 # 1MiB self.stServerAddress = stServerAddress self.peerCredentials = {} @@ -154,7 +158,7 @@ def _read(self, bufSize=4096, skipReadyCheck=False): else: return S_ERROR("Connection seems stalled. Closing...") except Exception as e: - return S_ERROR("Exception while reading from peer: %s" % str(e)) + return S_ERROR(f"Exception while reading from peer: {str(e)}") def _write(self, buf): return S_OK(self.oSocket.send(buf)) @@ -175,7 +179,9 @@ def sendData(self, uData, prefix=b""): return result sentBytes = result["Value"] except Exception as e: - return S_ERROR("Exception while sending data: %s" % e) + return S_ERROR(f"Exception while sending data: {e}") + if sentBytes < 0: + return S_ERROR("Unknown unrecoverable error from socket while sending data") if sentBytes == 0: return S_ERROR("Connection closed by peer") packSentBytes += sentBytes @@ -191,12 +197,12 @@ def receiveData(self, maxBufferSize=0, blockAfterKeepAlive=True, idleReceive=Fal maxBufferSize = max(maxBufferSize, 0) try: # Look either for message length of keep alive magic string - iSeparatorPosition = self.byteStream.find(b":", 0, 10) + iSeparatorPosition = self.byteStream.find(b":") keepAliveMagicLen = len(BaseTransport.keepAliveMagic) isKeepAlive = self.byteStream.find(BaseTransport.keepAliveMagic, 0, keepAliveMagicLen) == 0 # While not found the message length or the ka, keep receiving while iSeparatorPosition == -1 and not isKeepAlive: - retVal = self._read(16384) + retVal = self._read(TLS_PAYLOAD_SIZE) # If error return if not retVal["OK"]: return retVal @@ -204,17 +210,18 @@ def receiveData(self, maxBufferSize=0, blockAfterKeepAlive=True, idleReceive=Fal if not retVal["Value"]: return S_ERROR("Peer closed connection") # New data! - self.byteStream += retVal["Value"] - # Look again for either message length of ka magic string - iSeparatorPosition = self.byteStream.find(b":", 0, 10) + self.byteStream.extend(retVal["Value"]) + + # Look again for either message length or keep alive magic string + iSeparatorPosition = self.byteStream.find(b":") isKeepAlive = self.byteStream.find(BaseTransport.keepAliveMagic, 0, keepAliveMagicLen) == 0 # Over the limit? if maxBufferSize and len(self.byteStream) > maxBufferSize and iSeparatorPosition == -1: - return S_ERROR("Read limit exceeded (%s chars)" % maxBufferSize) + return S_ERROR(f"Read limit exceeded ({maxBufferSize} chars)") # Keep alive magic! if isKeepAlive: gLogger.debug("Received keep alive header") - # Remove the ka magic from the buffer and process the keep alive + # Remove the keep-alive magic from the buffer and process the keep-alive self.byteStream = self.byteStream[keepAliveMagicLen:] return self.__processKeepAlive(maxBufferSize, blockAfterKeepAlive) # From here it must be a real message! @@ -222,17 +229,18 @@ def receiveData(self, maxBufferSize=0, blockAfterKeepAlive=True, idleReceive=Fal pkgSize = int(self.byteStream[:iSeparatorPosition]) pkgData = self.byteStream[iSeparatorPosition + 1 :] readSize = len(pkgData) + if readSize >= pkgSize: # If we already have all the data we need data = pkgData[:pkgSize] - self.byteStream = pkgData[pkgSize:] + self.byteStream = self.byteStream[pkgSize + iSeparatorPosition + 1 :] else: # If we still need to read stuff pkgMem = BytesIO() pkgMem.write(pkgData) # Receive while there's still data to be received while readSize < pkgSize: - retVal = self._read(pkgSize - readSize, skipReadyCheck=True) + retVal = self._read(min(TLS_PAYLOAD_SIZE, pkgSize - readSize), skipReadyCheck=True) if not retVal["OK"]: return retVal if not retVal["Value"]: @@ -241,42 +249,43 @@ def receiveData(self, maxBufferSize=0, blockAfterKeepAlive=True, idleReceive=Fal readSize += len(rcvData) pkgMem.write(rcvData) if maxBufferSize and readSize > maxBufferSize: - return S_ERROR("Read limit exceeded (%s chars)" % maxBufferSize) + return S_ERROR(f"Read limit exceeded ({maxBufferSize} chars)") # Data is here! take it out from the bytestream, dencode and return if readSize == pkgSize: data = pkgMem.getvalue() - self.byteStream = b"" - else: # readSize > pkgSize: + self.byteStream = bytearray() # Reset the byteStream + else: pkgMem.seek(0, 0) data = pkgMem.read(pkgSize) - self.byteStream = pkgMem.read() + self.byteStream = bytearray(pkgMem.read()) # store the rest in bytearray + try: data = MixedEncode.decode(data)[0] except Exception as e: - return S_ERROR("Could not decode received data: %s" % str(e)) + return S_ERROR(f"Could not decode received data: {str(e)}") if idleReceive: self.receivedMessages.append(data) return S_OK() return data except Exception as e: gLogger.exception("Network error while receiving data") - return S_ERROR("Network error while receiving data: %s" % str(e)) + return S_ERROR(f"Network error while receiving data: {str(e)}") def __processKeepAlive(self, maxBufferSize, blockAfterKeepAlive=True): gLogger.debug("Received Keep Alive") # Next message down the stream will be the ka data result = self.receiveData(maxBufferSize, blockAfterKeepAlive=False) if not result["OK"]: - gLogger.debug("Error while receiving keep alive: %s" % result["Message"]) + gLogger.debug(f"Error while receiving keep alive: {result['Message']}") return result # Is it a valid ka? kaData = result["Value"] for reqField in ("id", "kaping"): if reqField not in kaData: - errMsg = "Invalid keep alive, missing %s" % reqField + errMsg = f"Invalid keep alive, missing {reqField}" gLogger.debug(errMsg) return S_ERROR(errMsg) - gLogger.debug("Received keep alive id %s" % kaData) + gLogger.debug(f"Received keep alive id {kaData}") # Need to check if it's one of the keep alives we sent or one started from the other side if kaData["kaping"]: # This is a keep alive PING. Let's send the PONG @@ -319,7 +328,7 @@ def getFormattedCredentials(self): peerCreds = self.getConnectingCredentials() address = self.getRemoteAddress() if "username" in peerCreds: - peerId = "[{}:{}]".format(peerCreds["group"], peerCreds["username"]) + peerId = f"[{peerCreds['group']}:{peerCreds['username']}]" else: peerId = "" if address[0].find(":") > -1: diff --git a/src/DIRAC/Core/DISET/private/Transports/M2SSLTransport.py b/src/DIRAC/Core/DISET/private/Transports/M2SSLTransport.py index 551672ce0bc..d044f4e0a3f 100755 --- a/src/DIRAC/Core/DISET/private/Transports/M2SSLTransport.py +++ b/src/DIRAC/Core/DISET/private/Transports/M2SSLTransport.py @@ -118,9 +118,11 @@ def initAsClient(self): # We ignore the returned sockaddr because SSL.Connection.connect needs # a host name. - addrInfoList = socket.getaddrinfo(host, port, socket.AF_UNSPEC, socket.SOCK_STREAM) - for (family, _socketType, _proto, _canonname, _socketAddress) in addrInfoList: - + try: + addrInfoList = socket.getaddrinfo(host, port, socket.AF_UNSPEC, socket.SOCK_STREAM) + except OSError as e: + return S_ERROR(f"DNS lookup failed {e!r}") + for family, _socketType, _proto, _canonname, _socketAddress in addrInfoList: try: self.oSocket = SSL.Connection(self.__ctx, family=family) @@ -491,7 +493,8 @@ def _write(self, buf): # And writting on a socket that received an RST packet # triggers a SIGPIPE. # In practice, this means that if the server replies to a - # dead client with less that 16384 bytes (see), + # dead client with less that 16384 bytes + # (see https://datatracker.ietf.org/doc/html/rfc8446#section-5.1), # we will never notice that we sent the answer to the vacuum. # And don't look for a fix, there just isn't. wrote = self.oSocket.write(buf) diff --git a/src/DIRAC/Core/DISET/private/Transports/PlainTransport.py b/src/DIRAC/Core/DISET/private/Transports/PlainTransport.py index 2973e65c58c..7a923a1558a 100755 --- a/src/DIRAC/Core/DISET/private/Transports/PlainTransport.py +++ b/src/DIRAC/Core/DISET/private/Transports/PlainTransport.py @@ -1,12 +1,8 @@ +import selectors import socket import time import os -try: - import selectors -except ImportError: - import selectors2 as selectors - from DIRAC.Core.DISET.private.Transports.BaseTransport import BaseTransport from DIRAC.FrameworkSystem.Client.Logger import gLogger from DIRAC.Core.Utilities.ReturnValues import S_ERROR, S_OK @@ -21,7 +17,7 @@ def initAsClient(self): self.oSocket = socket.create_connection(self.stServerAddress, timeout) except OSError as e: if e.args[0] != 115: - return S_ERROR("Can't connect: %s" % str(e)) + return S_ERROR(f"Can't connect: {str(e)}") # Connect in progress sel = selectors.DefaultSelector() sel.register(self.oSocket, selectors.EVENT_READ) @@ -30,7 +26,7 @@ def initAsClient(self): return S_ERROR("Connection timeout") errno = self.oSocket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) if errno != 0: - return S_ERROR("Can't connect: %s" % str((errno, os.strerror(errno)))) + return S_ERROR(f"Can't connect: {str((errno, os.strerror(errno)))}") self.remoteAddress = self.oSocket.getpeername() return S_OK(self.oSocket) @@ -87,9 +83,9 @@ def _read(self, bufSize=4096, skipReadyCheck=False): if e.errno == 11: time.sleep(0.001) else: - return S_ERROR("Exception while reading from peer: %s" % str(e)) + return S_ERROR(f"Exception while reading from peer: {str(e)}") except Exception as e: - return S_ERROR("Exception while reading from peer: %s" % str(e)) + return S_ERROR(f"Exception while reading from peer: {str(e)}") def _write(self, buf): sentBytes = 0 @@ -112,9 +108,9 @@ def _write(self, buf): if e.errno == 11: time.sleep(0.001) else: - return S_ERROR("Exception while sending to peer: %s" % str(e)) + return S_ERROR(f"Exception while sending to peer: {str(e)}") except Exception as e: - return S_ERROR("Error while sending: %s" % str(e)) + return S_ERROR(f"Error while sending: {str(e)}") return S_OK(sentBytes) diff --git a/src/DIRAC/Core/DISET/private/Transports/SSL/M2Utils.py b/src/DIRAC/Core/DISET/private/Transports/SSL/M2Utils.py index 67fc40a8c7b..79d4b1da003 100644 --- a/src/DIRAC/Core/DISET/private/Transports/SSL/M2Utils.py +++ b/src/DIRAC/Core/DISET/private/Transports/SSL/M2Utils.py @@ -4,8 +4,11 @@ """ import os import tempfile +import M2Crypto +from packaging.version import Version from M2Crypto import SSL, m2, X509 + from DIRAC.Core.DISET import DEFAULT_SSL_CIPHERS, DEFAULT_SSL_METHODS from DIRAC.Core.Security import Locations from DIRAC.Core.Security.m2crypto.X509Chain import X509Chain @@ -15,6 +18,30 @@ DEBUG_M2CRYPTO = os.getenv("DIRAC_DEBUG_M2CRYPTO", "No").lower() in ("yes", "true") +VERIFY_ALLOW_PROXY_CERTS = 0 + +# If the version of M2Crypto is recent enough, there is an API +# to accept proxy certificate, and we do not need to rely on +# OPENSSL_ALLOW_PROXY_CERT environment variable +# which was removed as of openssl 1.1 +# We need this to be merged in M2Crypto: https://gitlab.com/m2crypto/m2crypto/merge_requests/236 +# We set the proper verify flag to the X509Store of the context +# as described here https://www.openssl.org/docs/man1.1.1/man7/proxy-certificates.html +if hasattr(SSL, "verify_allow_proxy_certs"): + VERIFY_ALLOW_PROXY_CERTS = SSL.verify_allow_proxy_certs # pylint: disable=no-member +# As of M2Crypto 0.37, the `verify_allow_proxy_certs` flag was moved +# to X509 (https://gitlab.com/m2crypto/m2crypto/-/merge_requests/238) +# It is more consistent with all the other flags, +# but pySSL had it in SSL. Well... +elif hasattr(X509, "verify_allow_proxy_certs"): + VERIFY_ALLOW_PROXY_CERTS = X509.verify_allow_proxy_certs # pylint: disable=no-member +# As of M2Crypto 0.38, M2Crypto did not export the symbol correctly +# Anymore +# https://gitlab.com/m2crypto/m2crypto/-/issues/298 +elif Version(M2Crypto.__version__) >= Version("0.38.0"): + VERIFY_ALLOW_PROXY_CERTS = 64 + + def __loadM2SSLCTXHostcert(ctx): """Load hostcert & key from the default location and set them as the credentials for SSL context ctx. @@ -25,9 +52,9 @@ def __loadM2SSLCTXHostcert(ctx): raise RuntimeError("Hostcert/key location not set") hostcert, hostkey = certKeyTuple if not os.path.isfile(hostcert): - raise RuntimeError("Hostcert file (%s) is missing" % hostcert) + raise RuntimeError(f"Hostcert file ({hostcert}) is missing") if not os.path.isfile(hostkey): - raise RuntimeError("Hostkey file (%s) is missing" % hostkey) + raise RuntimeError(f"Hostkey file ({hostkey}) is missing") # Make sure we never stall on a password prompt if the hostkey has a password # by specifying a blank string. ctx.load_cert(hostcert, hostkey, callback=lambda: "") @@ -43,7 +70,7 @@ def __loadM2SSLCTXProxy(ctx, proxyPath=None): if not proxyPath: raise RuntimeError("Proxy location not set") if not os.path.isfile(proxyPath): - raise RuntimeError("Proxy file (%s) is missing" % proxyPath) + raise RuntimeError(f"Proxy file ({proxyPath}) is missing") # See __loadM2SSLCTXHostcert for description of why lambda is needed. ctx.load_cert_chain(proxyPath, proxyPath, callback=lambda: "") @@ -89,7 +116,9 @@ def getM2SSLContext(ctx=None, **kwargs): # CHRIS: I think clientMode was just an internal of pyGSI implementation # if kwargs.get('clientMode', False) and not kwargs.get('useCertificates', False): # if not kwargs.get('useCertificates', False): - if kwargs.get("bServerMode", False) or kwargs.get("useCertificates", False): + if kwargs.get("bServerMode", False) or ( + kwargs.get("useCertificates", False) and not kwargs.get("proxyLocation", False) + ): # Server mode always uses hostcert __loadM2SSLCTXHostcert(ctx) @@ -101,6 +130,8 @@ def getM2SSLContext(ctx=None, **kwargs): with tempfile.NamedTemporaryFile(mode="w") as tmpFile: tmpFilePath = tmpFile.name tmpFile.write(kwargs["proxyString"]) + # Flush, otherwise the file is empty in the subsequent call + tmpFile.flush() __loadM2SSLCTXProxy(ctx, proxyPath=tmpFilePath) else: # Use normal proxy @@ -120,24 +151,12 @@ def getM2SSLContext(ctx=None, **kwargs): if not caPath: raise RuntimeError("Failed to find CA location") if not os.path.isdir(caPath): - raise RuntimeError("CA path (%s) is not a valid directory" % caPath) + raise RuntimeError(f"CA path ({caPath}) is not a valid directory") ctx.load_verify_locations(capath=caPath) - # If the version of M2Crypto is recent enough, there is an API - # to accept proxy certificate, and we do not need to rely on - # OPENSSL_ALLOW_PROXY_CERT environment variable - # which was removed as of openssl 1.1 - # We need this to be merged in M2Crypto: https://gitlab.com/m2crypto/m2crypto/merge_requests/236 - # We set the proper verify flag to the X509Store of the context - # as described here https://www.openssl.org/docs/man1.1.1/man7/proxy-certificates.html - if hasattr(SSL, "verify_allow_proxy_certs"): - ctx.get_cert_store().set_flags(SSL.verify_allow_proxy_certs) # pylint: disable=no-member - # As of M2Crypto 0.37, the `verify_allow_proxy_certs` flag was moved - # to X509 (https://gitlab.com/m2crypto/m2crypto/-/merge_requests/238) - # It is more consistent with all the other flags, - # but pySSL had it in SSL. Well... - if hasattr(X509, "verify_allow_proxy_certs"): - ctx.get_cert_store().set_flags(X509.verify_allow_proxy_certs) # pylint: disable=no-member + # Allow proxy certificates to be used + if VERIFY_ALLOW_PROXY_CERTS: + ctx.get_cert_store().set_flags(VERIFY_ALLOW_PROXY_CERTS) # Other parameters sslMethods = kwargs.get("sslMethods", DEFAULT_SSL_METHODS) @@ -178,13 +197,13 @@ def getM2PeerInfo(conn): chain = X509Chain.generateX509ChainFromSSLConnection(conn) creds = chain.getCredentials(withRegistryInfo=False) if not creds["OK"]: - raise RuntimeError("Failed to get SSL peer info (%s)." % creds["Message"]) + raise RuntimeError(f"Failed to get SSL peer info ({creds['Message']}).") peer = creds["Value"] peer["x509Chain"] = chain isProxy = chain.isProxy() if not isProxy["OK"]: - raise RuntimeError("Failed to get SSL peer isProxy (%s)." % isProxy["Message"]) + raise RuntimeError(f"Failed to get SSL peer isProxy ({isProxy['Message']}).") peer["isProxy"] = isProxy["Value"] if peer["isProxy"]: @@ -194,7 +213,7 @@ def getM2PeerInfo(conn): isLimited = chain.isLimitedProxy() if not isLimited["OK"]: - raise RuntimeError("Failed to get SSL peer isProxy (%s)." % isLimited["Message"]) + raise RuntimeError(f"Failed to get SSL peer isProxy ({isLimited['Message']}).") peer["isLimitedProxy"] = isLimited["Value"] return peer diff --git a/src/DIRAC/Core/DISET/private/Transports/SSLTransport.py b/src/DIRAC/Core/DISET/private/Transports/SSLTransport.py index 2dd24535948..f5b42f03393 100755 --- a/src/DIRAC/Core/DISET/private/Transports/SSLTransport.py +++ b/src/DIRAC/Core/DISET/private/Transports/SSLTransport.py @@ -40,7 +40,9 @@ def checkSanity(urlTuple, kwargs): """ useCerts = False certFile = "" - if "useCertificates" in kwargs and kwargs["useCertificates"]: + if kwargs.get("proxyLocation"): + certFile = kwargs["proxyLocation"] + elif "useCertificates" in kwargs and kwargs["useCertificates"]: certTuple = Locations.getHostCertificateAndKeyLocation() if not certTuple: gLogger.error("No cert/key found! ") @@ -48,20 +50,17 @@ def checkSanity(urlTuple, kwargs): certFile = certTuple[0] useCerts = True elif "proxyString" in kwargs: - if not isinstance(kwargs["proxyString"], bytes): + if not isinstance(kwargs["proxyString"], str): gLogger.error("proxyString parameter is not a valid type", str(type(kwargs["proxyString"]))) return S_ERROR("proxyString parameter is not a valid type") else: - if "proxyLocation" in kwargs: - certFile = kwargs["proxyLocation"] - else: - certFile = Locations.getProxyLocation() + certFile = Locations.getProxyLocation() if not certFile: gLogger.error("No proxy found") return S_ERROR("No proxy found") elif not os.path.isfile(certFile): gLogger.error("Proxy file does not exist", certFile) - return S_ERROR("%s proxy file does not exist" % certFile) + return S_ERROR(f"{certFile} proxy file does not exist") # For certs always check CA's. For clients skipServerIdentityCheck if "skipCACheck" not in kwargs or not kwargs["skipCACheck"]: @@ -85,8 +84,8 @@ def checkSanity(urlTuple, kwargs): retVal = certObj.hasExpired() if not retVal["OK"]: - gLogger.error("Can't verify proxy or certificate file", "{}:{}".format(certFile, retVal["Message"])) - return S_ERROR("Can't verify file {}:{}".format(certFile, retVal["Message"])) + gLogger.error("Can't verify proxy or certificate file", f"{certFile}:{retVal['Message']}") + return S_ERROR(f"Can't verify file {certFile}:{retVal['Message']}") else: if retVal["Value"]: notAfter = certObj.getNotAfterDate() diff --git a/src/DIRAC/Core/DISET/private/Transports/test/README.md b/src/DIRAC/Core/DISET/private/Transports/test/README.md new file mode 100644 index 00000000000..a0b70dec73a --- /dev/null +++ b/src/DIRAC/Core/DISET/private/Transports/test/README.md @@ -0,0 +1,5 @@ +Generated with + +``` +dirac-proxy-init -C src/DIRAC/Core/Security/test/certs/user/usercert.pem -K src/DIRAC/Core/Security/test/certs/user/userkey.pem -u src/DIRAC/Core/DISET/private/Transports/test/proxy.pem --valid 87600:00 +``` diff --git a/src/DIRAC/Core/DISET/private/Transports/test/Test_SSLTransport.py b/src/DIRAC/Core/DISET/private/Transports/test/Test_SSLTransport.py index b9a492f51fc..0e8134652db 100644 --- a/src/DIRAC/Core/DISET/private/Transports/test/Test_SSLTransport.py +++ b/src/DIRAC/Core/DISET/private/Transports/test/Test_SSLTransport.py @@ -1,19 +1,14 @@ """ Test the SSLTransport mechanism """ import os -import socket +import selectors import threading -try: - import selectors -except ImportError: - import selectors2 as selectors - +from diraccfg import CFG from pytest import fixture -from diraccfg import CFG -from DIRAC.Core.Security.test.x509TestUtilities import CERTDIR, USERCERT, getCertOption from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData -from DIRAC.Core.DISET.private.Transports import PlainTransport, M2SSLTransport +from DIRAC.Core.DISET.private.Transports import M2SSLTransport, PlainTransport +from DIRAC.Core.Security.test.x509TestUtilities import CERTDIR, USERCERT, getCertOption # TODO: Expired hostcert # TODO: Expired usercert @@ -128,7 +123,7 @@ def transportByName(transport): return PlainTransport.PlainTransport elif transport.lower() == "m2": return M2SSLTransport.SSLTransport - raise RuntimeError("Unknown Transport Name: %s" % transport) + raise RuntimeError(f"Unknown Transport Name: {transport}") @fixture(scope="function", params=TRANSPORTTESTS) diff --git a/src/DIRAC/Core/DISET/private/Transports/test/proxy.pem b/src/DIRAC/Core/DISET/private/Transports/test/proxy.pem index bb735663024..c93e7ccc94b 100644 --- a/src/DIRAC/Core/DISET/private/Transports/test/proxy.pem +++ b/src/DIRAC/Core/DISET/private/Transports/test/proxy.pem @@ -1,40 +1,55 @@ -----BEGIN CERTIFICATE----- -MIIDujCCAaKgAwIBAgIKNDU0MDE1NDkzMjANBgkqhkiG9w0BAQsFADA6MRgwFgYD -VQQKDA9EaXJhYyBDb21wdXRpbmcxDTALBgNVBAoMBENFUk4xDzANBgNVBAMMBk1y -VXNlcjAeFw0xODA5MTExNDIxNDdaFw00NjAxMjcxNDM2NDdaME8xGDAWBgNVBAoM -D0RpcmFjIENvbXB1dGluZzENMAsGA1UECgwEQ0VSTjEPMA0GA1UEAwwGTXJVc2Vy -MRMwEQYDVQQDDAo1OTc5Mjg4MDk5MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKB -gQCkXKvhHqCAxiMIGYnf0/DeiKHfh1B1B7UPmNbZOjXV71Z+vdetX862dqABIa6g -p2oziOwOEZYMr7SnekSpwJp3ccTrXOFKZQNclf14n3ATrB1uh6EBkr51wYh2C4Ls -UEJ+tn+TdKxmDehD+mIQcXBFyz+mrSAIXUD85aN254ul0wIDAQABozEwLzAOBgNV -HQ8BAf8EBAMCBLAwHQYIKwYBBQUHAQ4BAf8EDjAMMAoGCCsGAQUFBxUBMA0GCSqG -SIb3DQEBCwUAA4ICAQBgj26i2HwIjBKMDdFfF+CtksQITEOQDTqyksOB2SWu98XU -nB29bgG4RHyizkYgxsUUQ+Fw6nnTGysbf+lwaagzTzkv5Ls3OD6Gy2TTL8dao2cz -2yBLy9U17vTXl9tFRnLBzRWhUfudvcB4ULyItsMYqcDGOM450512jtiHJr35OH9s -SmKsJYk8aHmLAAL5F9DU08xxyExp3UJK5XoqhtBnRrC1vgU2KLbZ8FQfe8MXxkG6 -nDnGK8uEI9xZGTkTi4FYQCFYzl+Wr7ZHP2xxhMGsWuADvHNSlWIbQjYkXDaGZqZ3 -nHoJpE80YSOs4LZNIid1jmPVrnYGs3hzdmJ1S8cNh8ZTFb3YTKqqyh8mUksFfskf -2Y7qYqnAN1Xr9lRkKMh05L3gmakwAxgvopIzVPOdLZMJHaFtrzvh1ky+hqfD1kqT -QxdGqZQx4gZVrTPcMtLPiVk9ChREwl/z0TQ5DisAL8xAJpvFQwaVRgYyn2SIVg7q -Q2rWO5CHvkc5imk/B1m6sIi3Mf8FuSOLdJjWCzDOClLgkl4NPt67BL8SiHgbhH8h -3ZncBM9O4s6BHjkCCM8fb2TIEUOOX2e1hqtewZv4fLdjOOgCKEMmNIZ/Pja9KRL6 -zO1aPEqibEKjgc9pUONmcXgwp+dfbRRrMJc3PcDgr3AbKq9CQcKuQugSjKtVDg== +MIIEOTCCAiGgAwIBAgIFAUXu3+AwDQYJKoZIhvcNAQELBQAwOjEYMBYGA1UECgwP +RGlyYWMgQ29tcHV0aW5nMQ0wCwYDVQQKDARDRVJOMQ8wDQYDVQQDDAZNclVzZXIw +HhcNMjMxMTI4MjExMDIzWhcNMzMxMTI1MjEyNTIzWjBPMRgwFgYDVQQKDA9EaXJh +YyBDb21wdXRpbmcxDTALBgNVBAoMBENFUk4xDzANBgNVBAMMBk1yVXNlcjETMBEG +A1UEAwwKNTQ2ODI1MDA4MDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEB +AP3qqOreMUBzNxGFCuqGb1Oh8H0oHRvkxptVixOw7+ckg36Lxn8l0tpHBCuAJeDQ +w8lKCix5SDVBK/FsKBhCCln8uMlv9dM+X8kez2rPFc0Hp6L6W5zEzW/0CbPs4Zwd +xvWysGcC3dVlxhXw/UAsUC4aJtD74VQGXUI03y7ozh/UmjUDTt5m33UIlIZEisPI +YZpA+6RMgNLJXx8EOhrdCJI3oD45+mBMqZUvRGWRAsIWYRr1jAdSmIEGLvLx/7dt +Dmi7L9Y4ZLcz7Wal83OWcUp1uX1y+k5yTss3SWzN2GLqQZIHL1XmgfOwrdqe50qa +Q3ii2qdWbH6ldS6ap2mGrrsCAwEAAaMxMC8wDgYDVR0PAQH/BAQDAgSwMB0GCCsG +AQUFBwEOAQH/BA4wDDAKBggrBgEFBQcVATANBgkqhkiG9w0BAQsFAAOCAgEABskd +Tjksd8cMhz7Ar8waASgIj9mjf/Jhg8L1BIMYKsCMi8rVTGX5Wl2F2WLfiMrp0RqO +MxSUk9sJ6b5JjrXNpvTQzFm4AKemmMC40k1N4FjkQngVemE0xiHxanxLtrggO2/9 +zxgTobEqaXV94gBXcRLrhIvHTcEDPekc9hBY6RRdH7PA7jpyZblF+RuwIbSCC+vM +JvL3PY5PiYRgzxeC1qDKXjo/NHO0cQ/LhjKDZoguFz8lYctLLezlCyocj0Wl5zwm +kbzejs9mA39UOYC7HYGJbfXXqEYjnNBtfcPE6naW5J0C26J6WPMrs+hm0jshIli5 +c1aQraEm9IYzY2s4ovmx0ZGfGUdnSOfbamJpYQLVsY0BNh4yzp5Qn3jEJm3ktoQ5 +0U54biUYbbUfg1WdthBqi+Zogk9+AbJ8b+C2ZzDwI/QsF5r9jPbh/J+dAshVUquz +UbCmERbq5dfNNO3qp1DAfCSAu20mDnO6ig105aU+58beL8Y6FphmZWm9JaIDnln4 +TXq1YGv+IuaYftuoBF/I2oa5k3WfteNVki7widk/bx3fW1luYitSROjsVDiU1ZkW +CWjclXEmsXKL/44mqx9iPzVAy3zhfkJ84gLr7gdhlnCP2KHJpyJWb9JOWFYi+fwf +2jTePuiDTA63Qb6/o4kLx3NR9fRO6G8/AZw8lvo= -----END CERTIFICATE----- -----BEGIN PRIVATE KEY----- -MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAKRcq+EeoIDGIwgZ -id/T8N6Iod+HUHUHtQ+Y1tk6NdXvVn69161fzrZ2oAEhrqCnajOI7A4RlgyvtKd6 -RKnAmndxxOtc4UplA1yV/XifcBOsHW6HoQGSvnXBiHYLguxQQn62f5N0rGYN6EP6 -YhBxcEXLP6atIAhdQPzlo3bni6XTAgMBAAECgYBMDYmGyHoyO/x3wgF3LYW2Ivjs -CpvjGybfybQYngPNM0sNqGCFG+D1sxxEicZZFj5hDElWFTMeOatZF41wEuwl/PHe -dZjdzB7lKC5EDnVBEgj9t/8VSfIxdsOMdZTjKz6L/UBZTL7hAaHG2O8Pk7O2m2IR -249R6jbjgA5BSFQwAQJBANQJNnkBRh0nDgIKfymvF/6ckpA1QwjoycVyUyzEJ9tA -GNpy4qRQ/xlFID9oca3wZ3a+BLcR9oAIIfTA8zuwvdMCQQDGcPEgoObi3VFP/QkA -+YW5sWKffe8jWJ5pGKq5Yw6TXmdqIcps/p6bziFI4BrAZhwU5kh9Ick3npC+6TS1 -oHgBAkBNiejvqzWWp5eJy4jhF3Sw2VUHg1K2SVqv57Te5ASnOvNbvzN+X1hKR2sZ -hPo9X3KWi7pxsBHylAbG2GCabXGdAkAVM5fvnoFMl8zKOQSvP/mTn2okFDZqlltG -a0ZCTF0QTbPK2RVhk8qqZtmTia4SBFbXvMrd47A16xEX9J6XETABAkEAgGSvP+VG -88Mno+nCUEipT3BrImB06UnSlf8XCdFpi5TOs69a8ILURKcvF/lu1cYQABmhsyk4 -SBGWOb7u1gR/sw== +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQD96qjq3jFAczcR +hQrqhm9TofB9KB0b5MabVYsTsO/nJIN+i8Z/JdLaRwQrgCXg0MPJSgoseUg1QSvx +bCgYQgpZ/LjJb/XTPl/JHs9qzxXNB6ei+lucxM1v9Amz7OGcHcb1srBnAt3VZcYV +8P1ALFAuGibQ++FUBl1CNN8u6M4f1Jo1A07eZt91CJSGRIrDyGGaQPukTIDSyV8f +BDoa3QiSN6A+OfpgTKmVL0RlkQLCFmEa9YwHUpiBBi7y8f+3bQ5ouy/WOGS3M+1m +pfNzlnFKdbl9cvpOck7LN0lszdhi6kGSBy9V5oHzsK3anudKmkN4otqnVmx+pXUu +mqdphq67AgMBAAECggEATc7DHVmiCQhlRxKYQj6Yza1xhsCsjtgffgkMGageE5vu +SXBmjp7WCno5jmTx9n6yiDOOg4tUs7D5WL0WWjyedG3LaDrNPwK9kmFFGQtFOHNQ +jNIgEZ2DAHvtHzwG9HJxfefYQ3Cu7o4F0cJrsGcD2OS9oUuWBEwA9uFBxNulEj5V +E4h6KMD8TMbvTIuRiJYoPl0vIARqeD2nY+/C3gw1n2376PZuC0omeIiMJY0EegfC +5MAoFx5DGwjwnMldFeY0iN+qY6wnrZK/5tV4K2TrW34JJa/QrA6SF+Pqt+mVaMD4 +c0hOvecj3EJCSEPv4msYSNBzY/ZAbwaj0PR2jxmcbQKBgQD/P5GFp5zwUEGBcfzR +2WVtQt3u8Y1hMK6N7uXOx+4l5VPMr7wExVCE8G4jOmwrFJDo3firxroVKItk5MMA +8/ka1LdW3X6Scr5iVtXeUXstBcoi8Qt0fsGkz2COlLXY/W8+ihlJPIeqMaYUoiB1 +81nQORuUvqhj59gUYm0njyZ0hQKBgQD+qhZicqE0mdpxeV5uS+1mYscUJySJKsZM +0vIui/UuNVmE2tAXRmC6zKdR857VvVZ/FVyftWo7EGEybAfF4zWcb5LwuNq3zQXt +WYN++69GEriLRuMHIQwhlJwt+X6XVB0a+urR7HM4+jlf6HGXaA2ZINYEhAmCpf8J +pg2SxpiaPwKBgEQLb0DhKQ5LZtsaRxquSMKy47UyQc1aC/6cZDkWxV7m3ssfQhFH +hKqb6dCMX4+wgN0DZ6prZOoFD/wKnA2h/JNxh5qpm3dxDV3r5kHJGPwsofFkrvgU +Xo0QF56K+FtrXH+gkxMaBtSRPcQcYGjxQc0nnDmwBfX0NX9hqdW07Lx9AoGAN4Y8 +JTDbBw34e787oI67bxRgVXuHUsTZwYxIs29egLmvD/FpZ3m3w2K1pH+ahP2oK0Ms +E8JJLCGRH55AP5wfZ0FIZ2XWgjaYcTyQGBKmD4ArbmqBO1+wNm4hc0Cvoiz7v5Mv +uZ91K9oawld61MkiFd3767YiILMynRbwZK0aPp8CgYBOVPxXPIPPTcE+Y66ARhQb +up0YsDn8YlvGIrSzF+ujGxXLEyJBSzr8eytmASpT3C+6xRkmdLMFzJHxeutl1VaI +IK02GbFZz0NWPBc7o4uwpVkvKWayvsoojfzAV6UwETRxCYTyMx2OaejeZRx3Z68z +QtFezp67ih0dS7gBa7eh7A== -----END PRIVATE KEY----- -----BEGIN CERTIFICATE----- MIIFszCCA5ugAwIBAgICEAEwDQYJKoZIhvcNAQELBQAwVDEYMBYGA1UECgwPRElS diff --git a/src/DIRAC/Core/DISET/test/Test_AuthManager.py b/src/DIRAC/Core/DISET/test/Test_AuthManager.py index 1e1b969981e..e509dc5e483 100644 --- a/src/DIRAC/Core/DISET/test/Test_AuthManager.py +++ b/src/DIRAC/Core/DISET/test/Test_AuthManager.py @@ -128,7 +128,6 @@ def tearDown(self): pass def test_userProperties(self): - # MethodAll accepts everybody result = self.authMgr.authQuery("MethodAll", self.userCredDict) self.assertTrue(result) @@ -166,7 +165,6 @@ def test_userProperties(self): self.assertTrue(result) def test_userGroup(self): - # MethodAllGroup accepts everybody from the right group result = self.authMgr.authQuery("MethodAllGroup", self.userCredDict) self.assertTrue(result) @@ -240,7 +238,6 @@ def test_userGroup(self): self.assertTrue(result) def test_userVO(self): - # MethodAllGroup accepts everybody from the right group result = self.authMgr.authQuery("MethodAllVO", self.userCredDict) self.assertTrue(result) @@ -278,7 +275,6 @@ def test_userVO(self): self.assertFalse(result) def test_hostProperties(self): - # MethodAll accepts everybody result = self.authMgr.authQuery("MethodAll", self.hostCredDict) self.assertTrue(result) diff --git a/src/DIRAC/Core/LCG/GGUSTicketsClient.py b/src/DIRAC/Core/LCG/GGUSTicketsClient.py index e7bf839fb9d..bf6198f2b58 100644 --- a/src/DIRAC/Core/LCG/GGUSTicketsClient.py +++ b/src/DIRAC/Core/LCG/GGUSTicketsClient.py @@ -18,9 +18,9 @@ def getGGUSURL(vo=None, siteName=None): ggusURL += "AFFECTED_SITE&show_columns_check[]=PRIORITY&show_columns_check[]=RESPONSIBLE_UNIT&show_" ggusURL += "columns_check[]=STATUS&show_columns_check[]=DATE_OF_CREATION&show_columns_check[]=LAST_UPDATE&" ggusURL += "show_columns_check[]=TYPE_OF_PROBLEM&show_columns_check[]=SUBJECT&ticket=&supportunit=all&su_" - ggusURL += "hierarchy=all&vo=%s&user=&keyword=&involvedsupporter=&assignto=" % vo + ggusURL += f"hierarchy=all&vo={vo}&user=&keyword=&involvedsupporter=&assignto=" if siteName: - ggusURL += "&affectedsite=%s" % siteName + ggusURL += f"&affectedsite={siteName}" ggusURL += "&specattrib=0&status=open&priority=all&typeofproblem=all&ticketcategory=&mouarea=&technology_" ggusURL += "provider=&date_type=creation+date&radiotf=1&timeframe=any&untouched_date=&orderticketsby=GHD_" ggusURL += "INT_REQUEST_ID&orderhow=descending" @@ -30,7 +30,6 @@ def getGGUSURL(vo=None, siteName=None): class GGUSTicketsClient: def __init__(self): - # create client instance using GGUS wsdl: self.gclient = Client("https://prod-ars.ggus.eu/arsys/WSDL/public/prod-ars/GGUS") authInfo = self.gclient.factory.create("AuthenticationInfo") @@ -53,7 +52,7 @@ def getTicketsList(self, siteName=None, startDate=None, endDate=None): # query = '\'GHD_Affected Site\'=\"' + siteName + '\" AND \'GHD_Affected VO\'="lhcb"' - query = "'GHD_Affected VO'=%s" % self.vo + query = f"'GHD_Affected VO'={self.vo}" if siteName is not None: query += " AND 'GHD_Affected Site'=\"" + siteName + '"' @@ -88,7 +87,6 @@ def globalStatistics(self, ticketList): terminalStates = ["solved", "unsolved", "verified", "closed"] for ticket in ticketList: - _id = str(ticket.GHD_Request_ID) _status = str(ticket.GHD_Status) _shortDescription = str(ticket.GHD_Short_Description) diff --git a/src/DIRAC/Core/LCG/GOCDBClient.py b/src/DIRAC/Core/LCG/GOCDBClient.py index d6e5374920a..5486302efb3 100644 --- a/src/DIRAC/Core/LCG/GOCDBClient.py +++ b/src/DIRAC/Core/LCG/GOCDBClient.py @@ -3,14 +3,15 @@ WARN: the URL of the GOC DB API is hardcoded, and is: https://goc.egi.eu/gocdbpi/public/ """ -import time import socket -import requests - +import time from datetime import datetime, timedelta +from urllib import parse from xml.dom import minidom -from DIRAC import S_OK, S_ERROR, gLogger +import requests + +from DIRAC import S_ERROR, S_OK, gLogger from DIRAC.Core.Security.Locations import getCAsLocation @@ -80,14 +81,14 @@ def getStatus(self, granularity, name=None, startDate=None, startingInHours=None .. code-block:: python {'OK': True, - 'Value': {'92569G0 lhcbsrm-kit.gridka.de': { + 'Value': {'92569G0 lhcbdcache-kit-tape.gridka.de': { 'DESCRIPTION': 'Annual site downtime for various major tasks i...', 'FORMATED_END_DATE': '2014-05-27 15:21', 'FORMATED_START_DATE': '2014-05-26 04:00', 'GOCDB_PORTAL_URL': 'https://goc.egi.eu/portal/index.php?Page_Type=Downtime&id=14051', 'HOSTED_BY': 'FZK-LCG2', - 'HOSTNAME': 'lhcbsrm-kit.gridka.de', - 'SERVICE_TYPE': 'SRM.nearline', + 'HOSTNAME': 'lhcbdcache-kit-tape.gridka.de', + 'SERVICE_TYPE': 'wlcg.webdav.tape', 'SEVERITY': 'OUTAGE'}, '99873G0 srm.pic.esSRM': { 'HOSTED_BY': 'pic', @@ -97,7 +98,7 @@ def getStatus(self, granularity, name=None, startDate=None, startingInHours=None 'URL': 'srm.pic.es', 'GOCDB_PORTAL_URL': 'https://goc.egi.eu/portal/index.php?Page_Type=Downtime&id=21303', 'FORMATED_START_DATE': '2016-09-14 06:00', - 'SERVICE_TYPE': 'SRM', + 'SERVICE_TYPE': 'webdav', 'FORMATED_END_DATE': '2016-09-14 15:00', 'DESCRIPTION': 'Outage declared due to network and dCache upgrades'} } @@ -198,7 +199,6 @@ def getCurrentDTLinkList(self): ############################################################################# def getHostnameDowntime(self, hostname, startDate=None, ongoing=False): - params = hostname if startDate and ongoing: @@ -210,15 +210,15 @@ def getHostnameDowntime(self, hostname, startDate=None, ongoing=False): if ongoing: params += "&ongoing_only=yes" - caPath = getCAsLocation() - try: response = requests.get( - "https://goc.egi.eu/gocdbpi/public/?method=get_downtime&topentity=" + params, verify=caPath + "https://goc.egi.eu/gocdbpi/public/?method=get_downtime&topentity=" + params, + verify=getCAsLocation(), + timeout=20, ) response.raise_for_status() except requests.exceptions.RequestException as e: - return S_ERROR("Error %s" % e) + return S_ERROR(f"Error {e}") return S_OK(response.text) @@ -269,8 +269,7 @@ def _downTimeCurlDownload(self, entity=None, startDate=None): gocdb_ep = gocdb_ep + "&topentity=" + entity gocdb_ep = gocdb_ep + when + gocdbpi_startDate + "&scope=" - caPath = getCAsLocation() - dtPage = requests.get(gocdb_ep, verify=caPath) + dtPage = requests.get(gocdb_ep, verify=getCAsLocation(), timeout=20) dt = dtPage.text @@ -294,8 +293,7 @@ def _getServiceEndpointCurlDownload(self, granularity, entity): # GOCDB-PI query gocdb_ep = "https://goc.egi.eu/gocdbpi/public/?method=get_service_endpoint&" + granularity + "=" + entity - caPath = getCAsLocation() - service_endpoint_page = requests.get(gocdb_ep, verify=caPath) + service_endpoint_page = requests.get(gocdb_ep, verify=getCAsLocation(), timeout=20) return service_endpoint_page.text @@ -346,7 +344,10 @@ def _downTimeXMLParsing(self, dt, siteOrRes, entities=None, startDateMax=None): urls = [] for epElement in affectedEndpoints[0].childNodes: try: - urls.append(_parseSingleElement(epElement, ["URL"])["URL"]) + url = _parseSingleElement(epElement, ["URL"])["URL"] + if "//" not in url: + url = f"//{url}" + urls.append(parse.urlparse(url).hostname) except (IndexError, KeyError): pass except (IndexError, KeyError): @@ -354,10 +355,11 @@ def _downTimeXMLParsing(self, dt, siteOrRes, entities=None, startDateMax=None): try: dtDict[str(dtElement.getAttributeNode("PRIMARY_KEY").nodeValue) + " " + elements["ENDPOINT"]] = elements - dtDict[str(dtElement.getAttributeNode("PRIMARY_KEY").nodeValue) + " " + elements["ENDPOINT"]][ - "URL" - ] = urls[0] - except Exception: + if urls: + dtDict[str(dtElement.getAttributeNode("PRIMARY_KEY").nodeValue) + " " + elements["ENDPOINT"]][ + "URL" + ] = urls[0] + except Exception as e: try: dtDict[ str(dtElement.getAttributeNode("PRIMARY_KEY").nodeValue) + " " + elements["HOSTNAME"] diff --git a/src/DIRAC/Core/LCG/test/Test_LCG.py b/src/DIRAC/Core/LCG/test/Test_LCG.py index 7677a1c0102..4ab3cf9aac5 100644 --- a/src/DIRAC/Core/LCG/test/Test_LCG.py +++ b/src/DIRAC/Core/LCG/test/Test_LCG.py @@ -71,6 +71,32 @@ xml_endpoint_and_affected_ongoing += "" + nowPlus24h + "" xml_endpoint_and_affected_ongoing += "\n" +xml_endpoint_and_affected_ongoing_diffURL = '\n' +xml_endpoint_and_affected_ongoing_diffURL += '' +xml_endpoint_and_affected_ongoing_diffURL += "109962G0" +xml_endpoint_and_affected_ongoing_diffURL += "lhcbsrm-kit.gridka.de" +xml_endpoint_and_affected_ongoing_diffURL += "SRM" +xml_endpoint_and_affected_ongoing_diffURL += "lhcbsrm-kit.gridka.deSRM" +xml_endpoint_and_affected_ongoing_diffURL += "FZK-LCG2" +xml_endpoint_and_affected_ongoing_diffURL += "https://goc.egi.eu/bof" +xml_endpoint_and_affected_ongoing_diffURL += "" +xml_endpoint_and_affected_ongoing_diffURL += "" +xml_endpoint_and_affected_ongoing_diffURL += "7517" +xml_endpoint_and_affected_ongoing_diffURL += "lhcbsrm-disk-kit" +xml_endpoint_and_affected_ongoing_diffURL += "https://lhcbsrm-disk-kit.gridka.de:123" +xml_endpoint_and_affected_ongoing_diffURL += "SRM" +xml_endpoint_and_affected_ongoing_diffURL += "N" +xml_endpoint_and_affected_ongoing_diffURL += "" +xml_endpoint_and_affected_ongoing_diffURL += "" +xml_endpoint_and_affected_ongoing_diffURL += "OUTAGE" +xml_endpoint_and_affected_ongoing_diffURL += "Namespace reordering" +xml_endpoint_and_affected_ongoing_diffURL += "1595233003" +xml_endpoint_and_affected_ongoing_diffURL += "1595314800" +xml_endpoint_and_affected_ongoing_diffURL += "1595343600" +xml_endpoint_and_affected_ongoing_diffURL += "" + nowLess12h + "" +xml_endpoint_and_affected_ongoing_diffURL += "" + nowPlus24h + "" +xml_endpoint_and_affected_ongoing_diffURL += "\n" + xml_endpoint_and_affected_ongoing_broken = '\n' xml_endpoint_and_affected_ongoing_broken += '' xml_endpoint_and_affected_ongoing_broken += "109962G0" @@ -304,7 +330,6 @@ def test__downTimeXMLParsing_affected(): - res = GOCCli._downTimeXMLParsing(xml_endpoint_and_affected_ongoing, "Resource") assert set(res) == {"109962G0 lhcbsrm-kit.gridka.deSRM"} assert res["109962G0 lhcbsrm-kit.gridka.deSRM"]["HOSTNAME"] == "lhcbsrm-kit.gridka.de" @@ -312,6 +337,13 @@ def test__downTimeXMLParsing_affected(): res = GOCCli._downTimeXMLParsing(xml_endpoint_and_affected_ongoing, "Site") assert res == {} + res = GOCCli._downTimeXMLParsing(xml_endpoint_and_affected_ongoing_diffURL, "Resource") + assert set(res) == {"109962G0 lhcbsrm-kit.gridka.deSRM"} + assert res["109962G0 lhcbsrm-kit.gridka.deSRM"]["HOSTNAME"] == "lhcbsrm-kit.gridka.de" + assert res["109962G0 lhcbsrm-kit.gridka.deSRM"]["URL"] == "lhcbsrm-disk-kit.gridka.de" + res = GOCCli._downTimeXMLParsing(xml_endpoint_and_affected_ongoing_diffURL, "Site") + assert res == {} + res = GOCCli._downTimeXMLParsing(xml_endpoint_and_affected_ongoing_broken, "Resource") assert list(res)[0] == "109962G0 lhcbsrm-kit.gridka.deSRM" assert res["109962G0 lhcbsrm-kit.gridka.deSRM"]["HOSTNAME"] == "lhcbsrm-kit.gridka.de" @@ -327,13 +359,12 @@ def test__downTimeXMLParsing_affected(): res = GOCCli._downTimeXMLParsing(xml_endpoint_and_affected_ongoing_2_endpoints, "Resource") assert list(res)[0] == "123 appsgrycap.i3m.upv.eses.upv.grycap.im" assert res["123 appsgrycap.i3m.upv.eses.upv.grycap.im"]["HOSTNAME"] == "appsgrycap.i3m.upv.es" - assert res["123 appsgrycap.i3m.upv.eses.upv.grycap.im"]["URL"] == "https://appsgrycap.i3m.upv.es:31443/im-web/" + assert res["123 appsgrycap.i3m.upv.eses.upv.grycap.im"]["URL"] == "appsgrycap.i3m.upv.es" res = GOCCli._downTimeXMLParsing(xml_endpoint_and_affected_ongoing_2_endpoints, "Site") assert res == {} def test__downTimeXMLParsing(): - res = GOCCli._downTimeXMLParsing(XML_site_ongoing, "Site") assert set(res) == {"28490G0 GRISU-ENEA-GRID"} assert res["28490G0 GRISU-ENEA-GRID"]["SITENAME"] == "GRISU-ENEA-GRID" diff --git a/src/DIRAC/Core/Security/BaseSecurity.py b/src/DIRAC/Core/Security/BaseSecurity.py deleted file mode 100644 index 080e6e79b90..00000000000 --- a/src/DIRAC/Core/Security/BaseSecurity.py +++ /dev/null @@ -1,90 +0,0 @@ -""" Base class for MyProxy and VOMS -""" -import os -import tempfile - -import DIRAC -from DIRAC import gConfig, S_OK, S_ERROR -from DIRAC.Core.Utilities import DErrno -from DIRAC.Core.Security.X509Chain import X509Chain # pylint: disable=import-error -from DIRAC.Core.Security import Locations - - -class BaseSecurity: - def __init__(self, server=False, serverCert=False, serverKey=False, timeout=False): - if timeout: - self._secCmdTimeout = timeout - else: - self._secCmdTimeout = 30 - if not server: - self._secServer = gConfig.getValue("/DIRAC/VOPolicy/MyProxyServer", "myproxy.cern.ch") - else: - self._secServer = server - ckLoc = Locations.getHostCertificateAndKeyLocation() - if serverCert: - self._secCertLoc = serverCert - else: - if ckLoc: - self._secCertLoc = ckLoc[0] - else: - self._secCertLoc = "%s/etc/grid-security/servercert.pem" % DIRAC.rootPath - if serverKey: - self._secKeyLoc = serverKey - else: - if ckLoc: - self._secKeyLoc = ckLoc[1] - else: - self._secKeyLoc = "%s/etc/grid-security/serverkey.pem" % DIRAC.rootPath - self._secRunningFromTrustedHost = gConfig.getValue("/DIRAC/VOPolicy/MyProxyTrustedHost", "True").lower() in ( - "y", - "yes", - "true", - ) - self._secMaxProxyHours = gConfig.getValue("/DIRAC/VOPolicy/MyProxyMaxDelegationTime", 168) - - def getMyProxyServer(self): - return self._secServer - - def getServiceDN(self): - chain = X509Chain() - retVal = chain.loadChainFromFile(self._secCertLoc) - if not retVal["OK"]: - return retVal - return chain.getCertInChain(0)["Value"].getSubjectDN() - - def _getExternalCmdEnvironment(self): - return dict(os.environ) - - def _unlinkFiles(self, files): - if isinstance(files, (list, tuple)): - for fileName in files: - self._unlinkFiles(fileName) - else: - try: - os.unlink(files) - except Exception: - pass - - def _generateTemporalFile(self): - try: - fd, filename = tempfile.mkstemp() - os.close(fd) - except OSError: - return S_ERROR(DErrno.ECTMPF) - return S_OK(filename) - - def _getUsername(self, proxyChain): - retVal = proxyChain.getCredentials() - if not retVal["OK"]: - return retVal - credDict = retVal["Value"] - if not credDict["isProxy"]: - return S_ERROR(DErrno.EX509, "chain does not contain a proxy") - if not credDict["validDN"]: - return S_ERROR(DErrno.EDISET, "DN %s is not known in dirac" % credDict["subject"]) - if not credDict["validGroup"]: - return S_ERROR( - DErrno.EDISET, "Group {} is invalid for DN {}".format(credDict["group"], credDict["subject"]) - ) - mpUsername = "{}:{}".format(credDict["group"], credDict["username"]) - return S_OK(mpUsername) diff --git a/src/DIRAC/Core/Security/DiracX.py b/src/DIRAC/Core/Security/DiracX.py new file mode 100644 index 00000000000..5576d9704df --- /dev/null +++ b/src/DIRAC/Core/Security/DiracX.py @@ -0,0 +1,158 @@ +from __future__ import annotations + +__all__ = ( + "addRPCStub", + "DiracXClient", + "diracxTokenFromPEM", + "executeRPCStub", + "FutureClient", +) + +import base64 +import functools +import hashlib +import importlib +import json +import re +import textwrap +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path +from tempfile import NamedTemporaryFile, gettempdir +from typing import Any + +try: + from diracx.client.sync import SyncDiracClient +except ImportError: + # TODO: Remove this once diracx is tagged + from diracx.client import DiracClient as SyncDiracClient +from diracx.core.models import TokenResponse +from diracx.core.preferences import DiracxPreferences +from diracx.core.utils import serialize_credentials + +from DIRAC import gConfig, gLogger +from DIRAC.Core.Utilities.File import secureOpenForWrite + +from DIRAC.ConfigurationSystem.Client.Helpers import Registry +from DIRAC.Core.Security.Locations import getDefaultProxyLocation +from DIRAC.Core.Utilities.ReturnValues import convertToReturnValue, returnValueOrRaise, isReturnStructure + + +PEM_BEGIN = "-----BEGIN DIRACX-----" +PEM_END = "-----END DIRACX-----" +RE_DIRACX_PEM = re.compile(rf"{PEM_BEGIN}\n(.*)\n{PEM_END}", re.MULTILINE | re.DOTALL) + + +@convertToReturnValue +def addTokenToPEM(pemPath, group): + from DIRAC.Core.Base.Client import Client + + vo = Registry.getVOMSVOForGroup(group) + if not vo: + gLogger.error(f"ERROR: Could not find VO for group {group}, DiracX will not work!") + disabledVOs = gConfig.getValue("/DiracX/DisabledVOs", []) + if vo and vo not in disabledVOs: + token_content = returnValueOrRaise( + Client(url="Framework/ProxyManager", proxyLocation=pemPath).exchangeProxyForToken() + ) + + token = TokenResponse( + access_token=token_content["access_token"], + expires_in=token_content["expires_in"], + token_type=token_content.get("token_type"), + refresh_token=token_content.get("refresh_token"), + ) + + token_pem = f"{PEM_BEGIN}\n" + data = base64.b64encode(serialize_credentials(token).encode("utf-8")).decode() + token_pem += textwrap.fill(data, width=64) + token_pem += f"\n{PEM_END}\n" + + with open(pemPath, "a") as f: + f.write(token_pem) + + +def diracxTokenFromPEM(pemPath) -> dict[str, Any] | None: + """Extract the DiracX token from the proxy PEM file""" + pem = Path(pemPath).read_text() + if match := RE_DIRACX_PEM.search(pem): + match = match.group(1) + return json.loads(base64.b64decode(match).decode("utf-8")) + + +class FutureClient: + """This is just a empty class to make sure that all the FutureClients + inherit from a common class. + """ + + ... + + +@contextmanager +def DiracXClient() -> Iterator[SyncDiracClient]: + """Get a DiracX client instance with the current user's credentials""" + diracxUrl = gConfig.getValue("/DiracX/URL") + if not diracxUrl: + raise ValueError("Missing mandatory /DiracX/URL configuration") + + proxyLocation = getDefaultProxyLocation() + diracxToken = diracxTokenFromPEM(proxyLocation) + if not diracxToken: + raise ValueError(f"No diracx token in the proxy file {proxyLocation}") + + hash = hashlib.sha256(diracxToken["refresh_token"].split(".")[1].encode()) + token_file = Path(gettempdir()) / f"dx_{hash.hexdigest()}" + if not token_file.exists(): + token_file.parent.mkdir(parents=True, exist_ok=True) + with secureOpenForWrite(token_file) as (fd, _): + fd.write(json.dumps(diracxToken)) + + pref = DiracxPreferences(url=diracxUrl, credentials_path=token_file) + with SyncDiracClient(diracx_preferences=pref) as api: + yield api + + +def addRPCStub(meth): + """Decorator to add an rpc like stub to DiracX adapter method + to be called by the ForwardDISET operation + + """ + + @functools.wraps(meth) + def inner(self, *args, **kwargs): + dCls = self.__class__.__name__ + dMod = self.__module__ + res = meth(self, *args, **kwargs) + if isReturnStructure(res): + res["rpcStub"] = { + "dCls": dCls, + "dMod": dMod, + "dMeth": meth.__name__, + "args": args, + "kwargs": kwargs, + } + return res + + return inner + + +def executeRPCStub(stub: dict): + className = stub.get("dCls") + modName = stub.get("dMod") + methName = stub.get("dMeth") + methArgs = stub.get("args") + methKwArgs = stub.get("kwargs") + # Load the module + mod = importlib.import_module(modName) + # import the class + cl = getattr(mod, className) + + # Check that cl is a subclass of JSerializable, + # and that we are not putting ourselves in trouble... + if not (isinstance(cl, type) and issubclass(cl, FutureClient)): + raise TypeError("Only subclasses of FutureClient can be decoded") + + # Instantiate the object + obj = cl() + meth = getattr(obj, methName) + return meth(*methArgs, **methKwArgs) diff --git a/src/DIRAC/Core/Security/IAMService.py b/src/DIRAC/Core/Security/IAMService.py new file mode 100644 index 00000000000..a92789c6815 --- /dev/null +++ b/src/DIRAC/Core/Security/IAMService.py @@ -0,0 +1,173 @@ +""" IAMService class encapsulates connection to the IAM service for a given VO +""" + +import requests + +from DIRAC import S_OK, gConfig, gLogger +from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import getVO + + +def convert_dn(inStr): + """Convert a string separated DN into the slash one, like + CN=Christophe Haen,CN=705305,CN=chaen,OU=Users,OU=Organic Units,DC=cern,DC=ch + /DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=chaen/CN=705305/CN=Christophe Haen + """ + return "/" + "/".join(inStr.split(",")[::-1]) + + +class IAMService: + def __init__(self, access_token, vo=None, forceNickname=False): + """c'tor + + :param str access_token: the token used to talk to IAM, with the scim:read property + :param str vo: name of the virtual organization (community) + :param bool forceNickname: if enforce the presence of a nickname attribute and do not fall back to username in IAM + + """ + self.log = gLogger.getSubLogger(self.__class__.__name__) + + if not access_token: + raise ValueError("access_token not set") + + if vo is None: + vo = getVO() + if not vo: + raise Exception("No VO name given") + + self.forceNickname = forceNickname + + self.vo = vo + + self.iam_url = None + + id_provider = gConfig.getValue(f"/Registry/VO/{self.vo}/IdProvider") + if not id_provider: + raise ValueError(f"/Registry/VO/{self.vo}/IdProvider not found") + result = gConfig.getOptionsDict(f"/Resources/IdProviders/{id_provider}") + if result["OK"]: + self.iam_url = result["Value"]["issuer"] + gLogger.verbose("Using IAM server", self.iam_url) + else: + raise ValueError(f"/Resources/IdProviders/{id_provider}") + + self.userDict = None + self.access_token = access_token + self.iam_users_raw = [] + + def _getIamUserDump(self): + """List the users from IAM""" + + if not self.iam_users_raw: + headers = {"Authorization": f"Bearer {self.access_token}"} + iam_list_url = f"{self.iam_url}/scim/Users" + startIndex = 1 + # These are just initial values, they are updated + # while we loop to their actual values + totalResults = 1000 # total number of users + itemsPerPage = 10 + while startIndex <= totalResults: + resp = requests.get(iam_list_url, headers=headers, params={"startIndex": startIndex}) + resp.raise_for_status() + data = resp.json() + # These 2 should never change while looping + # but you may have a new user appearing + # while looping + totalResults = data["totalResults"] + itemsPerPage = data["itemsPerPage"] + + startIndex += itemsPerPage + self.iam_users_raw.extend(data["Resources"]) + return self.iam_users_raw + + def convert_iam_to_voms(self, iam_output): + """Convert an IAM entry into the voms style, i.e. DN based""" + converted_output = {} + + for cert in iam_output["urn:indigo-dc:scim:schemas:IndigoUser"]["certificates"]: + cert_dict = {} + dn = convert_dn(cert["subjectDn"]) + ca = convert_dn(cert["issuerDn"]) + cert_dict["CA"] = ca + + # The nickname is available in the list of attributes + # (if configured so) + # in the form {'name': 'nickname', 'value': 'chaen'} + # otherwise, we take the userName unless we forceNickname + try: + cert_dict["nickname"] = [ + attr["value"] + for attr in iam_output["urn:indigo-dc:scim:schemas:IndigoUser"]["attributes"] + if attr["name"] == "nickname" + ][0] + except (KeyError, IndexError): + if not self.forceNickname: + cert_dict["nickname"] = iam_output["userName"] + + # This is not correct, we take the overall status instead of the certificate one + # however there are no known case of cert suspended while the user isn't + cert_dict["certSuspended"] = not iam_output["active"] + # There are still bugs in IAM regarding the active status vs voms suspended + + cert_dict["suspended"] = not iam_output["active"] + # The mail may be different, in particular for robot accounts + cert_dict["mail"] = iam_output["emails"][0]["value"].lower() + + # https://github.com/indigo-iam/voms-importer/blob/main/vomsimporter.py + roles = [] + + for role in iam_output["groups"]: + role_name = role["display"] + if "/" in role_name: + role_name = role_name.replace("/", "/Role=") + roles.append(f"/{role_name}") + + cert_dict["Roles"] = roles + converted_output[dn] = cert_dict + return converted_output + + def getUsers(self): + """Extract users from IAM user dump. + + :return: dictionary of: "Users": user dictionary keyed by the user DN, "Errors": list of error messages + """ + self.iam_users_raw = self._getIamUserDump() + users = {} + errors = [] + for user in self.iam_users_raw: + try: + users.update(self.convert_iam_to_voms(user)) + except Exception as e: + errors.append(f"{user['name']} {e!r}") + self.log.error("Could not convert", f"{user['name']} {e!r}") + self.log.error("There were in total", f"{len(errors)} errors") + self.userDict = dict(users) + result = S_OK({"Users": users, "Errors": errors}) + return result + + def getUsersSub(self) -> dict[str, str]: + """ + Return the mapping based on IAM sub: + {nickname : sub} + """ + iam_users_raw = self._getIamUserDump() + diracx_user_section = {} + for user_info in iam_users_raw: + # The nickname is available in the list of attributes + # (if configured so) + # in the form {'name': 'nickname', 'value': 'chaen'} + # otherwise, we take the userName + try: + nickname = [ + attr["value"] + for attr in user_info["urn:indigo-dc:scim:schemas:IndigoUser"]["attributes"] + if attr["name"] == "nickname" + ][0] + except (KeyError, IndexError): + nickname = user_info["userName"] + sub = user_info["id"] + + diracx_user_section[nickname] = sub + # reorder it + diracx_user_section = dict(sorted(diracx_user_section.items())) + + return diracx_user_section diff --git a/src/DIRAC/Core/Security/Locations.py b/src/DIRAC/Core/Security/Locations.py index 4787eecc74f..ba0e2c79d75 100644 --- a/src/DIRAC/Core/Security/Locations.py +++ b/src/DIRAC/Core/Security/Locations.py @@ -1,10 +1,15 @@ """ Collection of utilities for locating certs, proxy, CAs """ import os + import DIRAC from DIRAC import gConfig g_SecurityConfPath = "/DIRAC/Security" +DEFAULT_VOMSES_LOCATION = f"{DIRAC.rootPath}/etc/grid-security/vomses" +SYSTEM_VOMSES_LOCATION = "/etc/vomses" +DEFAULT_VOMSDIR_LOCATION = f"{DIRAC.rootPath}/etc/grid-security/vomsdir" +SYSTEM_VOMSDIR_LOCATION = "/etc/grid-security/vomsdir" def getProxyLocation(): @@ -17,58 +22,72 @@ def getProxyLocation(): return proxyPath # /tmp/x509up_u proxyName = "x509up_u%d" % os.getuid() - if os.path.isfile("/tmp/%s" % proxyName): - return "/tmp/%s" % proxyName - - # No gridproxy found - return False - - -# Retrieve CA's location + if os.path.isfile(f"/tmp/{proxyName}"): + return f"/tmp/{proxyName}" def getCAsLocation(): """Retrieve the CA's files location""" # Grid-Security - retVal = gConfig.getOption("%s/Grid-Security" % g_SecurityConfPath) + retVal = gConfig.getOption(f"{g_SecurityConfPath}/Grid-Security") if retVal["OK"]: - casPath = "%s/certificates" % retVal["Value"] + casPath = f"{retVal['Value']}/certificates" if os.path.isdir(casPath): return casPath # CAPath - retVal = gConfig.getOption("%s/CALocation" % g_SecurityConfPath) + retVal = gConfig.getOption(f"{g_SecurityConfPath}/CALocation") if retVal["OK"]: casPath = retVal["Value"] if os.path.isdir(casPath): return casPath + # Other locations + return getCAsLocationNoConfig() + + +def getCAsLocationNoConfig(): + """Retrieve the CA's files location""" # Look up the X509_CERT_DIR environment variable if "X509_CERT_DIR" in os.environ: casPath = os.environ["X509_CERT_DIR"] return casPath # rootPath./etc/grid-security/certificates - casPath = "%s/etc/grid-security/certificates" % DIRAC.rootPath + casPath = f"{DIRAC.rootPath}/etc/grid-security/certificates" if os.path.isdir(casPath): return casPath # /etc/grid-security/certificates casPath = "/etc/grid-security/certificates" if os.path.isdir(casPath): return casPath - # No CA's location found - return False - - -# Retrieve CA's location - - -def getCAsDefaultLocation(): - """Retrievethe CAs Location inside DIRAC etc directory""" # rootPath./etc/grid-security/certificates - casPath = "%s/etc/grid-security/certificates" % DIRAC.rootPath - return casPath + casPath = f"{DIRAC.rootPath}/etc/grid-security/certificates" + if os.path.isdir(casPath): + return casPath -# TODO: Static depending on files specified on CS -# Retrieve certificate +def getVOMSLocation(): + """Retrieve the CA's files location""" + # Grid-Security + retVal = gConfig.getOption(f"{g_SecurityConfPath}/Grid-Security") + if retVal["OK"]: + vomsPath = f"{retVal['Value']}/vomsdir" + if os.path.isdir(vomsPath): + return vomsPath + # Look up the X509_VOMS_DIR environment variable + if "X509_VOMS_DIR" in os.environ: + vomsPath = os.environ["X509_VOMS_DIR"] + return vomsPath + # rootPath./etc/grid-security/vomsdir + vomsPath = f"{DIRAC.rootPath}/etc/grid-security/vomsdir" + if os.path.isdir(vomsPath): + return vomsPath + # /etc/grid-security/vomsdir + vomsPath = "/etc/grid-security/vomsdir" + if os.path.isdir(vomsPath): + return vomsPath + # rootPath./etc/grid-security/vomsdir + vomsPath = f"{DIRAC.rootPath}/etc/grid-security/vomsdir" + if os.path.isdir(vomsPath): + return vomsPath def getHostCertificateAndKeyLocation(specificLocation=None): @@ -111,10 +130,10 @@ def getHostCertificateAndKeyLocation(specificLocation=None): for filePrefix in ("server", "host", "dirac", "service"): # Possible grid-security's paths = [] - retVal = gConfig.getOption("%s/Grid-Security" % g_SecurityConfPath) + retVal = gConfig.getOption(f"{g_SecurityConfPath}/Grid-Security") if retVal["OK"]: paths.append(retVal["Value"]) - paths.append("%s/etc/grid-security/" % DIRAC.rootPath) + paths.append(f"{DIRAC.rootPath}/etc/grid-security/") for path in paths: filePath = os.path.realpath(f"{path}/{filePrefix}{fileType}.pem") if os.path.isfile(filePath): @@ -140,9 +159,9 @@ def getCertificateAndKeyLocation(): if "X509_USER_CERT" in os.environ: if os.path.exists(os.environ["X509_USER_CERT"]): certfile = os.environ["X509_USER_CERT"] - if not certfile: - if os.path.exists(os.environ["HOME"] + "/.globus/usercert.pem"): - certfile = os.environ["HOME"] + "/.globus/usercert.pem" + if not certfile and (home := os.environ.get("HOME")): + if os.path.exists(home + "/.globus/usercert.pem"): + certfile = home + "/.globus/usercert.pem" if not certfile: return False @@ -151,9 +170,9 @@ def getCertificateAndKeyLocation(): if "X509_USER_KEY" in os.environ: if os.path.exists(os.environ["X509_USER_KEY"]): keyfile = os.environ["X509_USER_KEY"] - if not keyfile: - if os.path.exists(os.environ["HOME"] + "/.globus/userkey.pem"): - keyfile = os.environ["HOME"] + "/.globus/userkey.pem" + if not keyfile and (home := os.environ.get("HOME")): + if os.path.exists(home + "/.globus/userkey.pem"): + keyfile = home + "/.globus/userkey.pem" if not keyfile: return False @@ -171,4 +190,36 @@ def getDefaultProxyLocation(): # /tmp/x509up_u proxyName = "x509up_u%d" % os.getuid() - return "/tmp/%s" % proxyName + return f"/tmp/{proxyName}" + + +def getVomsesLocation(): + """Get the location of the directory containing the vomses files""" + if "X509_VOMSES" in os.environ: + return os.environ["X509_VOMSES"] + elif os.path.isdir(DEFAULT_VOMSES_LOCATION): + return DEFAULT_VOMSES_LOCATION + elif os.path.isdir(SYSTEM_VOMSES_LOCATION): + return SYSTEM_VOMSES_LOCATION + else: + raise Exception( + "The env variable X509_VOMSES is not set. " + "DIRAC does not know where to look for etc/grid-security/vomses. " + "Please set X509_VOMSES." + ) + + +def getVomsdirLocation(): + """Get the location of the directory containing the vomsdir files""" + if "X509_VOMS_DIR" in os.environ: + return os.environ["X509_VOMS_DIR"] + elif os.path.isdir(DEFAULT_VOMSDIR_LOCATION): + return DEFAULT_VOMSDIR_LOCATION + elif os.path.isdir(SYSTEM_VOMSDIR_LOCATION): + return SYSTEM_VOMSDIR_LOCATION + else: + raise Exception( + "The env variable X509_VOMS_DIR is not set. " + "DIRAC does not know where to look for etc/grid-security/vomsdir. " + "Please set X509_VOMS_DIR." + ) diff --git a/src/DIRAC/Core/Security/MyProxy.py b/src/DIRAC/Core/Security/MyProxy.py deleted file mode 100644 index e64c091be76..00000000000 --- a/src/DIRAC/Core/Security/MyProxy.py +++ /dev/null @@ -1,234 +0,0 @@ -""" Utility class for dealing with MyProxy -""" - -import re -from DIRAC import gLogger, S_OK, S_ERROR -from DIRAC.Core.Utilities.Subprocess import shellCall -from DIRAC.Core.Utilities import List -from DIRAC.Core.Security.ProxyFile import multiProxyArgument, deleteMultiProxy -from DIRAC.Core.Security.X509Chain import X509Chain # pylint: disable=import-error -from DIRAC.Core.Security.BaseSecurity import BaseSecurity - - -class MyProxy(BaseSecurity): - def uploadProxy(self, proxy=False, useDNAsUserName=False): - """ - Upload a proxy to myproxy service. - proxy param can be: - : Default -> use current proxy - : string -> upload file specified as proxy - : X509Chain -> use chain - """ - retVal = multiProxyArgument(proxy) - if not retVal["OK"]: - return retVal - proxyDict = retVal["Value"] - chain = proxyDict["chain"] - proxyLocation = proxyDict["file"] - - timeLeft = int(chain.getRemainingSecs()["Value"] / 3600) - - cmdArgs = ["-n"] - cmdArgs.append('-s "%s"' % self._secServer) - cmdArgs.append('-c "%s"' % (timeLeft - 1)) - cmdArgs.append('-t "%s"' % self._secMaxProxyHours) - cmdArgs.append('-C "%s"' % proxyLocation) - cmdArgs.append('-y "%s"' % proxyLocation) - if useDNAsUserName: - cmdArgs.append("-d") - else: - retVal = self._getUsername(chain) - if not retVal["OK"]: - deleteMultiProxy(proxyDict) - return retVal - mpUsername = retVal["Value"] - cmdArgs.append('-l "%s"' % mpUsername) - - mpEnv = self._getExternalCmdEnvironment() - # Hack to upload properly - mpEnv["GT_PROXY_MODE"] = "old" - - cmd = "myproxy-init %s" % " ".join(cmdArgs) - result = shellCall(self._secCmdTimeout, cmd, env=mpEnv) - - deleteMultiProxy(proxyDict) - - if not result["OK"]: - errMsg = "Call to myproxy-init failed: %s" % retVal["Message"] - return S_ERROR(errMsg) - - status, _, error = result["Value"] - - # Clean-up files - if status: - errMsg = "Call to myproxy-init failed" - extErrMsg = f"Command: {cmd}; StdOut: {result}; StdErr: {error}" - return S_ERROR(f"{errMsg} {extErrMsg}") - - return S_OK() - - def getDelegatedProxy(self, proxyChain, lifeTime=604800, useDNAsUserName=False): - """ - Get delegated proxy from MyProxy server - return S_OK( X509Chain ) / S_ERROR - """ - # TODO: Set the proxy coming in proxyString to be the proxy to use - - # Get myproxy username diracgroup:diracuser - retVal = multiProxyArgument(proxyChain) - if not retVal["OK"]: - return retVal - proxyDict = retVal["Value"] - chain = proxyDict["chain"] - proxyLocation = proxyDict["file"] - - retVal = self._generateTemporalFile() - if not retVal["OK"]: - deleteMultiProxy(proxyDict) - return retVal - newProxyLocation = retVal["Value"] - - # myproxy-get-delegation works only with environment variables - cmdEnv = self._getExternalCmdEnvironment() - if self._secRunningFromTrustedHost: - cmdEnv["X509_USER_CERT"] = self._secCertLoc - cmdEnv["X509_USER_KEY"] = self._secKeyLoc - if "X509_USER_PROXY" in cmdEnv: - del cmdEnv["X509_USER_PROXY"] - else: - cmdEnv["X509_USER_PROXY"] = proxyLocation - - cmdArgs = [] - cmdArgs.append("-s '%s'" % self._secServer) - cmdArgs.append("-t '%s'" % (int(lifeTime / 3600))) - cmdArgs.append("-a '%s'" % proxyLocation) - cmdArgs.append("-o '%s'" % newProxyLocation) - if useDNAsUserName: - cmdArgs.append("-d") - else: - retVal = self._getUsername(chain) - if not retVal["OK"]: - deleteMultiProxy(proxyDict) - return retVal - mpUsername = retVal["Value"] - cmdArgs.append('-l "%s"' % mpUsername) - - cmd = "myproxy-logon %s" % " ".join(cmdArgs) - gLogger.verbose("myproxy-logon command:\n%s" % cmd) - - result = shellCall(self._secCmdTimeout, cmd, env=cmdEnv) - - deleteMultiProxy(proxyDict) - - if not result["OK"]: - errMsg = "Call to myproxy-logon failed: %s" % result["Message"] - deleteMultiProxy(proxyDict) - return S_ERROR(errMsg) - - status, _, error = result["Value"] - - # Clean-up files - if status: - errMsg = "Call to myproxy-logon failed" - extErrMsg = f"Command: {cmd}; StdOut: {result}; StdErr: {error}" - deleteMultiProxy(proxyDict) - return S_ERROR(f"{errMsg} {extErrMsg}") - - chain = X509Chain() - retVal = chain.loadProxyFromFile(newProxyLocation) - if not retVal["OK"]: - deleteMultiProxy(proxyDict) - return S_ERROR("myproxy-logon failed when reading delegated file: %s" % retVal["Message"]) - - deleteMultiProxy(proxyDict) - return S_OK(chain) - - def getInfo(self, proxyChain, useDNAsUserName=False): - """ - Get info from myproxy server - - :return: S_OK( { 'username' : myproxyusername, - 'owner' : owner DN, - 'timeLeft' : secs left } ) / S_ERROR - """ - # TODO: Set the proxy coming in proxyString to be the proxy to use - - # Get myproxy username diracgroup:diracuser - retVal = multiProxyArgument(proxyChain) - if not retVal["OK"]: - return retVal - proxyDict = retVal["Value"] - chain = proxyDict["chain"] - proxyLocation = proxyDict["file"] - - # myproxy-get-delegation works only with environment variables - cmdEnv = self._getExternalCmdEnvironment() - if self._secRunningFromTrustedHost: - cmdEnv["X509_USER_CERT"] = self._secCertLoc - cmdEnv["X509_USER_KEY"] = self._secKeyLoc - if "X509_USER_PROXY" in cmdEnv: - del cmdEnv["X509_USER_PROXY"] - else: - cmdEnv["X509_USER_PROXY"] = proxyLocation - - cmdArgs = [] - cmdArgs.append("-s '%s'" % self._secServer) - if useDNAsUserName: - cmdArgs.append("-d") - else: - retVal = self._getUsername(chain) - if not retVal["OK"]: - deleteMultiProxy(proxyDict) - return retVal - mpUsername = retVal["Value"] - cmdArgs.append('-l "%s"' % mpUsername) - - cmd = "myproxy-info %s" % " ".join(cmdArgs) - gLogger.verbose("myproxy-info command:\n%s" % cmd) - - result = shellCall(self._secCmdTimeout, cmd, env=cmdEnv) - - deleteMultiProxy(proxyDict) - - if not result["OK"]: - errMsg = "Call to myproxy-info failed: %s" % result["Message"] - deleteMultiProxy(proxyDict) - return S_ERROR(errMsg) - - status, output, error = result["Value"] - - # Clean-up files - if status: - errMsg = "Call to myproxy-info failed" - extErrMsg = f"Command: {cmd}; StdOut: {result}; StdErr: {error}" - return S_ERROR(f"{errMsg} {extErrMsg}") - - infoDict = {} - usernameRE = re.compile(r"username\s*:\s*(\S*)") - ownerRE = re.compile(r"owner\s*:\s*(\S*)") - timeLeftRE = re.compile(r"timeleft\s*:\s*(\S*)") - for line in List.fromChar(output, "\n"): - match = usernameRE.search(line) - if match: - infoDict["username"] = match.group(1) - match = ownerRE.search(line) - if match: - infoDict["owner"] = match.group(1) - match = timeLeftRE.search(line) - if match: - try: - fields = List.fromChar(match.group(1), ":") - fields.reverse() - secsLeft = 0 - for iP in range(len(fields)): - if iP == 0: - secsLeft += int(fields[iP]) - elif iP == 1: - secsLeft += int(fields[iP]) * 60 - elif iP == 2: - secsLeft += int(fields[iP]) * 3600 - infoDict["timeLeft"] = secsLeft - except Exception as x: - print(x) - - return S_OK(infoDict) diff --git a/src/DIRAC/Core/Security/Properties.py b/src/DIRAC/Core/Security/Properties.py index 35496360ff4..0ca629242bc 100644 --- a/src/DIRAC/Core/Security/Properties.py +++ b/src/DIRAC/Core/Security/Properties.py @@ -5,7 +5,7 @@ import operator from enum import Enum -from typing import Union +from typing import Callable, Union class SecurityProperty(str, Enum): @@ -26,8 +26,6 @@ class SecurityProperty(str, Enum): JOB_MONITOR = "JobMonitor" #: Accounting Monitor - can see accounting data for all groups ACCOUNTING_MONITOR = "AccountingMonitor" - #: Private pilot - PILOT = "Pilot" #: Generic pilot GENERIC_PILOT = "GenericPilot" #: Site Manager @@ -44,20 +42,20 @@ class SecurityProperty(str, Enum): PRIVATE_LIMITED_DELEGATION = "PrivateLimitedDelegation" #: Allow managing proxies PROXY_MANAGEMENT = "ProxyManagement" - #: Allow managing production + #: Allow managing all productions PRODUCTION_MANAGEMENT = "ProductionManagement" + #: Allow managing all productions in the same group + PRODUCTION_SHARING = "ProductionSharing" + #: Allows user to manage productions they own only + PRODUCTION_USER = "ProductionUser" #: Allow production request approval on behalf of PPG PPG_AUTHORITY = "PPGAuthority" #: Allow Bookkeeping Management BOOKKEEPING_MANAGEMENT = "BookkeepingManagement" - #: Allow to set notifications and manage alarms - ALARMS_MANAGEMENT = "AlarmsManagement" #: Allow FC Management - FC root user FC_MANAGEMENT = "FileCatalogManagement" #: Allow staging files STAGE_ALLOWED = "StageAllowed" - #: Allow VMDIRAC Operations via various handlers - VM_RPC_OPERATION = "VmRpcOperation" def __str__(self) -> str: return str(self.name) @@ -65,17 +63,17 @@ def __str__(self) -> str: def __repr__(self) -> str: return f"{self.__class__.__name__}.{self.name}" - def __and__(self, value: Union[SecurityProperty, UnevaluatedProperty]) -> UnevaluatedExpression: + def __and__(self, value: SecurityProperty | UnevaluatedProperty) -> UnevaluatedExpression: if not isinstance(value, UnevaluatedProperty): value = UnevaluatedProperty(value) return UnevaluatedProperty(self) & value - def __or__(self, value: Union[SecurityProperty, UnevaluatedProperty]) -> UnevaluatedExpression: + def __or__(self, value: SecurityProperty | UnevaluatedProperty) -> UnevaluatedExpression: if not isinstance(value, UnevaluatedProperty): value = UnevaluatedProperty(value) return UnevaluatedProperty(self) | value - def __xor__(self, value: Union[SecurityProperty, UnevaluatedProperty]) -> UnevaluatedExpression: + def __xor__(self, value: SecurityProperty | UnevaluatedProperty) -> UnevaluatedExpression: if not isinstance(value, UnevaluatedProperty): value = UnevaluatedProperty(value) return UnevaluatedProperty(self) ^ value @@ -94,7 +92,7 @@ def __str__(self) -> str: def __repr__(self) -> str: return repr(self.property) - def __call__(self, allowed_properties: list[SecurityProperty]): + def __call__(self, allowed_properties: list[SecurityProperty]) -> bool: return self.property in allowed_properties def __and__(self, value: UnevaluatedProperty) -> UnevaluatedExpression: @@ -111,7 +109,7 @@ def __invert__(self) -> UnevaluatedExpression: class UnevaluatedExpression(UnevaluatedProperty): - def __init__(self, operator: callable, *args: list[UnevaluatedProperty]): + def __init__(self, operator: Callable[..., bool], *args: UnevaluatedProperty): self.operator = operator self.args = args @@ -128,7 +126,7 @@ def __str__(self) -> str: def __repr__(self) -> str: return f"{self.operator.__name__}({', '.join(map(repr, self.args))})" - def __call__(self, properties): + def __call__(self, properties: list[SecurityProperty]) -> bool: return self.operator(*(a(properties) for a in self.args)) @@ -141,7 +139,6 @@ def __call__(self, properties): JOB_ADMINISTRATOR = SecurityProperty.JOB_ADMINISTRATOR.value JOB_MONITOR = SecurityProperty.JOB_MONITOR.value ACCOUNTING_MONITOR = SecurityProperty.ACCOUNTING_MONITOR.value -PILOT = SecurityProperty.PILOT.value GENERIC_PILOT = SecurityProperty.GENERIC_PILOT.value SITE_MANAGER = SecurityProperty.SITE_MANAGER.value USER_MANAGER = SecurityProperty.USER_MANAGER.value @@ -153,7 +150,5 @@ def __call__(self, properties): PRODUCTION_MANAGEMENT = SecurityProperty.PRODUCTION_MANAGEMENT.value PPG_AUTHORITY = SecurityProperty.PPG_AUTHORITY.value BOOKKEEPING_MANAGEMENT = SecurityProperty.BOOKKEEPING_MANAGEMENT.value -ALARMS_MANAGEMENT = SecurityProperty.ALARMS_MANAGEMENT.value FC_MANAGEMENT = SecurityProperty.FC_MANAGEMENT.value STAGE_ALLOWED = SecurityProperty.STAGE_ALLOWED.value -VM_RPC_OPERATION = SecurityProperty.VM_RPC_OPERATION.value diff --git a/src/DIRAC/Core/Security/ProxyFile.py b/src/DIRAC/Core/Security/ProxyFile.py index cd031025eb9..d5218f00734 100644 --- a/src/DIRAC/Core/Security/ProxyFile.py +++ b/src/DIRAC/Core/Security/ProxyFile.py @@ -1,11 +1,12 @@ """ Collection of utilities for dealing with security files (i.e. proxy files) """ import os -import stat import tempfile from DIRAC import S_OK, S_ERROR from DIRAC.Core.Utilities import DErrno +from DIRAC.Core.Security.DiracX import addTokenToPEM +from DIRAC.Core.Utilities.File import secureOpenForWrite from DIRAC.Core.Security.X509Chain import X509Chain # pylint: disable=import-error from DIRAC.Core.Security.Locations import getProxyLocation @@ -17,22 +18,24 @@ def writeToProxyFile(proxyContents, fileName=False): - proxyContents : string object to dump to file - fileName : filename to dump to """ - if not fileName: - try: - fd, proxyLocation = tempfile.mkstemp() - os.close(fd) - except OSError: - return S_ERROR(DErrno.ECTMPF) - fileName = proxyLocation try: - with open(fileName, "w") as fd: + with secureOpenForWrite(fileName) as (fd, fileName): fd.write(proxyContents) except Exception as e: - return S_ERROR(DErrno.EWF, " {}: {}".format(fileName, repr(e).replace(",)", ")"))) - try: - os.chmod(fileName, stat.S_IRUSR | stat.S_IWUSR) - except Exception as e: - return S_ERROR(DErrno.ESPF, "{}: {}".format(fileName, repr(e).replace(",)", ")"))) + return S_ERROR(DErrno.EWF, f" {fileName}: {repr(e).replace(',)', ')')}") + + # Add DiracX token to the file + proxy = X509Chain() + retVal = proxy.loadProxyFromFile(fileName) + if not retVal["OK"]: + return S_ERROR(DErrno.EPROXYREAD, f"ProxyLocation: {fileName}") + retVal = proxy.getDIRACGroup(ignoreDefault=True) + if not retVal["OK"]: + return S_ERROR(DErrno.EPROXYREAD, f"No DIRAC group found in proxy: {fileName}") + retVal = addTokenToPEM(fileName, retVal["Value"]) # pylint: disable=unsubscriptable-object + if not retVal["OK"]: # pylint: disable=unsubscriptable-object + return retVal + return S_OK(fileName) @@ -118,9 +121,11 @@ def multiProxyArgument(proxy=False): return S_ERROR(DErrno.EPROXYFIND) if isinstance(proxy, str): proxyLoc = proxy + else: + raise NotImplementedError(f"Unknown proxy type ({type(proxy)})") # Load proxy proxy = X509Chain() retVal = proxy.loadProxyFromFile(proxyLoc) if not retVal["OK"]: - return S_ERROR(DErrno.EPROXYREAD, "ProxyLocation: %s" % proxyLoc) + return S_ERROR(DErrno.EPROXYREAD, f"ProxyLocation: {proxyLoc}") return S_OK({"file": proxyLoc, "chain": proxy, "tempFile": tempFile}) diff --git a/src/DIRAC/Core/Security/ProxyInfo.py b/src/DIRAC/Core/Security/ProxyInfo.py index 80a30e9b57e..1434c139965 100644 --- a/src/DIRAC/Core/Security/ProxyInfo.py +++ b/src/DIRAC/Core/Security/ProxyInfo.py @@ -3,13 +3,13 @@ """ import base64 -from DIRAC import S_OK, S_ERROR, gLogger -from DIRAC.Core.Utilities import DErrno -from DIRAC.Core.Security.X509Chain import X509Chain # pylint: disable=import-error -from DIRAC.Core.Security.VOMS import VOMS -from DIRAC.Core.Security import Locations - +from DIRAC import S_ERROR, S_OK, gLogger from DIRAC.ConfigurationSystem.Client.Helpers import Registry +from DIRAC.Core.Security import Locations +from DIRAC.Core.Security.DiracX import diracxTokenFromPEM +from DIRAC.Core.Security.VOMS import VOMS +from DIRAC.Core.Security.X509Chain import X509Chain # pylint: disable=import-error +from DIRAC.Core.Utilities import DErrno def getProxyInfo(proxy=False, disableVOMS=False): @@ -25,9 +25,11 @@ def getProxyInfo(proxy=False, disableVOMS=False): * 'validDN' : Valid DN in DIRAC * 'validGroup' : Valid Group in DIRAC * 'secondsLeft' : Seconds left + * 'hasDiracxToken' * values that can be there * 'path' : path to the file, * 'group' : DIRAC group + * 'VO' : DIRAC VO * 'groupProperties' : Properties that apply to the DIRAC Group * 'username' : DIRAC username * 'identity' : DN that generated the proxy @@ -49,7 +51,7 @@ def getProxyInfo(proxy=False, disableVOMS=False): chain = X509Chain() retVal = chain.loadProxyFromFile(proxyLocation) if not retVal["OK"]: - return S_ERROR(DErrno.EPROXYREAD, "{}: {} ".format(proxyLocation, retVal["Message"])) + return S_ERROR(DErrno.EPROXYREAD, f"{proxyLocation}: {retVal['Message']} ") retVal = chain.getCredentials() if not retVal["OK"]: @@ -67,6 +69,14 @@ def getProxyInfo(proxy=False, disableVOMS=False): infoDict["VOMS"] = retVal["Value"] else: infoDict["VOMSError"] = retVal["Message"].strip() + + if "group" in infoDict: + infoDict["VO"] = Registry.getVOForGroup(infoDict["group"]) + + infoDict["hasDiracxToken"] = False + if proxyLocation: + infoDict["hasDiracxToken"] = bool(diracxTokenFromPEM(proxyLocation)) + return S_OK(infoDict) @@ -91,9 +101,9 @@ def formatProxyInfoAsString(infoDict): "subject", "issuer", "identity", - "subproxyUser", ("secondsLeft", "timeleft"), ("group", "DIRAC group"), + ("hasDiracxToken", "DiracX"), "rfc", "path", "username", @@ -156,7 +166,7 @@ def formatProxyStepsInfoAsString(infoList): """ contentsList = [] for i in range(len(infoList)): - contentsList.append(" + Step %s" % i) + contentsList.append(f" + Step {i}") stepInfo = infoList[i] for key in ( "subject", @@ -175,7 +185,7 @@ def formatProxyStepsInfoAsString(infoList): try: # b16encode needs a string, while the serial # may be a long - value = base64.b16encode("%s" % value) + value = base64.b16encode(f"{value}") except Exception as e: gLogger.exception("Could not read serial:", lException=e) if key == "lifetime": diff --git a/src/DIRAC/Core/Security/Utilities.py b/src/DIRAC/Core/Security/Utilities.py index 9231dda0aef..5149848e0af 100644 --- a/src/DIRAC/Core/Security/Utilities.py +++ b/src/DIRAC/Core/Security/Utilities.py @@ -76,7 +76,7 @@ def generateCAFile(location=None): continue fd.write(chain.dumpAllToString()["Value"]) - gLogger.info("CAs used from: %s" % str(fn)) + gLogger.info(f"CAs used from: {str(fn)}") return S_OK(fn) except OSError as err: gLogger.warn(err) diff --git a/src/DIRAC/Core/Security/VOMS.py b/src/DIRAC/Core/Security/VOMS.py index 1ca8edf8d2f..578bd7f1536 100644 --- a/src/DIRAC/Core/Security/VOMS.py +++ b/src/DIRAC/Core/Security/VOMS.py @@ -1,30 +1,63 @@ """ Module for dealing with VOMS (Virtual Organization Membership Service) """ + from datetime import datetime import os -import stat import tempfile import shlex import shutil from DIRAC import S_OK, S_ERROR, gConfig, rootPath, gLogger from DIRAC.Core.Utilities import DErrno +from DIRAC.Core.Security import Locations from DIRAC.Core.Security.ProxyFile import multiProxyArgument, deleteMultiProxy -from DIRAC.Core.Security.BaseSecurity import BaseSecurity from DIRAC.Core.Security.X509Chain import X509Chain # pylint: disable=import-error from DIRAC.Core.Utilities.Subprocess import shellCall from DIRAC.Core.Utilities import List +# This is a variable so it can be monkeypatched in tests +VOMS_PROXY_INIT_CMD = "voms-proxy-init" + + +def voms_init_cmd( + vo: str, attribute: str | None, chain: X509Chain, in_fn: str, out_fn: str, vomsesPath: str | None +) -> list[str]: + secs = chain.getRemainingSecs()["Value"] - 300 + if secs < 0: + return S_ERROR(DErrno.EVOMS, "Proxy length is less that 300 secs") + hours = int(secs / 3600) + mins = int((secs - hours * 3600) / 60) + + bitStrength = chain.getStrength()["Value"] + + cmd = [VOMS_PROXY_INIT_CMD] + if chain.isLimitedProxy()["Value"]: + cmd.append("-limited") + cmd += ["-cert", in_fn] + cmd += ["-key", in_fn] + cmd += ["-out", out_fn] + cmd += ["-voms"] + cmd += [f"{vo}:{attribute}" if attribute and attribute != "NoRole" else vo] + cmd += ["-valid", f"{hours}:{mins}"] + cmd += ["-bits", str(bitStrength)] + if vomsesPath: + cmd += ["-vomses", vomsesPath] -class VOMS(BaseSecurity): - def __init__(self, timeout=80, *args, **kwargs): + if chain.isRFC().get("Value"): + cmd += ["-r"] + cmd += ["-timeout", "12"] + + return cmd + + +class VOMS: + def __init__(self, *args, **kwargs): """Create VOMS class, setting specific timeout for VOMS shell commands.""" # Per-server timeout for voms-proxy-init, should be at maximum timeout/2*n # where n as the number of voms servers to try. # voms-proxy-init will try each server *twice* before moving to the next one # once for new interface mode, once for legacy. - self._servTimeout = 12 - super().__init__(timeout=timeout, *args, **kwargs) + self._secCmdTimeout = 80 def getVOMSAttributes(self, proxy, switch="all"): """ @@ -38,7 +71,7 @@ def getVOMSAttributes(self, proxy, switch="all"): # Get all possible info from voms proxy result = self.getVOMSProxyInfo(proxy, "all") if not result["OK"]: - return S_ERROR(DErrno.EVOMS, "Failed to extract info from proxy: %s" % result["Message"]) + return S_ERROR(DErrno.EVOMS, f"Failed to extract info from proxy: {result['Message']}") vomsInfoOutput = List.fromChar(result["Value"], "\n") @@ -47,7 +80,7 @@ def getVOMSAttributes(self, proxy, switch="all"): result = gConfig.getSections("/Registry/Groups") if result["OK"]: for group in result["Value"]: - vA = gConfig.getValue("/Registry/Groups/%s/VOMSRole" % group, "") + vA = gConfig.getValue(f"/Registry/Groups/{group}/VOMSRole", "") if vA and vA not in validVOMSAttrs: validVOMSAttrs.append(vA) @@ -85,6 +118,8 @@ def getVOMSAttributes(self, proxy, switch="all"): returnValue = nickName elif switch == "all": returnValue = attributes + else: + raise NotImplementedError(switch) return S_OK(returnValue) @@ -118,7 +153,7 @@ def getVOMSProxyInfo(self, proxy, option=False): """ validOptions = ["actimeleft", "timeleft", "identity", "fqan", "all"] if option and option not in validOptions: - return S_ERROR(DErrno.EVOMS, "invalid option %s" % option) + return S_ERROR(DErrno.EVOMS, f"invalid option {option}") retVal = multiProxyArgument(proxy) if not retVal["OK"]: @@ -141,7 +176,7 @@ def getVOMSProxyInfo(self, proxy, option=False): left = proxyDict["chain"].getNotAfterDate()["Value"] - now return S_OK("%d\n" % left.total_seconds()) if option == "identity": - return S_OK("%s\n" % data["subject"]) + return S_OK(f"{data['subject']}\n") if option == "fqan": return S_OK( "\n".join([f.replace("/Role=NULL", "").replace("/Capability=NULL", "") for f in data["fqan"]]) @@ -149,9 +184,9 @@ def getVOMSProxyInfo(self, proxy, option=False): if option == "all": lines = [] creds = proxyDict["chain"].getCredentials()["Value"] - lines.append("subject : %s" % creds["subject"]) - lines.append("issuer : %s" % creds["issuer"]) - lines.append("identity : %s" % creds["identity"]) + lines.append(f"subject : {creds['subject']}") + lines.append(f"issuer : {creds['issuer']}") + lines.append(f"identity : {creds['identity']}") if proxyDict["chain"].isRFC().get("Value"): lines.append("type : RFC compliant proxy") else: @@ -164,14 +199,14 @@ def getVOMSProxyInfo(self, proxy, option=False): "timeleft : %s:%s:%s\nkey usage : Digital Signature, Key Encipherment, Data Encipherment" % (h, m, s) ) - lines.append("== VO %s extension information ==" % data["vo"]) - lines.append("VO: %s" % data["vo"]) - lines.append("subject : %s" % data["subject"]) - lines.append("issuer : %s" % data["issuer"]) + lines.append(f"== VO {data['vo']} extension information ==") + lines.append(f"VO: {data['vo']}") + lines.append(f"subject : {data['subject']}") + lines.append(f"issuer : {data['issuer']}") for fqan in data["fqan"]: - lines.append("attribute : %s" % fqan) + lines.append(f"attribute : {fqan}") if "attribute" in data: - lines.append("attribute : %s" % data["attribute"]) + lines.append(f"attribute : {data['attribute']}") now = datetime.utcnow() left = (data["notAfter"] - now).total_seconds() h = int(left / 3600) @@ -188,34 +223,7 @@ def getVOMSProxyInfo(self, proxy, option=False): self._unlinkFiles(proxyDict["file"]) def getVOMSESLocation(self): - # Transition code to new behaviour - if "DIRAC_VOMSES" not in os.environ and "X509_VOMSES" not in os.environ: - os.environ["X509_VOMSES"] = os.path.join(rootPath, "etc", "grid-security", "vomses") - gLogger.notice( - "You did not set X509_VOMSES in your bashrc. DIRAC searches $DIRAC/etc/grid-security/vomses . " - "Please use X509_VOMSES, this auto discovery will be dropped." - ) - elif "DIRAC_VOMSES" in os.environ and "X509_VOMSES" in os.environ: - os.environ["X509_VOMSES"] = "{}:{}".format(os.environ["DIRAC_VOMSES"], os.environ["X509_VOMSES"]) - gLogger.notice( - "You set both variables DIRAC_VOMSES and X509_VOMSES in your bashrc. " - "DIRAC_VOMSES will be dropped in a future version, please use only X509_VOMSES" - ) - elif "DIRAC_VOMSES" in os.environ and "X509_VOMSES" not in os.environ: - os.environ["X509_VOMSES"] = os.environ["DIRAC_VOMSES"] - gLogger.notice( - "You set the variables DIRAC_VOMSES in your bashrc. " - "DIRAC_VOMSES will be dropped in a future version, please use X509_VOMSES" - ) - # End of transition code - if "X509_VOMSES" not in os.environ: - raise Exception( - "The env variable X509_VOMSES is not set. " - "DIRAC does not know where to look for etc/grid-security/vomses. " - "Please set X509_VOMSES in your bashrc." - ) - vomsesPaths = os.environ["X509_VOMSES"].split(":") - for vomsesPath in vomsesPaths: + for vomsesPath in Locations.getVomsesLocation().split(":"): if not os.path.exists(vomsesPath): continue if os.path.isfile(vomsesPath): @@ -251,49 +259,29 @@ def setVOMSAttributes(self, proxy, attribute=None, vo=None): chain = proxyDict["chain"] proxyLocation = proxyDict["file"] - secs = chain.getRemainingSecs()["Value"] - 300 - if secs < 0: - return S_ERROR(DErrno.EVOMS, "Proxy length is less that 300 secs") - hours = int(secs / 3600) - mins = int((secs - hours * 3600) / 60) - - # Ask VOMS a proxy the same strength as the one we already have - bitStrength = chain.getStrength()["Value"] - retVal = self._generateTemporalFile() if not retVal["OK"]: deleteMultiProxy(proxyDict) return retVal newProxyLocation = retVal["Value"] - cmd = ["voms-proxy-init"] - if chain.isLimitedProxy()["Value"]: - cmd.append("-limited") - cmd += ["-cert", proxyLocation] - cmd += ["-key", proxyLocation] - cmd += ["-out", newProxyLocation] - cmd += ["-voms"] - cmd += [f"{vo}:{attribute}" if attribute and attribute != "NoRole" else vo] - cmd += ["-valid", f"{hours}:{mins}"] - cmd += ["-bits", str(bitStrength)] - tmpDir = False - vomsesPath = self.getVOMSESLocation() - if vomsesPath: - cmd += ["-vomses", vomsesPath] - - if chain.isRFC().get("Value"): - cmd += ["-r"] - cmd += ["-timeout", str(self._servTimeout)] - - result = shellCall(self._secCmdTimeout, shlex.join(cmd)) - if tmpDir: - shutil.rmtree(tmpDir) + cmd = voms_init_cmd(vo, attribute, chain, proxyLocation, newProxyLocation, self.getVOMSESLocation()) + + result = shellCall( + self._secCmdTimeout, + shlex.join(cmd), + env=os.environ + | { + "X509_CERT_DIR": Locations.getCAsLocation(), + "X509_VOMS_DIR": Locations.getVomsdirLocation(), + }, + ) deleteMultiProxy(proxyDict) if not result["OK"]: self._unlinkFiles(newProxyLocation) - return S_ERROR(DErrno.EVOMS, "Failed to call voms-proxy-init: %s" % result["Message"]) + return S_ERROR(DErrno.EVOMS, f"Failed to call voms-proxy-init: {result['Message']}") status, output, error = result["Value"] @@ -308,7 +296,7 @@ def setVOMSAttributes(self, proxy, attribute=None, vo=None): retVal = newChain.loadProxyFromFile(newProxyLocation) self._unlinkFiles(newProxyLocation) if not retVal["OK"]: - return S_ERROR(DErrno.EVOMS, "Can't load new proxy: %s" % retVal["Message"]) + return S_ERROR(DErrno.EVOMS, f"Can't load new proxy: {retVal['Message']}") return S_OK(newChain) @@ -324,7 +312,7 @@ def vomsInfoAvailable(self): if not vpInfoCmd: return S_ERROR(DErrno.EVOMS, "Missing voms-proxy-info") - cmd = "%s -h" % vpInfoCmd + cmd = f"{vpInfoCmd} -h" result = shellCall(self._secCmdTimeout, cmd) if not result["OK"]: return False @@ -332,3 +320,21 @@ def vomsInfoAvailable(self): if status: return False return True + + def _unlinkFiles(self, files): + if isinstance(files, (list, tuple)): + for fileName in files: + self._unlinkFiles(fileName) + else: + try: + os.unlink(files) + except Exception: + pass + + def _generateTemporalFile(self): + try: + fd, filename = tempfile.mkstemp() + os.close(fd) + except OSError: + return S_ERROR(DErrno.ECTMPF) + return S_OK(filename) diff --git a/src/DIRAC/Core/Security/VOMSService.py b/src/DIRAC/Core/Security/VOMSService.py index 23f611c32de..642a9983fae 100644 --- a/src/DIRAC/Core/Security/VOMSService.py +++ b/src/DIRAC/Core/Security/VOMSService.py @@ -1,19 +1,23 @@ """ VOMSService class encapsulates connection to the VOMS service for a given VO """ + import requests from DIRAC import gConfig, gLogger, S_OK, S_ERROR from DIRAC.Core.Utilities import DErrno from DIRAC.Core.Security.Locations import getProxyLocation, getCAsLocation +from DIRAC.Core.Utilities.Decorators import deprecated from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOOption from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import getVO class VOMSService: - def __init__(self, vo=None): + def __init__(self, vo=None, compareWithIAM=False, useIAM=False): """c'tor :param str vo: name of the virtual organization (community) + :param compareWithIAM: if true, also dump the list of users from IAM and compare + :param useIAM: if True, use Iam instead of VOMS """ if vo is None: @@ -22,53 +26,44 @@ def __init__(self, vo=None): raise Exception("No VO name given") self.vo = vo + self.vomsVO = getVOOption(vo, "VOMSName") if not self.vomsVO: - raise Exception("Can not get VOMS name for VO %s" % vo) + raise Exception(f"Can not get VOMS name for VO {vo}") self.urls = [] - result = gConfig.getSections("/Registry/VO/%s/VOMSServers" % self.vo) + result = gConfig.getSections(f"/Registry/VO/{self.vo}/VOMSServers") if result["OK"]: for server in result["Value"]: gLogger.verbose(f"Adding 'https://{server}:8443/voms/{self.vomsVO}/apiv2/users'") self.urls.append(f"https://{server}:8443/voms/{self.vomsVO}/apiv2/users") else: - gLogger.error("Section '/Registry/VO/%s/VOMSServers' not found" % self.vo) + gLogger.error(f"Section '/Registry/VO/{self.vo}/VOMSServers' not found") + + self.iam_url = None + self.compareWithIAM = compareWithIAM + self.useIAM = useIAM + if compareWithIAM or useIAM: + id_provider = gConfig.getValue(f"/Registry/VO/{self.vo}/IdProvider") + if not id_provider: + raise ValueError(f"/Registry/VO/{self.vo}/IdProvider not found") + result = gConfig.getOptionsDict(f"/Resources/IdProviders/{id_provider}") + if result["OK"]: + self.iam_url = result["Value"]["issuer"] + gLogger.verbose("Using IAM server", self.iam_url) + else: + raise ValueError(f"/Resources/IdProviders/{id_provider}") self.userDict = None - def attGetUserNickname(self, dn, _ca=None): - """Get user nickname for a given DN if any - - :param str dn: user DN - :param str _ca: CA, kept for backward compatibility - :return: S_OK with Value: nickname - """ - - if self.userDict is None: - result = self.getUsers() - if not result["OK"]: - return result - - uDict = self.userDict.get(dn) - if not uDict: - return S_ERROR(DErrno.EVOMS, "No nickname defined") - nickname = uDict.get("nickname") - if not nickname: - return S_ERROR(DErrno.EVOMS, "No nickname defined") - return S_OK(nickname) - def getUsers(self): """Get all the users of the VOMS VO with their detailed information - :return: user dictionary keyed by the user DN + :return: dictionary of: "Users": user dictionary keyed by the user DN, "Errors": empty list """ - if not self.urls: return S_ERROR(DErrno.ENOAUTH, "No VOMS server defined") - userProxy = getProxyLocation() - caPath = getCAsLocation() rawUserList = [] result = None for url in self.urls: @@ -82,8 +77,8 @@ def getUsers(self): result = requests.get( url, headers={"X-VOMS-CSRF-GUARD": "y"}, - cert=userProxy, - verify=caPath, + cert=getProxyLocation(), + verify=getCAsLocation(), params={"startIndex": str(startIndex), "pageSize": "100"}, ) except requests.ConnectionError as exc: @@ -92,7 +87,7 @@ def getUsers(self): continue if result.status_code != 200: - error = "Failed to contact the VOMS server: %s" % result.text + error = f"Failed to contact the VOMS server: {result.text}" urlDone = True continue @@ -109,7 +104,7 @@ def getUsers(self): break if error: - return S_ERROR(DErrno.ENOAUTH, "Failed to contact the VOMS server: %s" % error) + return S_ERROR(DErrno.ENOAUTH, f"Failed to contact the VOMS server: {error}") # We have got the user info, reformat it resultDict = {} @@ -129,4 +124,5 @@ def getUsers(self): resultDict[dn]["nickname"] = attribute.get("value") self.userDict = dict(resultDict) - return S_OK(resultDict) + # for consistency with IAM interface, we add Errors + return S_OK({"Users": resultDict, "Errors": []}) diff --git a/src/DIRAC/Core/Security/m2crypto/X509CRL.py b/src/DIRAC/Core/Security/m2crypto/X509CRL.py index 3c73e9fd85a..f942c2c41a6 100644 --- a/src/DIRAC/Core/Security/m2crypto/X509CRL.py +++ b/src/DIRAC/Core/Security/m2crypto/X509CRL.py @@ -1,15 +1,13 @@ """ X509CRL is a class for managing X509CRL This class is used to manage the revoked certificates.... """ -import stat -import os -import tempfile import re import datetime -import M2Crypto +import M2Crypto.X509 from DIRAC import S_OK, S_ERROR from DIRAC.Core.Utilities import DErrno +from DIRAC.Core.Utilities.File import secureOpenForWrite # pylint: disable=broad-except @@ -42,7 +40,7 @@ def loadCRLFromFile(self, crlLocation): try: self.__revokedCert = M2Crypto.X509.load_crl(crlLocation) except Exception as e: - return S_ERROR(DErrno.ECERTREAD, "%s" % repr(e).replace(",)", ")")) + return S_ERROR(DErrno.ECERTREAD, f"{repr(e).replace(',)', ')')}") self.__loadedCert = True with open(crlLocation) as crlFile: pemData = crlFile.read() @@ -72,17 +70,10 @@ def dumpAllToFile(self, filename=False): if not self.__loadedCert: return S_ERROR("No certificate loaded") try: - if not filename: - fd, filename = tempfile.mkstemp() - os.close(fd) - with open(filename, "w", encoding="ascii") as fd: + with secureOpenForWrite(filename) as (fd, filename): fd.write(self.__pemData) except Exception as e: - return S_ERROR(DErrno.EWF, "{}: {}".format(filename, repr(e).replace(",)", ")"))) - try: - os.chmod(filename, stat.S_IRUSR | stat.S_IWUSR) - except Exception as e: - return S_ERROR(DErrno.ESPF, "{}: {}".format(filename, repr(e).replace(",)", ")"))) + return S_ERROR(DErrno.EWF, f"{filename}: {repr(e).replace(',)', ')')}") return S_OK(filename) def hasExpired(self): diff --git a/src/DIRAC/Core/Security/m2crypto/X509Certificate.py b/src/DIRAC/Core/Security/m2crypto/X509Certificate.py index 87c628acd4d..0e47bdb8d8e 100644 --- a/src/DIRAC/Core/Security/m2crypto/X509Certificate.py +++ b/src/DIRAC/Core/Security/m2crypto/X509Certificate.py @@ -10,7 +10,9 @@ import random import time -import M2Crypto +import M2Crypto.m2 +import M2Crypto.ASN1 +import M2Crypto.X509 from DIRAC import S_OK, S_ERROR @@ -97,11 +99,11 @@ def generateProxyCertFromIssuer(cls, x509Issuer, x509ExtensionStack, proxyKey, l proxySubject = M2Crypto.X509.X509_Name() issuerSubjectObj = x509Issuer.__certObj.get_subject() - issuerSubjectParts = issuerSubjectObj.as_text().split(", ") - for isPart in issuerSubjectParts: - nid, val = isPart.split("=", 1) - proxySubject.add_entry_by_txt(field=nid, type=M2Crypto.ASN1.MBSTRING_ASC, entry=val, len=-1, loc=-1, set=0) + # Copy the X509 entry components into the new name + for entry in issuerSubjectObj: + # pylint: disable=no-member + M2Crypto.m2.x509_name_add_entry(proxySubject.x509_name, entry.x509_name_entry, -1, 0) # Finally we add a random Common Name component. And we might as well use the serial.. :) proxySubject.add_entry_by_txt( @@ -165,7 +167,7 @@ def loadFromFile(self, certLocation): pemData = fd.read() return self.loadFromString(pemData) except OSError: - return S_ERROR(DErrno.EOF, "Can't open %s file" % certLocation) + return S_ERROR(DErrno.EOF, f"Can't open {certLocation} file") def loadFromString(self, pemData): """ @@ -180,7 +182,7 @@ def loadFromString(self, pemData): try: self.__certObj = M2Crypto.X509.load_cert_string(pemData, M2Crypto.X509.FORMAT_PEM) except Exception as e: - return S_ERROR(DErrno.ECERTREAD, "Can't load pem data: %s" % e) + return S_ERROR(DErrno.ECERTREAD, f"Can't load pem data: {e}") self._certLoaded = True return S_OK() @@ -209,8 +211,10 @@ def getNotAfterDate(self): :returns: S_OK( datetime )/S_ERROR """ - - notAfter = self.__certObj.get_not_after().get_datetime() + # Here we use the M2Crypto low level API, as the high level API is notably + # slower due to the conversion to a string and then back to an ASN1_TIME. + rawNotAfter = M2Crypto.m2.x509_get_not_after(self.__certObj.x509) # pylint: disable=no-member + notAfter = M2Crypto.ASN1.ASN1_TIME(rawNotAfter).get_datetime() # M2Crypto does things correctly by setting a timezone info in the datetime # However, we do not in DIRAC, and so we can't compare the dates. @@ -230,7 +234,7 @@ def getStrength(self): try: return S_OK(self.__certObj.get_pubkey().size() * 8) except Exception as e: - return S_ERROR("Cannot get certificate strength: %s" % e) + return S_ERROR(f"Cannot get certificate strength: {e}") @executeOnlyIfCertLoaded def getNotBeforeDate(self): @@ -240,7 +244,10 @@ def getNotBeforeDate(self): :returns: S_OK( datetime )/S_ERROR """ - return S_OK(self.__certObj.get_not_before().get_datetime()) + # Here we use the M2Crypto low level API, as the high level API is notably + # slower due to the conversion to a string and then back to an ASN1_TIME. + rawNotBefore = M2Crypto.m2.x509_get_not_before(self.__certObj.x509) # pylint: disable=no-member + return S_OK(M2Crypto.ASN1.ASN1_TIME(rawNotBefore).get_datetime()) # @executeOnlyIfCertLoaded # def setNotBefore(self, notbefore): diff --git a/src/DIRAC/Core/Security/m2crypto/X509Chain.py b/src/DIRAC/Core/Security/m2crypto/X509Chain.py index 4df854bb620..5aa2659071b 100644 --- a/src/DIRAC/Core/Security/m2crypto/X509Chain.py +++ b/src/DIRAC/Core/Security/m2crypto/X509Chain.py @@ -3,28 +3,20 @@ Link to the RFC 3820: https://tools.ietf.org/html/rfc3820 In particular, limited proxy: https://tools.ietf.org/html/rfc3820#section-3.8 -There are also details available about Per-User Sub-Proxies (PUSP) -here: https://wiki.egi.eu/wiki/Usage_of_the_per_user_sub_proxy_in_EGI - """ import copy -import os -import stat -import tempfile import hashlib - import re -import M2Crypto - +import M2Crypto.X509 -from DIRAC import S_OK, S_ERROR -from DIRAC.Core.Utilities import DErrno -from DIRAC.Core.Utilities.Decorators import executeOnlyIf, deprecated +from DIRAC import S_ERROR, S_OK from DIRAC.ConfigurationSystem.Client.Helpers import Registry -from DIRAC.Core.Security.m2crypto import PROXY_OID, LIMITED_PROXY_OID, DIRAC_GROUP_OID, DEFAULT_PROXY_STRENGTH +from DIRAC.Core.Security.m2crypto import DEFAULT_PROXY_STRENGTH, DIRAC_GROUP_OID, LIMITED_PROXY_OID, PROXY_OID from DIRAC.Core.Security.m2crypto.X509Certificate import X509Certificate - +from DIRAC.Core.Utilities import DErrno +from DIRAC.Core.Utilities.Decorators import executeOnlyIf +from DIRAC.Core.Utilities.File import secureOpenForWrite # Decorator to check that _certList is not empty needCertList = executeOnlyIf("_certList", S_ERROR(DErrno.ENOCHAIN)) @@ -106,7 +98,7 @@ class X509Chain: # The client side signs the request, with its proxy # Assume the proxy chain was already loaded one way or the otjer - # The proxy will contain a "bullshit private key" + # The proxy will not contain a private key res = proxyChain.generateChainFromRequestString(reqStr, lifetime=lifetime) # This is sent back to the server @@ -165,22 +157,6 @@ def __init__(self, certList=False, keyObj=False): if keyObj: self._keyObj = keyObj - @classmethod - @deprecated("Use loadChainFromFile instead", onlyOnce=True) - def instanceFromFile(cls, chainLocation): - """Class method to generate a X509Chain from a file - - :param chainLocation: path to the file - - :returns: S_OK(X509Chain) - """ - chain = cls() - result = chain.loadChainFromFile(chainLocation) - if not result["OK"]: - return result - - return S_OK(chain) - @staticmethod def generateX509ChainFromSSLConnection(sslConnection): """Returns an instance of X509Chain from the SSL connection @@ -214,7 +190,7 @@ def loadChainFromFile(self, chainLocation): with open(chainLocation) as fd: pemData = fd.read() except OSError as e: - return S_ERROR(DErrno.EOF, "{}: {}".format(chainLocation, repr(e).replace(",)", ")"))) + return S_ERROR(DErrno.EOF, f"{chainLocation}: {repr(e).replace(',)', ')')}") return self.loadChainFromString(pemData) def loadChainFromString(self, data): @@ -228,7 +204,7 @@ def loadChainFromString(self, data): try: self._certList = self.__certListFromPemString(data) except Exception as e: - return S_ERROR(DErrno.ECERTREAD, "%s" % repr(e).replace(",)", ")")) + return S_ERROR(DErrno.ECERTREAD, f"{repr(e).replace(',)', ')')}") if not self._certList: return S_ERROR(DErrno.EX509) @@ -273,7 +249,7 @@ def loadKeyFromFile(self, chainLocation, password=False): with open(chainLocation) as fd: pemData = fd.read() except Exception as e: - return S_ERROR(DErrno.EOF, "{}: {}".format(chainLocation, repr(e).replace(",)", ")"))) + return S_ERROR(DErrno.EOF, f"{chainLocation}: {repr(e).replace(',)', ')')}") return self.loadKeyFromString(pemData, password) def loadKeyFromString(self, pemData, password=False): @@ -293,7 +269,7 @@ def loadKeyFromString(self, pemData, password=False): try: self._keyObj = M2Crypto.EVP.load_key_string(pemData, lambda x: password) except Exception as e: - return S_ERROR(DErrno.ECERTREAD, "%s (Probably bad pass phrase?)" % repr(e).replace(",)", ")")) + return S_ERROR(DErrno.ECERTREAD, f"{repr(e).replace(',)', ')')} (Probably bad pass phrase?)") return S_OK() @@ -317,7 +293,7 @@ def loadProxyFromFile(self, chainLocation): with open(chainLocation) as fd: pemData = fd.read() except Exception as e: - return S_ERROR(DErrno.EOF, "{}: {}".format(chainLocation, repr(e).replace(",)", ")"))) + return S_ERROR(DErrno.EOF, f"{chainLocation}: {repr(e).replace(',)', ')')}") return self.loadProxyFromString(pemData) def loadProxyFromString(self, pemData): @@ -356,14 +332,14 @@ def __getProxyExtensionList(diracGroup=False, rfcLimited=False): # Mandatory extension to be a proxy policyOID = LIMITED_PROXY_OID if rfcLimited else PROXY_OID - ext = M2Crypto.X509.new_extension("proxyCertInfo", "critical, language:%s" % (policyOID), critical=1) + ext = M2Crypto.X509.new_extension("proxyCertInfo", f"critical, language:{policyOID}", critical=1) extStack.push(ext) # Add a dirac group if diracGroup and isinstance(diracGroup, str): # the str cast is needed because M2Crypto does not play it cool with unicode here it seems # Also one needs to specify the ASN1 type. That's what it is... - dGext = M2Crypto.X509.new_extension(DIRAC_GROUP_OID, str("ASN1:IA5:%s" % diracGroup)) + dGext = M2Crypto.X509.new_extension(DIRAC_GROUP_OID, str(f"ASN1:IA5:{diracGroup}")) extStack.push(dGext) return extStack @@ -393,24 +369,6 @@ def getIssuerCert(self): return S_OK(self._certList[self.__firstProxyStep + 1]) return S_OK(self._certList[-1]) - @deprecated("Only here for compatibility reason", onlyOnce=True) - @needPKey - def getPKeyObj(self): - """ - Get the pkey obj - - :returns: ~M2Crypto.EVP.PKey object - """ - return S_OK(self._keyObj) - - @deprecated("Only here for compatibility reason") - @needCertList - def getCertList(self): - """ - Get the cert list - """ - return S_OK(self._certList) - @needCertList def getNumCertsInChain(self): """ @@ -447,6 +405,9 @@ def generateProxyToString( issuerCert = self._certList[0] + # If this is a certificate signing request then the private key will be + # appended by the server and we don't need to include it in the proxy + include_private_key = not proxyKey if not proxyKey: # Generating key is a two step process: create key object and then assign RSA key. # This contains both the private and public key @@ -464,10 +425,9 @@ def generateProxyToString( proxyCert.sign(self._keyObj, "sha256") # Generate the proxy string - proxyString = "{}{}".format( - proxyCert.asPem(), - proxyKey.as_pem(cipher=None, callback=M2Crypto.util.no_passphrase_callback).decode("ascii"), - ) + proxyString = proxyCert.asPem() + if include_private_key: + proxyString += proxyKey.as_pem(cipher=None, callback=M2Crypto.util.no_passphrase_callback).decode("ascii") for i in range(len(self._certList)): crt = self._certList[i] proxyString += crt.asPem() @@ -490,14 +450,10 @@ def generateProxyToFile(self, filePath, lifetime, diracGroup=False, strength=DEF if not retVal["OK"]: return retVal try: - with open(filePath, "w") as fd: + with secureOpenForWrite(filePath) as (fd, _filename): fd.write(retVal["Value"]) except Exception as e: - return S_ERROR(DErrno.EWF, "{} :{}".format(filePath, repr(e).replace(",)", ")"))) - try: - os.chmod(filePath, stat.S_IRUSR | stat.S_IWUSR) - except Exception as e: - return S_ERROR(DErrno.ESPF, "{} :{}".format(filePath, repr(e).replace(",)", ")"))) + return S_ERROR(DErrno.EWF, f"{filePath} :{repr(e).replace(',)', ')')}") return S_OK() @needCertList @@ -610,7 +566,6 @@ def __checkProxyness(self): # Here we make sure that each certificate in the chain was # signed by the previous one for step in range(len(self._certList) - 1): - # this is a cryptographic check with the keys issuerMatch = self.__checkIssuer(step, step + 1) if not issuerMatch: @@ -720,11 +675,6 @@ def getDIRACGroup(self, ignoreDefault=False): if not self.__isProxy: return S_ERROR(DErrno.EX509, "Chain does not contain a valid proxy") - # If it is a PUSP, we do a lookup based on the certificate - # (you can't do a PUSP out of a proxy) - if self.isPUSP()["Value"]: - return self._certList[self.__firstProxyStep - 2].getDIRACGroup(ignoreDefault=ignoreDefault) - # The code below will find the first match of the DIRAC group for cert in reversed(self._certList): # We specifically say we do not want the default to first check inside the proxy @@ -825,13 +775,13 @@ def generateChainFromRequestString(self, pemData, lifetime=86400, requireLimited req = M2Crypto.X509.load_request_string(pemData, format=M2Crypto.X509.FORMAT_PEM) except Exception as e: - return S_ERROR(DErrno.ECERTREAD, "Can't load request data: %s" % repr(e).replace(",)", ")")) + return S_ERROR(DErrno.ECERTREAD, f"Can't load request data: {repr(e).replace(',)', ')')}") # I am not sure this test makes sense. # You can't request a limit proxy if you are yourself not limited ?! # I think it should be a "or" instead of "and" limited = requireLimited and self.isLimitedProxy().get("Value", False) - return self.generateProxyToString(lifetime, diracGroup, None, limited, req.get_pubkey()) + return self.generateProxyToString(lifetime, diracGroup, DEFAULT_PROXY_STRENGTH, limited, req.get_pubkey()) @needCertList def getRemainingSecs(self): @@ -879,17 +829,10 @@ def dumpAllToFile(self, filename=False): return retVal pemData = retVal["Value"] try: - if not filename: - fd, filename = tempfile.mkstemp() - os.close(fd) - with open(filename, "w") as fp: - fp.write(pemData) - except Exception as e: - return S_ERROR(DErrno.EWF, "{} :{}".format(filename, repr(e).replace(",)", ")"))) - try: - os.chmod(filename, stat.S_IRUSR | stat.S_IWUSR) + with secureOpenForWrite(filename) as (fh, filename): + fh.write(pemData) except Exception as e: - return S_ERROR(DErrno.ESPF, "{} :{}".format(filename, repr(e).replace(",)", ")"))) + return S_ERROR(DErrno.EWF, f"{filename} :{repr(e).replace(',)', ')')}") return S_OK(filename) @needCertList @@ -915,9 +858,9 @@ def __str__(self): """String representation""" repStr = ") result = loader.loadModules(instances) if result["OK"]: for module in loader.getModules().values(): - handler = module["classObj"] + # Make a separate class per component that can be individually configured + tornadoHandlerClassName = f"{module['modName'].replace('/', '')}Handler" fullComponentName = module["modName"] + handler = type( + tornadoHandlerClassName, (module["classObj"],), {"_fullComponentName": fullComponentName} + ) - # Define the system and component name as the attributes of the handler that belongs to them - handler.SYSTEM_NAME, handler.COMPONENT_NAME = fullComponentName.split("/") - - gLogger.info("Found new handler", f"{fullComponentName}: {handler}") + gLogger.info("Found new handler", f"{fullComponentName}: {module['classObj']}") # at this stage we run the basic handler initialization # see DIRAC.Core.Tornado.Server.private.BaseRequestHandler for more details # this method should return a list of routes associated with the handler, it is a regular expressions # see https://www.tornadoweb.org/en/stable/routing.html#tornado.routing.URLSpec, ``pattern`` argument. - urls = handler._BaseRequestHandler__pre_initialize() + urls = handler._BaseRequestHandler__pre_initialize() # pylint: disable=no-member # First of all check if we can find route if not urls: diff --git a/src/DIRAC/Core/Tornado/Server/TornadoREST.py b/src/DIRAC/Core/Tornado/Server/TornadoREST.py index 9ca72705caa..c9bacff2118 100644 --- a/src/DIRAC/Core/Tornado/Server/TornadoREST.py +++ b/src/DIRAC/Core/Tornado/Server/TornadoREST.py @@ -5,14 +5,14 @@ import os import inspect +from functools import partial +from urllib.parse import unquote + from tornado.escape import json_decode from tornado.web import url as TornadoURL -from urllib.parse import unquote -from functools import partial -from DIRAC import gLogger from DIRAC.ConfigurationSystem.Client import PathFinder -from DIRAC.Core.Tornado.Server.private.BaseRequestHandler import * +from DIRAC.Core.Tornado.Server.private.BaseRequestHandler import BaseRequestHandler, set_attribute # decorator to determine the path to access the target method location = partial(set_attribute, "location") @@ -152,6 +152,9 @@ def options_job(self): METHOD_PREFIX = None DEFAULT_LOCATION = "/" + # Never use the activity monitoring here + activityMonitoringReporter = False + @classmethod def _pre_initialize(cls) -> list: """This method is run by the Tornado server to prepare the handler for launch @@ -171,7 +174,7 @@ def _pre_initialize(cls) -> list: """ urls = [] # Look for methods that are exported - for prefix in [cls.METHOD_PREFIX] if cls.METHOD_PREFIX else cls.SUPPORTED_METHODS: + for prefix in [cls.METHOD_PREFIX] if cls.METHOD_PREFIX else [f"{pref}_" for pref in cls.SUPPORTED_METHODS]: prefix = prefix.lower() for mName, mObj in inspect.getmembers(cls, lambda x: callable(x) and x.__name__.startswith(prefix)): methodName = mName[len(prefix) :] @@ -264,14 +267,14 @@ def _getComponentInfoDict(cls, fullComponentName: str, fullURL: str) -> dict: return {} @classmethod - def _getCSAuthorizarionSection(cls, apiName): + def _getCSAuthorizationSection(cls, apiName): """Search endpoint auth section. :param str apiName: API name, see :py:meth:`_getFullComponentName` :return: str """ - return "%s/Authorization" % PathFinder.getAPISection(apiName) + return f"{PathFinder.getAPISection(apiName)}/Authorization" def _getMethod(self): """Get target method function to call. By default we read the first section in the path @@ -341,4 +344,4 @@ def post_note(self, pos_only, /, standard, *, kwd_only): # Wrap argument with annotated type keywordArguments[name] = _type(value) if _type else value - return (positionalArguments, keywordArguments) + return positionalArguments, keywordArguments diff --git a/src/DIRAC/Core/Tornado/Server/TornadoServer.py b/src/DIRAC/Core/Tornado/Server/TornadoServer.py index 11ccc0617ed..f252855c986 100644 --- a/src/DIRAC/Core/Tornado/Server/TornadoServer.py +++ b/src/DIRAC/Core/Tornado/Server/TornadoServer.py @@ -8,11 +8,11 @@ import asyncio import psutil -import M2Crypto +import M2Crypto.SSL import tornado.iostream -tornado.iostream.SSLIOStream.configure( +tornado.iostream.SSLIOStream.configure( # pylint: disable=no-member "tornado_m2crypto.m2iostream.M2IOStream" ) # pylint: disable=wrong-import-position @@ -25,7 +25,6 @@ from DIRAC.Core.Security import Locations from DIRAC.Core.Utilities import Network, TimeUtilities from DIRAC.Core.Tornado.Server.HandlerManager import HandlerManager -from DIRAC.ConfigurationSystem.Client import PathFinder from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations sLog = gLogger.getSubLogger(__name__) @@ -91,7 +90,7 @@ def __init__(self, services=True, endpoints=False, port=None): self.__appsSettings = {} # Default port, if enother is not discover if port is None: - port = gConfig.getValue("/Systems/Tornado/%s/Port" % PathFinder.getSystemInstance("Tornado"), 8443) + port = gConfig.getValue(f"/Systems/Tornado/Port", 8443) self.port = port # Handler manager initialization with default settings @@ -176,6 +175,12 @@ def startTornado(self): Starts the tornado server when ready. This method never returns. """ + + # If we are running with python3, Tornado will use asyncio, + # and we have to convince it to let us run in a different thread + # This statement must be placed before setting PeriodicCallback + asyncio.set_event_loop_policy(tornado.platform.asyncio.AnyThreadEventLoopPolicy()) + # If there is no services loaded: if not self.__calculateAppSettings(): raise Exception("There is no services loaded, please check your configuration") @@ -205,13 +210,7 @@ def startTornado(self): self.__report = self.__startReportToMonitoringLoop() # Response time # Starting monitoring, IOLoop waiting time in ms, __monitoringLoopDelay is defined in seconds - tornado.ioloop.PeriodicCallback( - self.__reportToMonitoring(self.__elapsedTime), self.__monitoringLoopDelay * 1000 - ).start() - - # If we are running with python3, Tornado will use asyncio, - # and we have to convince it to let us run in a different thread - asyncio.set_event_loop_policy(tornado.platform.asyncio.AnyThreadEventLoopPolicy()) + tornado.ioloop.PeriodicCallback(self.__reportToMonitoring, self.__monitoringLoopDelay * 1000).start() for port, app in self.__appsSettings.items(): sLog.debug(" - %s" % "\n - ".join([f"{k} = {ssl_options[k]}" for k in ssl_options])) @@ -229,11 +228,11 @@ def startTornado(self): except Exception as e: # pylint: disable=broad-except sLog.exception("Exception starting HTTPServer", e) raise - sLog.always("Listening on port %s" % port) + sLog.always(f"Listening on port {port}") tornado.ioloop.IOLoop.current().start() - def __reportToMonitoring(self, responseTime): + def __reportToMonitoring(self): """ Periodically reports to Monitoring """ @@ -248,13 +247,24 @@ def __reportToMonitoring(self, responseTime): "ServiceName": "Tornado", "MemoryUsage": self.__report[2], "CpuPercentage": percentage, - "ResponseTime": responseTime, } ) self.activityMonitoringReporter.commit() # Save memory usage and save realtime/CPU time for next call self.__report = self.__startReportToMonitoringLoop() + # For each handler, + for urlSpec in self.handlerManager.getHandlersDict().values(): + # If there is more than one URL, it's + # most likely something that inherit from TornadoREST + # so don't even try to monitor... + if len(urlSpec["URLs"]) > 1: + continue + handler = urlSpec["URLs"][0].handler_class + # If there is a Monitoring reporter, call commit on it + if getattr(handler, "activityMonitoringReporter", None): + handler.activityMonitoringReporter.commit() + def __startReportToMonitoringLoop(self): """ Snapshot of resources to be taken at the beginning diff --git a/src/DIRAC/Core/Tornado/Server/TornadoService.py b/src/DIRAC/Core/Tornado/Server/TornadoService.py index dd84ee99f6c..e9e638c77c1 100644 --- a/src/DIRAC/Core/Tornado/Server/TornadoService.py +++ b/src/DIRAC/Core/Tornado/Server/TornadoService.py @@ -152,14 +152,14 @@ def _getComponentInfoDict(cls, serviceName, fullURL): return cls._serviceInfoDict @classmethod - def _getCSAuthorizarionSection(cls, serviceName): + def _getCSAuthorizationSection(cls, serviceName): """Search service auth section. :param str serviceName: service name, see :py:meth:`_getFullComponentName` :return: str """ - return "%s/Authorization" % PathFinder.getServiceSection(serviceName) + return f"{PathFinder.getServiceSection(serviceName)}/Authorization" def _getMethod(self) -> str: """Get target function name""" @@ -193,7 +193,7 @@ def export_ping(self): startTime = self._startTime dInfo["service start time"] = self._startTime serviceUptime = datetime.utcnow() - startTime - dInfo["service uptime"] = serviceUptime.days * 3600 + serviceUptime.seconds + dInfo["service uptime"] = int(serviceUptime.total_seconds()) # Load average try: with open("/proc/loadavg") as oFD: diff --git a/src/DIRAC/Core/Tornado/Server/private/BaseRequestHandler.py b/src/DIRAC/Core/Tornado/Server/private/BaseRequestHandler.py index 3edc967b412..bb57e93f2df 100644 --- a/src/DIRAC/Core/Tornado/Server/private/BaseRequestHandler.py +++ b/src/DIRAC/Core/Tornado/Server/private/BaseRequestHandler.py @@ -14,19 +14,18 @@ from functools import partial import jwt -import tornado from tornado.web import RequestHandler, HTTPError from tornado.ioloop import IOLoop -import DIRAC - from DIRAC import gConfig, gLogger, S_OK, S_ERROR +from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.Core.Utilities import DErrno from DIRAC.Core.DISET.AuthManager import AuthManager from DIRAC.Core.Utilities.JEncode import decode, encode +from DIRAC.Core.Utilities import Network, TimeUtilities from DIRAC.Core.Utilities.ReturnValues import isReturnStructure from DIRAC.Core.Security.X509Chain import X509Chain # pylint: disable=import-error -from DIRAC.Resources.IdProvider.Utilities import getProvidersForInstance +from DIRAC.Resources.IdProvider.Utilities import getIdProviderIdentifiers from DIRAC.Resources.IdProvider.IdProviderFactory import IdProviderFactory @@ -175,10 +174,6 @@ class BaseRequestHandler(RequestHandler): Override the class variable ``SUPPORTED_METHODS`` by writing down the necessary methods there. Note that by default all HTTP methods are supported. - It is important to understand that the handler belongs to the system. - The class variable ``SYSTEM_NAME`` displays the system name. By default it is taken from the module name. - This value is used to generate the full component name, see :py:meth:`_getFullComponentName` method - This class also defines some variables for writing your handler's methods: - ``DEFAULT_AUTHORIZATION`` describes the general authorization rules for the entire handler @@ -199,7 +194,7 @@ class BaseRequestHandler(RequestHandler): Also, if necessary, you can create a new type of authorization by simply creating the appropriate method:: def _authzMYAUTH(self): - '''Another authorization algoritm.''' + '''Another authorization algorithm.''' # Do somthing return S_OK(credentials) # return user credentials as a dictionary @@ -211,13 +206,12 @@ def _authzMYAUTH(self): The class contains methods that require implementation: - :py:meth:`_pre_initialize` - - :py:meth:`_getCSAuthorizarionSection` + - :py:meth:`_getCSAuthorizationSection` - :py:meth:`_getMethod` - :py:meth:`_getMethodArgs` Some methods have basic behavior, but developers can rewrite them: - - :py:meth:`_getFullComponentName` - :py:meth:`_getComponentInfoDict` - :py:meth:`_monitorRequest` @@ -238,7 +232,7 @@ def _authzMYAUTH(self): At startup, :py:class:`HandlerManager ` call :py:meth:`__pre_initialize` handler method that inspects the handler and its methods to generate tornados URLs of access to it: - - specifies the full name of the component, including the name of the system to which it belongs, see :py:meth:`_getFullComponentName`. + - specifies the full name of the component, including the name of the system to which it belongs as /. - initialization of the main authorization class, see :py:class:`AuthManager ` for more details. - call :py:meth:`__pre_initialize` that should explore the handler, prepare all the necessary attributes and most importantly - return the list of URL tornadoes @@ -246,12 +240,12 @@ def _authzMYAUTH(self): - load all registered identity providers for authentication with access token, see :py:meth:`__loadIdPs`. - create a ``cls.log`` logger that should be used in the children classes instead of directly ``gLogger`` (this allows to carry the ``tornadoComponent`` information, crutial for centralized logging) - - initialization of the monitoring specific to this handler, see :py:meth:`__initMonitoring`. + - initialization of the monitoring specific to this handler, see :py:meth:`_initMonitoring`. - initialization of the target handler that inherit this one, see :py:meth:`initializeHandler`. Next, first of all the tornados prepare method is called which does the following: - - determines determines the name of the target method and checks its presence, see :py:meth:`_getMethod`. + - determines the name of the target method and checks its presence, see :py:meth:`_getMethod`. - request monitoring, see :py:meth:`_monitorRequest`. - authentication request using one of the available algorithms called ``DEFAULT_AUTHENTICATION``, see :py:meth:`_gatherPeerCredentials` for more details. - and finally authorizing the request to access the component, see :py:meth:`authQuery ` for more details. @@ -261,7 +255,7 @@ def _authzMYAUTH(self): - execute the target method in an executor a separate thread. - defines the arguments of the target method, see :py:meth:`_getMethodArgs`. - - initialization of the each request, see :py:meth:`initializeRequest`. + - initialization of each request, see :py:meth:`initializeRequest`. - the result of the target method is processed in the main thread and returned to the client, see :py:meth:`__execute`. """ @@ -278,13 +272,10 @@ def _authzMYAUTH(self): # The variable that will contain the result of the request, see __execute method __result = None - # Below are variables that the developer can OVERWRITE as needed + # Full component name in the form / + _fullComponentName = None - # System name with which this component is associated. - # Developer can overwrite this - # if your handler is outside the DIRAC system package (src/DIRAC/XXXSystem/) - SYSTEM_NAME = None - COMPONENT_NAME = None + # Below are variables that the developer can OVERWRITE as needed # Base system URL. If defined, it is added as a prefix to the handler generated. BASE_URL = None @@ -316,6 +307,11 @@ def _authzMYAUTH(self): encode = staticmethod(encode) decode = staticmethod(decode) + # Class instance of monitoringReporter to use + # It is initialized in __initialize + # If it is set to False, do not instanciate it + activityMonitoringReporter = None + @classmethod def __pre_initialize(cls) -> list: """This method is run by the Tornado server to prepare the handler for launch, @@ -324,20 +320,18 @@ def __pre_initialize(cls) -> list: :returns: a list of URL (not the string with "https://..." but the tornado object) see http://www.tornadoweb.org/en/stable/web.html#tornado.web.URLSpec """ - # Set full component name, e.g.: / - cls._fullComponentName = cls._getFullComponentName() # Define base request path if not cls.DEFAULT_LOCATION: - # By default use full component name as location + # By default, use the full component name as location cls.DEFAULT_LOCATION = cls._fullComponentName - # SUPPORTED_METHODS should be a list type - if isinstance(cls.SUPPORTED_METHODS, str): - cls.SUPPORTED_METHODS = (cls.SUPPORTED_METHODS,) + # SUPPORTED_METHODS should be a tuple + if not isinstance(cls.SUPPORTED_METHODS, tuple): + raise TypeError("SUPPORTED_METHODS should be a tuple") # authorization manager initialization - cls._authManager = AuthManager(cls._getCSAuthorizarionSection(cls._fullComponentName)) + cls._authManager = AuthManager(cls._getCSAuthorizationSection(cls._fullComponentName)) if not (urls := cls._pre_initialize()): cls.log.warn("no target method found", f"{cls.__name__}") @@ -368,54 +362,41 @@ def _pre_initialize(cls) -> list: raise NotImplementedError("Please, create the _pre_initialize class method") @classmethod - def __initMonitoring(cls, fullComponentName: str, fullUrl: str) -> dict: + def _initMonitoring(cls, fullComponentName: str, fullUrl: str) -> dict: """ Initialize the monitoring specific to this handler This has to be called only by :py:meth:`.__initialize` to ensure thread safety and unicity of the call. - :param componentName: relative URL ``//`` + :param fullComponentName: relative URL ``/`` :param fullUrl: full URl like ``https://://`` """ cls._stats = {"requests": 0, "monitorLastStatsUpdate": time.time()} return S_OK() - @classmethod - def _getFullComponentName(cls) -> str: - """Search the full name of the component, including the name of the system to which it belongs. - CAN be implemented by developer. - """ - if cls.SYSTEM_NAME is None: - # If the system name is not specified, it is taken from the module. - cls.SYSTEM_NAME = ([m[:-6] for m in cls.__module__.split(".") if m.endswith("System")] or [None]).pop() - if cls.COMPONENT_NAME is None: - # If the service name is not specified, it is taken from the handler. - cls.COMPONENT_NAME = cls.__name__[: -len("Handler")] - return f"{cls.SYSTEM_NAME}/{cls.COMPONENT_NAME}" if cls.SYSTEM_NAME else cls.COMPONENT_NAME - @classmethod def __loadIdPs(cls) -> None: """Load identity providers that will be used to verify tokens""" cls.log.debug("Load identity providers..") # Research Identity Providers - result = getProvidersForInstance("Id") + result = getIdProviderIdentifiers() if result["OK"]: for providerName in result["Value"]: result = cls._idps.getIdProvider(providerName) if result["OK"]: cls._idp[result["Value"].issuer.strip("/")] = result["Value"] else: - cls.log.error("Error getting IDP", "{}: {}".format(providerName, result["Message"])) + cls.log.error("Error getting Identity Provider", f"{providerName}: {result['Message']}") @classmethod - def _getCSAuthorizarionSection(cls, fullComponentName: str) -> str: + def _getCSAuthorizationSection(cls, fullComponentName: str) -> str: """Search component authorization section in CS. SHOULD be implemented by developer. - :param fullComponentName: full component name, see :py:meth:`_getFullComponentName` + :param fullComponentName: full component name / """ - raise NotImplementedError("Please, create the _getCSAuthorizarionSection class method") + raise NotImplementedError("Please, create the _getCSAuthorizationSection class method") @classmethod def _getComponentInfoDict(cls, fullComponentName: str, fullURL: str) -> dict: @@ -423,7 +404,7 @@ def _getComponentInfoDict(cls, fullComponentName: str, fullURL: str) -> dict: e.g.: 'serviceName', 'serviceSectionPath', 'csPaths'. SHOULD be implemented by developer. - :param fullComponentName: full component name, see :py:meth:`_getFullComponentName` + :param fullComponentName: full component name / :param fullURL: incoming request path """ raise NotImplementedError("Please, create the _getComponentInfoDict class method") @@ -445,7 +426,6 @@ def __initialize(cls, request): # Otherwise, do the work but with a lock with cls.__init_lock: - # Check again that the initialization was not done by another thread # while we were waiting for the lock if cls.__init_done: @@ -465,12 +445,25 @@ def __initialize(cls, request): cls.log.info("Initializing method for first use", f"{cls._fullComponentName}, initializing..") # component monitoring initialization - cls.__initMonitoring(cls._fullComponentName, absoluteUrl) + cls._initMonitoring(cls._fullComponentName, absoluteUrl) cls._componentInfoDict = cls._getComponentInfoDict(cls._fullComponentName, absoluteUrl) + # Set the level of the logger + # The level has to be set here because we first need to initialize + # cls._componentInfoDict for src_getCSOption to work + logLevel = cls.srv_getCSOption("LogLevel", "INFO") + cls.log.setLevel(logLevel) + cls.initializeHandler(cls._componentInfoDict) + if cls.activityMonitoringReporter is not False and "Monitoring" in Operations().getMonitoringBackends( + monitoringType="ServiceMonitoring" + ): + from DIRAC.MonitoringSystem.Client.MonitoringReporter import MonitoringReporter + + cls.activityMonitoringReporter = MonitoringReporter(monitoringType="ServiceMonitoring") + cls.__init_done = True return S_OK() @@ -615,9 +608,9 @@ def __prepare(self): if not authorized: extraInfo = "" if self.credDict.get("ID"): - extraInfo += "ID: %s" % self.credDict["ID"] + extraInfo += f"ID: {self.credDict['ID']}" elif self.credDict.get("DN"): - extraInfo += "DN: %s" % self.credDict["DN"] + extraInfo += f"DN: {self.credDict['DN']}" self.log.error( "Unauthorized access", f"Identity {self.srv_getFormattedRemoteCredentials()}; path {self.request.path}; {extraInfo}", @@ -658,16 +651,20 @@ def on_finish(self): Called after the end of HTTP request. Log the request duration """ - elapsedTime = 1000.0 * self.request.request_time() + elapsedTime = self.request.request_time() + credentials = self.srv_getFormattedRemoteCredentials() argsString = f"OK {self._status_code}" + monitoringRetStatus = "Unknown" # Finish with DIRAC result if isReturnStructure(self.__result): if self.__result["OK"]: argsString = "OK" + monitoringRetStatus = "OK" else: argsString = f"ERROR: {self.__result['Message']}" + monitoringRetStatus = "ERROR" if callStack := self.__result.pop("CallStack", None): argsString += "\n" + "".join(callStack) # If bad HTTP status code @@ -675,9 +672,25 @@ def on_finish(self): argsString = f"ERROR {self._status_code}: {self._reason}" self.log.notice( - "Returning response", f"{credentials} {self._fullComponentName} ({elapsedTime:.2f} ms) {argsString}" + "Returning response", + f"{credentials} {self._fullComponentName} ({1000.0 * elapsedTime:.2f} ms) {argsString}", ) + if self.activityMonitoringReporter: + record = { + "timestamp": int(TimeUtilities.toEpochMilliSeconds()), + "Host": Network.getFQDN(), + "ServiceName": "_".join(self._fullComponentName.split("/")), + "Location": self.request.uri, + "ResponseTime": elapsedTime, + # Take the method name from the POST call + "MethodName": self.request.arguments.get("method", [b"Unknown"])[0].decode(), + "Protocol": "https", + "Status": monitoringRetStatus, + } + + self.activityMonitoringReporter.addRecord(record) + def _gatherPeerCredentials(self, grants: list = None) -> dict: """Return a dictionary designed to work with the :py:class:`AuthManager `, already written for DISET and re-used for HTTPS. @@ -704,15 +717,15 @@ def _gatherPeerCredentials(self, grants: list = None) -> dict: # everyone will have access as anonymous@visitor for grant in grants or self.DEFAULT_AUTHENTICATION or "VISITOR": grant = grant.upper() - grantFunc = getattr(self, "_authz%s" % grant, None) + grantFunc = getattr(self, f"_authz{grant}", None) # pylint: disable=not-callable - result = grantFunc() if callable(grantFunc) else S_ERROR("%s authentication type is not supported." % grant) + result = grantFunc() if callable(grantFunc) else S_ERROR(f"{grant} authentication type is not supported.") if result["OK"]: for e in err: self.log.debug(e) - self.log.debug("%s authentication success." % grant) + self.log.debug(f"{grant} authentication success.") return result["Value"] - err.append("{} authentication: {}".format(grant, result["Message"])) + err.append(f"{grant} authentication: {result['Message']}") # Report on failed authentication attempts raise Exception("; ".join(err)) @@ -728,13 +741,19 @@ def _authzSSL(self): # If 'IOStream' object has no attribute 'get_ssl_certificate' derCert = None + # Boolean whether we are behind a balancer and can trust headers + balancer = gConfig.getValue("/WebApp/Balancer", "none") != "none" + # Get client certificate as pem if derCert: chainAsText = derCert.as_pem().decode("ascii") # Read all certificate chain chainAsText += "".join([cert.as_pem().decode("ascii") for cert in self.request.get_ssl_certificate_chain()]) - elif self.request.headers.get("X-Ssl_client_verify") == "SUCCESS" and self.request.headers.get("X-SSL-CERT"): - chainAsText = unquote(self.request.headers.get("X-SSL-CERT")) + elif balancer: + if self.request.headers.get("X-Ssl_client_verify") == "SUCCESS" and self.request.headers.get("X-SSL-CERT"): + chainAsText = unquote(self.request.headers.get("X-SSL-CERT")) + else: + return S_ERROR(DErrno.ECERTFIND, "Valid certificate not found.") else: return S_ERROR(DErrno.ECERTFIND, "Valid certificate not found.") @@ -797,7 +816,7 @@ def _authzJWT(self, accessToken=None): self.log.debug("Verify access token") result = self._idp[issuer].verifyToken(accessToken) self.log.debug("Search user group") - return self._idp[issuer].researchGroup(result["Value"], accessToken) if result["OK"] else result + return self._idp[issuer].getUserGroups(accessToken) if result["OK"] else result def _authzVISITOR(self): """Visitor access @@ -830,13 +849,9 @@ def srv_getCSOption(cls, optionName, defaultValue=False): """ if optionName[0] == "/": return gConfig.getValue(optionName, defaultValue) - for csPath in cls._componentInfoDict["csPaths"]: + for csPath in cls._componentInfoDict.get("csPaths", []): result = gConfig.getOption( - "%s/%s" - % ( - csPath, - optionName, - ), + f"{csPath}/{optionName}", defaultValue, ) if result["OK"]: @@ -902,7 +917,7 @@ def srv_getFormattedRemoteCredentials(self): # Depending on where this is call, it may be that credDict is not yet filled. # (reminder: AuthQuery fills part of it..) try: - peerId = "[{}:{}]".format(self.credDict.get("group", "visitor"), self.credDict.get("username", "anonymous")) + peerId = f"[{self.credDict.get('group', 'visitor')}:{self.credDict.get('username', 'anonymous')}]" except (AttributeError, KeyError): pass @@ -937,7 +952,7 @@ async def __execute(self, *args, **kwargs): # pylint: disable=arguments-differ # you need to define the finish_ method. # This method will be started after _executeMethod is completed. elif callable(finishFunc := getattr(self, f"finish_{self.__methodName}", None)): - finishFunc() + finishFunc() # pylint: disable=not-callable # In case nothing is returned elif self.__result is None: diff --git a/src/DIRAC/Core/Tornado/scripts/tornado_start_CS.py b/src/DIRAC/Core/Tornado/scripts/tornado_start_CS.py index a12e13887e2..e303b719464 100644 --- a/src/DIRAC/Core/Tornado/scripts/tornado_start_CS.py +++ b/src/DIRAC/Core/Tornado/scripts/tornado_start_CS.py @@ -13,7 +13,6 @@ @Script() def main(): - if os.environ.get("DIRAC_USE_TORNADO_IOLOOP", "false").lower() not in ("yes", "true"): raise RuntimeError( "DIRAC_USE_TORNADO_IOLOOP is not defined in the environment." @@ -34,9 +33,7 @@ def main(): gRefresher.disable() localCfg = Script.localCfg - localCfg.addMandatoryEntry("/DIRAC/Setup") localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") - localCfg.addDefaultEntry("LogLevel", "INFO") localCfg.addDefaultEntry("LogColor", True) resultDict = localCfg.loadUserData() if not resultDict["OK"]: @@ -48,9 +45,9 @@ def main(): gLogger.initialize("Tornado-CS", "/") - # get the specific master CS port + # get the specific controller CS port try: - csPort = int(gConfigurationData.extractOptionFromCFG("%s/Port" % getServiceSection("Configuration/Server"))) + csPort = int(gConfigurationData.extractOptionFromCFG(f"{getServiceSection('Configuration/Server')}/Port")) except TypeError: csPort = None diff --git a/src/DIRAC/Core/Tornado/scripts/tornado_start_all.py b/src/DIRAC/Core/Tornado/scripts/tornado_start_all.py index 919011d3a56..9fb91abbe8c 100644 --- a/src/DIRAC/Core/Tornado/scripts/tornado_start_all.py +++ b/src/DIRAC/Core/Tornado/scripts/tornado_start_all.py @@ -13,7 +13,6 @@ @Script() def main(): - if os.environ.get("DIRAC_USE_TORNADO_IOLOOP", "false").lower() not in ("yes", "true"): raise RuntimeError( "DIRAC_USE_TORNADO_IOLOOP is not defined in the environment." @@ -31,10 +30,8 @@ def main(): from DIRAC.FrameworkSystem.Client.Logger import gLogger localCfg = Script.localCfg - localCfg.setConfigurationForServer("Tornado/Tornado") - localCfg.addMandatoryEntry("/DIRAC/Setup") + localCfg.setConfigurationForTornado() localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") - localCfg.addDefaultEntry("LogLevel", "INFO") localCfg.addDefaultEntry("LogColor", True) resultDict = localCfg.loadUserData() if not resultDict["OK"]: @@ -46,10 +43,10 @@ def main(): gLogger.initialize("Tornado", "/") - # We check if there is no configuration server started as master - # If you want to start a master CS you should use Configuration_Server.cfg and + # We check if there is no configuration server started as controller + # If you want to start a controller CS you should use Configuration_Server.cfg and # use tornado-start-CS.py - key = "/Systems/Configuration/%s/Services/Server/Protocol" % PathFinder.getSystemInstance("Configuration") + key = f"/Systems/Configuration/Services/Server/Protocol" if gConfigurationData.isMaster() and gConfig.getValue(key, "dips").lower() == "https": gLogger.fatal("You can't run the CS and services in the same server!") sys.exit(0) diff --git a/src/DIRAC/Core/Utilities/Adler.py b/src/DIRAC/Core/Utilities/Adler.py index 6fc17165ba8..011c30ee6bd 100755 --- a/src/DIRAC/Core/Utilities/Adler.py +++ b/src/DIRAC/Core/Utilities/Adler.py @@ -41,7 +41,7 @@ def hexAdlerToInt(hexAdler, pos=True): hexAdler = hexAdler[-8:] hexAdler = hexAdler.replace("x", "0") if not pos: - hexAdler = "-%s" % hexAdler + hexAdler = f"-{hexAdler}" try: # Will always try to return the positive integer value of the provided hex return int(hexAdler, 16) & 0xFFFFFFFF diff --git a/src/DIRAC/Core/Utilities/CGroups2.py b/src/DIRAC/Core/Utilities/CGroups2.py new file mode 100644 index 00000000000..f874f85483f --- /dev/null +++ b/src/DIRAC/Core/Utilities/CGroups2.py @@ -0,0 +1,328 @@ +#!/usr/bin/env python3 +"""cgroup2 support for DIRAC pilot.""" + +import os +import functools +import subprocess +from DIRAC import S_OK, S_ERROR, gLogger +from DIRAC.Core.Utilities.DIRACSingleton import DIRACSingleton +from DIRAC.Core.Utilities import Subprocess + + +class CG2Manager(metaclass=DIRACSingleton): + """A class to manage cgroup2 hierachy for a typical pilot job use-case. + + This creates a group for all of the pilot processes (anything in the + group at the start. This is a requirement for controlling the + sub-groups (no processes in non-leaf groups). + + A group is then created on request for each "slot" under the pilot, + with the requested limits. + """ + + # Paths used to lookup cgroup info + FILE_MOUNTS = "/proc/mounts" + FILE_CUR_CGROUP = f"/proc/{os.getpid()}/cgroup" + # Control file names within the cgroup2 hierachy + CTRL_CONTROLLERS = "cgroup.controllers" + CTRL_PROCS = "cgroup.procs" + CTRL_SUBTREE = "cgroup.subtree_control" + CTRL_MEM_OOM_GROUP = "memory.oom.group" + CTRL_MEM_EVENTS = "memory.events" + CTRL_MEM_MAX = "memory.max" + CTRL_MEM_SWAP_MAX = "memory.swap.max" + CTRL_MEM_PEAK = "memory.peak" + CTRL_CPU_MAX = "cpu.max" + # CPU controller constants + # Weight is the max value for 1 CPU core + CPU_WEIGHT = 100000 + # Period is the averaging time in us to apply the limit + # The default is 100k and I see no particularly reason this should change + CPU_PERIOD = 100000 + # Name of the group for the existing pilot processes + PILOT_GROUP = f"dirac_pilot_{os.getpid()}" + + def __init__(self): + """Set-up CGroup2 manager.""" + # This boolean will be set to True if the cgroups are configured + # in the expected way + self._ready = False + # A counter of number of subgroups created + # Used to create unique group names + self._subproc_num = 0 + # Physical path to the starting cgroup for this process + # (i.e. the base of our hierachy) + self._cgroup_path = None + # Logger + self.log = gLogger.getSubLogger("CG2Manager") + + @staticmethod + def _filter_file(path, filterfcn): + """Opens a file and runs filterfcn for each line. + If filterfcn returns any value, that value will be returned + by this function. + Returns None if no line matches. + """ + with open(path, encoding="ascii") as file_in: + for line in file_in.readlines(): + line = line.strip() + if res := filterfcn(line): + return res + return None + + def _detect_root(self): + """Find the cgroup2 filesystem mountpoint on this system. + Returns the mountpoint path or None if it isn't found. + """ + + def filt(line): + """Filter function to find the first cgroup2 mount point + from a standard /proc/mounts layout file. + """ + parts = line.split(" ") + if len(parts) < 3: + return None + if parts[2] == "cgroup2": + return parts[1] + return None + + return self._filter_file(self.FILE_MOUNTS, filt) + + def _detect_path(self): + """Finds the full physical path to the current cgroup control dir. + Sets self._cgroup_path on success. + Raises a RuntimeError if the path cannot be determined. + """ + + def filt(line): + """Filter to find the current cgroup2 name for the current + process, without the leading /. + """ + if line.startswith("0::/"): + return line[4:] + return False + + if not (root_path := self._detect_root()): + raise RuntimeError("Failed to find cgroup mount point") + if not (cur_group := self._filter_file(self.FILE_CUR_CGROUP, filt)): + raise RuntimeError("Failed to find current cgroup") + self._cgroup_path = os.path.join(root_path, cur_group) + + def _create_group(self, group_name, isolate_oom=True): + """Creates a new group. + If "isolate_oom" is True, the new group will be decoupled + from the parent's OOM group. + Raises a RuntimeError if the group cannot be created. + """ + try: + os.mkdir(os.path.join(self._cgroup_path, group_name)) + except PermissionError as err: + raise RuntimeError(f"Permission denied creating sub-cgroup '{group_name}'") from err + if isolate_oom: + self._write_control(group_name, self.CTRL_MEM_OOM_GROUP, "0") + + def _remove_group(self, group_name): + """Removes a group.""" + os.rmdir(os.path.join(self._cgroup_path, group_name)) + + def _move_init_procs(self): + """Creates the pilot sub-group and moves all of the initial processes + from the top group into the new sub-group. + Will raise a RuntimeError if any cgroup configuration problem + prevents this from completing succesfully. + """ + self._create_group(self.PILOT_GROUP, isolate_oom=False) + cur_pids = self._read_control("", self.CTRL_PROCS) + self._write_control(self.PILOT_GROUP, self.CTRL_PROCS, cur_pids) + + def _read_control(self, group_name, ctrl_name): + """Reads a control value for the given group_name (relative to our base path). + The returned value varies depending on the value content: + - For a single token value, a string containing that token will be returned. + - For a single line value with space-seperated tokens, a list of tokens will be returned. + - For a multi-line value (where each line is a token), a list of tokens will be returned. + All tokens in the return values are strings. + A RuntimeError will be raised if the control cannot be read. + """ + try: + with open( + os.path.join(self._cgroup_path, group_name, ctrl_name), + encoding="ascii", + ) as file_in: + values = [line.strip() for line in file_in.readlines()] + if " " in values and len(values) == 1: + values = values[0].split(" ") + if len(values) == 1: + values = values[0] + return values + except PermissionError as err: + raise RuntimeError(f"Access denied reading read control '{group_name}/{ctrl_name}'") from err + + def _write_control(self, group_name, ctrl_name, value): + """Writes a control value for a given group_name (relative to our base path). + The value can be a string or an iterable of strings. The values should not + contain any whitespace characters. + A RuntimeError will be raised if the control cannot be set. + """ + try: + ctrl_path = os.path.join(self._cgroup_path, group_name, ctrl_name) + with open(ctrl_path, "w", encoding="ascii") as file_out: + if isinstance(value, str): + value = [value] + for arg in value: + file_out.write(f"{arg}\n") + # Flush is critical here as setting multiple values at the same time may fail + file_out.flush() + except PermissionError as err: + raise RuntimeError(f"Access denied writing control '{group_name}/{ctrl_name}'") from err + except OSError as err: + # This generally happens if we're trying to set a value that is + # considered invalid, for example delegating a controller that isn't enabled + # in the first place. + raise RuntimeError(f"Error writing control '{group_name}/{ctrl_name}' = {value}") from err + + def _get_oom_count(self, slot_name): + """Extracts the OOM counter as an int for the given slot. + Returns an int on success, can return a None if the memory.events + doesn't contain an oom counter or throws RuntimeError on failure. + """ + + def filt(line): + """Filter to find the oom counter from a memory.events file.""" + if line.startswith("oom "): + return int(line[4:]) + return False + + mem_events = os.path.join(self._cgroup_path, slot_name, self.CTRL_MEM_EVENTS) + return self._filter_file(mem_events, filt) + + def _set_limits(self, group_name, cores=None, memory=None, noswap=False): + """Sets the limits for an existing group. + See create_slot for a description of the other parameters. + This will raise a RuntimeError if appyling any of the limits fail to apply. + """ + if cores: + proc_max = int(cores * self.CPU_WEIGHT) + self._write_control(group_name, self.CTRL_CPU_MAX, f"{proc_max} {self.CPU_PERIOD}") + if memory: + self._write_control(group_name, self.CTRL_MEM_MAX, f"{memory}") + if noswap: + self._write_control(group_name, self.CTRL_MEM_SWAP_MAX, "0") + + def _prepare(self): + """Sets up the cgroup tree for the current process. + Should be called once, before using any of the other functions in this class. + + Note that this function (specifcally the _move_init_procs call) assumes that + the list of processes is static. If the process list changes while this is running, + it is likely that this will fail to set things up properly. + """ + self._detect_path() + controllers = self._read_control("", self.CTRL_CONTROLLERS) + if not controllers: + raise RuntimeError("No controllers enabled") + for ctrl in ["cpu", "memory"]: + if not ctrl in controllers: + raise RuntimeError(f"{ctrl} controller not enabled") + self._move_init_procs() + self._write_control("", self.CTRL_SUBTREE, ["+cpu", "+memory"]) + self._ready = True + + def _create_slot(self, slot_name, cores=None, memory=None, noswap=False): + """Creates a slot for a job with the given slot_name. + Cores is a float, number of CPU cores this group may use. + Memory is a string or int, either a number of bytes to limit the group RSS, + or a string limit with a unit suffix, e.g. "1G" as supported by the cgroup memory + controller. + If noswap is set to true, the swap memory limit will be set to 0; this is mostly + useful for testing (where the system may swap memory instead of triggering an + OOM, which may allow a process to use more than the memory limit). + This will raise a RuntimeError if setting up the slot fails. + """ + if not self._ready: + return + self._create_group(slot_name) + self._set_limits(slot_name, cores, memory, noswap) + + def _remove_slot(self, slot_name): + """Removes a slot with the given name. + Can raise usual filesystem OSError if the slot doesn't exist. + """ + if not self._ready: + return + self._remove_group(slot_name) + + def _setup_subproc(self, slot_name): + """A subprocess preexec function for setting up cgroups. + This will move te current process into the given cgroup slot. + On failure, no error will be reported. + """ + # Threading danger! + # There are potential threading issues with preexec functions + # They must not hold any locks that the parent process might already + # be holding, including ones in standard library functions. + # This function should be kept as minimal as possible. + try: + self._write_control(slot_name, self.CTRL_PROCS, f"{os.getpid()}") + except Exception as err: + # We can't even really log here as we're in the set-up + # context of the new proces + pass + + def setUp(self): + """Creates the base cgroup tree if possible. Should be called once + per process before using systemCall. + Returns S_OK/S_ERROR. + """ + try: + self._prepare() + except Exception as err: + # The majority of CGroup failures will be RuntimeError + # However we don't want any unexpected failure to crash the upstream module, + # We just want to continue without cgroup support instead + return S_ERROR(str(err)) + return S_OK() + + def systemCall(self, *args, **kwargs): + """A proxy function for Subprocess.systemCall but will create a cgroup2 slot + if the functionality is available. An optional ceParameters dictionary + may be included, which will be searched for specific cgroup memory options. + Returns the usual S_OK/S_ERROR from Subprocess.systemCall. + """ + preexec_fn = None + slot_name = f"subproc_{os.getpid()}_{self._subproc_num}" + self._subproc_num += 1 + if self._ready: + self.log.info(f"Creating slot cgroup {slot_name}") + cores = None + memory = None + noswap = False + if "ceParameters" in kwargs: + if cpuLimit := kwargs["ceParameters"].get("CPULimit", None): + cores = float(cpuLimit) + if memoryMB := int(kwargs["ceParameters"].get("MemoryLimitMB", 0)): + memory = memoryMB * 1024 * 1024 + if kwargs["ceParameters"].get("MemoryNoSwap", "no").lower() in ("yes", "true"): + noswap = True + try: + self.log.info(f"CGroup Limits, CPU: {cores}, Mem: {memory}, NoSwap: {noswap}") + self._create_slot(slot_name, cores=cores, memory=memory, noswap=noswap) + preexec_fn = functools.partial(CG2Manager._setup_subproc, self, slot_name) + except Exception as err: + self.log.warn("Failed to create slot cgroup:", str(err)) + kwargs["preexec_fn"] = preexec_fn + kwargs.pop("ceParameters", None) + res = Subprocess.systemCall(*args, **kwargs) + if self._ready: + self.log.info(f"Removing slot cgroup {slot_name}") + try: + oom_count = self._get_oom_count(slot_name) + if oom_count: + # Child process triggered an OOM + # We can't readily report this upstream (child process will probably + # fail with an error code), so just log it and continue + self.log.info(f"OOM detected from child process (slot {slot_name})") + self._remove_slot(slot_name) + except Exception as err: + self.log.warn(f"Failed to delete slot {slot_name} cgroup:", str(err)) + return res diff --git a/src/DIRAC/Core/Utilities/ClassAd/ClassAdLight.py b/src/DIRAC/Core/Utilities/ClassAd/ClassAdLight.py index 1943c590bb9..fd1f6417af3 100755 --- a/src/DIRAC/Core/Utilities/ClassAd/ClassAdLight.py +++ b/src/DIRAC/Core/Utilities/ClassAd/ClassAdLight.py @@ -287,7 +287,7 @@ def getAttributeFloat(self, name): value = None return value - def getAttributes(self): + def getAttributes(self) -> list[str]: """Get the list of all the attribute names :return: list of names as strings diff --git a/src/DIRAC/Core/Utilities/CountryMapping.py b/src/DIRAC/Core/Utilities/CountryMapping.py index f97f54a771e..895a2ca28cf 100644 --- a/src/DIRAC/Core/Utilities/CountryMapping.py +++ b/src/DIRAC/Core/Utilities/CountryMapping.py @@ -6,11 +6,11 @@ def getCountryMapping(country): """Determines the associated country from the country code""" mappedCountries = [country] while True: - mappedCountry = gConfig.getValue("/Resources/Countries/%s/AssignedTo" % country, country) + mappedCountry = gConfig.getValue(f"/Resources/Countries/{country}/AssignedTo", country) if mappedCountry == country: break elif mappedCountry in mappedCountries: - return S_ERROR("Circular mapping detected for %s" % country) + return S_ERROR(f"Circular mapping detected for {country}") else: country = mappedCountry mappedCountries.append(mappedCountry) @@ -23,7 +23,7 @@ def getCountryMappingTier1(country): if not res["OK"]: return res mappedCountry = res["Value"] - tier1 = gConfig.getValue("/Resources/Countries/%s/Tier1" % mappedCountry, "") + tier1 = gConfig.getValue(f"/Resources/Countries/{mappedCountry}/Tier1", "") if not tier1: - return S_ERROR("No Tier1 assigned to %s" % mappedCountry) + return S_ERROR(f"No Tier1 assigned to {mappedCountry}") return S_OK(tier1) diff --git a/src/DIRAC/Core/Utilities/DEncode.py b/src/DIRAC/Core/Utilities/DEncode.py index 777aa0a7d43..bae1a2179c1 100755 --- a/src/DIRAC/Core/Utilities/DEncode.py +++ b/src/DIRAC/Core/Utilities/DEncode.py @@ -379,7 +379,7 @@ def encodeDateTime(oValue, eList): # corrected by KGG encode( tTime, eList ) g_dEncodeFunctions[type(tTime)](tTime, eList) else: - raise Exception("Unexpected type %s while encoding a datetime object" % str(type(oValue))) + raise Exception(f"Unexpected type {str(type(oValue))} while encoding a datetime object") def decodeDateTime(data, i): @@ -396,7 +396,7 @@ def decodeDateTime(data, i): elif dataType == _ord("t"): dtObject = datetime.time(*tupleObject) else: - raise Exception("Unexpected type %s while decoding a datetime object" % dataType) + raise Exception(f"Unexpected type {dataType} while decoding a datetime object") return (dtObject, i) @@ -493,7 +493,6 @@ def decodeDict(data, i): oD = {} i += 1 while data[i] != _ord("e"): - if DIRAC_DEBUG_DENCODE_CALLSTACK: # If we have numbers as keys if data[i] in (_ord("i"), _ord("I"), _ord("f")): @@ -522,14 +521,14 @@ def decode(data): if not data: return data # print("DECODE FUNCTION : %s" % g_dDecodeFunctions[ sStream [ iIndex ] ]) - if not isinstance(data, bytes): + if not isinstance(data, (bytes, bytearray)): raise NotImplementedError("This should never happen") return g_dDecodeFunctions[data[0]](data, 0) if __name__ == "__main__": gObject = {2: "3", True: (3, None), 2.0 * 10**20: 2.0 * 10**-10} - print("Initial: %s" % gObject) + print(f"Initial: {gObject}") gData = encode(gObject) - print("Encoded: %s" % gData) + print(f"Encoded: {gData}") print("Decoded: %s, [%s]" % decode(gData)) diff --git a/src/DIRAC/Core/Utilities/DErrno.py b/src/DIRAC/Core/Utilities/DErrno.py index 54bb7797e5c..d8e2dc69889 100644 --- a/src/DIRAC/Core/Utilities/DErrno.py +++ b/src/DIRAC/Core/Utilities/DErrno.py @@ -98,7 +98,7 @@ EMQUKN = 1140 EMQNOM = 1141 EMQCONN = 1142 -# Elasticsearch +# OpenSearch EELNOFOUND = 1146 # Tokens EATOKENFIND = 1150 @@ -182,7 +182,7 @@ 1140: "EMQUKN", 1141: "EMQNOM", 1142: "EMQCONN", - # Elasticsearch + # OpenSearch 1146: "EELNOFOUND", # 115X: Tokens 1150: "EATOKENFIND", @@ -259,7 +259,7 @@ EMQUKN: "Unknown MQ Error", EMQNOM: "No messages", EMQCONN: "MQ connection failure", - # 114X Elasticsearch + # 114X OpenSearch EELNOFOUND: "Index not found", # 115X: Tokens EATOKENFIND: "Can't find a bearer access token.", @@ -305,7 +305,7 @@ def strerror(code: int) -> str: if code == 0: return "Undefined error" - errMsg = "Unknown error %s" % code + errMsg = f"Unknown error {code}" try: errMsg = dStrError[code] @@ -350,7 +350,7 @@ def cmpError(inErr, candidate): elif isinstance(inErr, int): return inErr == candidate else: - raise TypeError("Unknown input error type %s" % type(inErr)) + raise TypeError(f"Unknown input error type {type(inErr)}") def includeExtensionErrors(): @@ -362,7 +362,7 @@ def includeExtensionErrors(): if extension == "DIRAC": continue try: - ext_derrno = importlib.import_module("%s.Core.Utilities.DErrno" % extension) + ext_derrno = importlib.import_module(f"{extension}.Core.Utilities.DErrno") except ImportError: pass else: diff --git a/src/DIRAC/Core/Utilities/DIRACScript.py b/src/DIRAC/Core/Utilities/DIRACScript.py deleted file mode 100644 index fad6a0b3893..00000000000 --- a/src/DIRAC/Core/Utilities/DIRACScript.py +++ /dev/null @@ -1,7 +0,0 @@ -from DIRAC.Core.Base.Script import Script -from DIRAC.Core.Utilities.Decorators import deprecated - -# TODO: remove it in 8.1 -@deprecated("DIRACScript is deprecated, use 'from DIRAC.Core.Base.Script import Script' instead.") -class DIRACScript(Script): - pass diff --git a/src/DIRAC/Core/Utilities/Decorators.py b/src/DIRAC/Core/Utilities/Decorators.py index fdc0ffe3f33..713f28ed3d6 100644 --- a/src/DIRAC/Core/Utilities/Decorators.py +++ b/src/DIRAC/Core/Utilities/Decorators.py @@ -92,7 +92,7 @@ def innerFunc(*args, **kwargs): """ # fail calling the function if environment variable is set if os.environ.get("DIRAC_DEPRECATED_FAIL", None): - raise NotImplementedError("ERROR: using deprecated function or class: %s" % reason) + raise NotImplementedError(f"ERROR: using deprecated function or class: {reason}") # Get the details of the deprecated object if clsName: objName = clsName diff --git a/src/DIRAC/Core/Utilities/Devloader.py b/src/DIRAC/Core/Utilities/Devloader.py index 25bdcfe72a5..45ce17e87cf 100644 --- a/src/DIRAC/Core/Utilities/Devloader.py +++ b/src/DIRAC/Core/Utilities/Devloader.py @@ -38,11 +38,11 @@ def __restart(self): for stuff in self.__stuffToClose: try: - self.__log.always("Closing %s" % stuff) + self.__log.always(f"Closing {stuff}") sys.stdout.flush() stuff.close() except Exception: - gLogger.exception("Could not close %s" % stuff) + gLogger.exception(f"Could not close {stuff}") python = sys.executable os.execl(python, python, *sys.argv) @@ -84,5 +84,5 @@ def __checkFile(self, path): self.__modifyTimes[path] = modified return if self.__modifyTimes[path] != modified: - self.__log.always("File system changed (%s). Restarting..." % (path)) + self.__log.always(f"File system changed ({path}). Restarting...") self.__restart() diff --git a/src/DIRAC/Core/Utilities/DictCache.py b/src/DIRAC/Core/Utilities/DictCache.py index 0b0d5a6b07e..7ed1028fdf9 100644 --- a/src/DIRAC/Core/Utilities/DictCache.py +++ b/src/DIRAC/Core/Utilities/DictCache.py @@ -1,8 +1,15 @@ """ - DictCache. +DictCache and TwoLevelCache """ import datetime import threading +import weakref +from collections import defaultdict +from collections.abc import Callable +from concurrent.futures import Future, ThreadPoolExecutor, wait +from typing import Any + +from cachetools import TTLCache # DIRAC from DIRAC.Core.Utilities.LockRing import LockRing @@ -27,7 +34,6 @@ class MockLockRing: def doNothing(self, *args, **kwargs): """Really does nothing !""" - pass acquire = release = doNothing @@ -64,6 +70,9 @@ def __init__(self, deleteFunction=False, threadLocal=False): # Function to clean the elements self.__deleteFunction = deleteFunction + # Called when this object is deleted or the program ends + self.__finalizer = weakref.finalize(self, _purgeAll, None, self.__cache, self.__deleteFunction) + @property def lock(self): """Return the lock. @@ -105,9 +114,9 @@ def exists(self, cKey, validSeconds=0): # If it's valid return True! if expTime > datetime.datetime.now() + datetime.timedelta(seconds=validSeconds): return True - else: - # Delete expired - self.delete(cKey) + + # Delete expired + self.delete(cKey) return False finally: self.lock.release() @@ -159,9 +168,9 @@ def get(self, cKey, validSeconds=0): # If it's valid return True! if expTime > datetime.datetime.now() + datetime.timedelta(seconds=validSeconds): return self.__cache[cKey]["value"] - else: - # Delete expired - self.delete(cKey) + + # Delete expired + self.delete(cKey) return None finally: self.lock.release() @@ -174,11 +183,11 @@ def showContentsInString(self): self.lock.acquire() try: data = [] - for cKey in self.__cache: - data.append("%s:" % str(cKey)) - data.append("\tExp: %s" % self.__cache[cKey]["expirationTime"]) - if self.__cache[cKey]["value"]: - data.append("\tVal: %s" % self.__cache[cKey]["value"]) + for cKey, cValue in self.__cache.items(): + data.append(f"{cKey}:") + data.append(f"\tExp: {cValue['expirationTime']}") + if cValue["value"]: + data.append(f"\tVal: {cValue['Value']}") return "\n".join(data) finally: self.lock.release() @@ -194,8 +203,8 @@ def getKeys(self, validSeconds=0): try: keys = [] limitTime = datetime.datetime.now() + datetime.timedelta(seconds=validSeconds) - for cKey in self.__cache: - if self.__cache[cKey]["expirationTime"] > limitTime: + for cKey, cValue in self.__cache.items(): + if cValue["expirationTime"] > limitTime: keys.append(cKey) return keys finally: @@ -210,43 +219,136 @@ def purgeExpired(self, expiredInSeconds=0): try: keys = [] limitTime = datetime.datetime.now() + datetime.timedelta(seconds=expiredInSeconds) - for cKey in self.__cache: - if self.__cache[cKey]["expirationTime"] < limitTime: + for cKey, cValue in self.__cache.items(): + if cValue["expirationTime"] < limitTime: keys.append(cKey) - for cKey in keys: + for key in keys: if self.__deleteFunction: - self.__deleteFunction(self.__cache[cKey]["value"]) - del self.__cache[cKey] + self.__deleteFunction(self.__cache[key]["value"]) + del self.__cache[key] finally: self.lock.release() def purgeAll(self, useLock=True): """Purge all entries - CAUTION: useLock parameter should ALWAYS be True except when called from __del__ + CAUTION: useLock parameter should ALWAYS be True :param bool useLock: use lock """ - if useLock: - self.lock.acquire() - try: - for cKey in list(self.__cache): - if self.__deleteFunction: - self.__deleteFunction(self.__cache[cKey]["value"]) - del self.__cache[cKey] - finally: - if useLock: - self.lock.release() - - def __del__(self): - """When the DictCache is deleted, all the entries should be purged. - This is particularly useful when the DictCache manages files - CAUTION: if you carefully read the python doc, you will see all the - caveat of __del__. In particular, no guaranty that it is called... - (https://docs.python.org/2/reference/datamodel.html#object.__del__) + _purgeAll(self.lock if useLock else None, self.__cache, self.__deleteFunction) + + +def _purgeAll(lock, cache, deleteFunction): + """Purge all entries + + This is split in to a helper function to be used by the finalizer without + needing to add a reference to the DictCache object itself. + """ + if lock: + lock.acquire() + try: + for cKey in list(cache): + if deleteFunction: + deleteFunction(cache[cKey]["value"]) + del cache[cKey] + finally: + if lock: + lock.release() + + +class TwoLevelCache: + """A two-level caching system with soft and hard time-to-live (TTL) expiration. + + This cache implements a two-tier caching mechanism to allow for background refresh + of cached values. It uses a soft TTL for quick access and a hard TTL as a fallback, + which helps in reducing latency and maintaining data freshness. + + Attributes: + soft_cache (TTLCache): A cache with a shorter TTL for quick access. + hard_cache (TTLCache): A cache with a longer TTL as a fallback. + locks (defaultdict): Thread-safe locks for each cache key. + futures (dict): Stores ongoing asynchronous population tasks. + pool (ThreadPoolExecutor): Thread pool for executing cache population tasks. + + Args: + soft_ttl (int): Time-to-live in seconds for the soft cache. + hard_ttl (int): Time-to-live in seconds for the hard cache. + max_workers (int): Maximum number of workers in the thread pool. + max_items (int): Maximum number of items in the cache. + + Example: + >>> cache = TwoLevelCache(soft_ttl=60, hard_ttl=300) + >>> def populate_func(): + ... return "cached_value" + >>> value = cache.get("key", populate_func) + + Note: + The cache uses a ThreadPoolExecutor with a maximum of 10 workers to + handle concurrent cache population requests. + """ + + def __init__(self, soft_ttl: int, hard_ttl: int, *, max_workers: int = 10, max_items: int = 1_000_000): + """Initialize the TwoLevelCache with specified TTLs.""" + self.soft_cache = TTLCache(max_items, soft_ttl) + self.hard_cache = TTLCache(max_items, hard_ttl) + self.locks = defaultdict(threading.Lock) + self.futures: dict[str, Future] = {} + self.pool = ThreadPoolExecutor(max_workers=max_workers) + + def get(self, key: str, populate_func: Callable[[], Any]) -> dict: + """Retrieve a value from the cache, populating it if necessary. + + This method first checks the soft cache for the key. If not found, + it checks the hard cache while initiating a background refresh. + If the key is not in either cache, it waits for the populate_func + to complete and stores the result in both caches. + + Locks are used to ensure there is never more than one concurrent + population task for a given key. + + Args: + key (str): The cache key to retrieve or populate. + populate_func (Callable[[], Any]): A function to call to populate the cache + if the key is not found. + + Returns: + Any: The cached value associated with the key. + + Note: + This method is thread-safe and handles concurrent requests for the same key. """ - self.purgeAll(useLock=False) - del self.__lock - if self.__threadLocal: - del self.__threadLocalCache - else: - del self.__sharedCache + if result := self.soft_cache.get(key): + return result + with self.locks[key]: + if key not in self.futures: + self.futures[key] = self.pool.submit(self._work, key, populate_func) + if result := self.hard_cache.get(key): + self.soft_cache[key] = result + return result + # It is critical that ``future`` is waited for outside of the lock as + # _work aquires the lock before filling the caches. This also means + # we can guarantee that the future has not yet been removed from the + # futures dict. + future = self.futures[key] + wait([future]) + return self.hard_cache[key] + + def _work(self, key: str, populate_func: Callable[[], Any]) -> None: + """Internal method to execute the populate_func and update caches. + + This method is intended to be run in a separate thread. It calls the + populate_func, stores the result in both caches, and cleans up the + associated future. + + Args: + key (str): The cache key to populate. + populate_func (Callable[[], Any]): The function to call to get the value. + + Note: + This method is not intended to be called directly by users of the class. + """ + result = populate_func() + with self.locks[key]: + self.futures.pop(key) + self.hard_cache[key] = result + self.soft_cache[key] = result diff --git a/src/DIRAC/Core/Utilities/ElasticSearchDB.py b/src/DIRAC/Core/Utilities/ElasticSearchDB.py index 4d9c32f814b..4f845d550cd 100644 --- a/src/DIRAC/Core/Utilities/ElasticSearchDB.py +++ b/src/DIRAC/Core/Utilities/ElasticSearchDB.py @@ -1,32 +1,38 @@ """ -This class a wrapper around elasticsearch-py. -It is used to query Elasticsearch instances. +This class a wrapper around opensearchpy +It is used to query Opensearch instances. """ -from datetime import datetime -from datetime import timedelta - -import certifi import copy import functools import json +import time +from datetime import datetime, timedelta +from urllib import parse as urlparse + +import certifi +from opensearchpy import OpenSearch +from opensearchpy.exceptions import ( + ConflictError, + NotFoundError, + RequestError, + TransportError, +) +from opensearchpy.exceptions import ( + ConnectionError as ElasticConnectionError, +) +from opensearchpy.helpers import BulkIndexError, bulk try: - from opensearchpy import OpenSearch as Elasticsearch - from opensearch_dsl import Search, Q, A - from opensearchpy.exceptions import ConnectionError, TransportError, NotFoundError, RequestError - from opensearchpy.helpers import BulkIndexError, bulk + from opensearchpy import A, Q, Search except ImportError: - from elasticsearch import Elasticsearch - from elasticsearch_dsl import Search, Q, A - from elasticsearch.exceptions import ConnectionError, TransportError, NotFoundError, RequestError - from elasticsearch.helpers import BulkIndexError, bulk + from opensearch_dsl import A, Q, Search + -from DIRAC import gLogger, S_OK, S_ERROR +from DIRAC import S_ERROR, S_OK, gLogger from DIRAC.Core.Utilities import DErrno, TimeUtilities from DIRAC.FrameworkSystem.Client.BundleDeliveryClient import BundleDeliveryClient - sLog = gLogger.getSubLogger(__name__) @@ -37,9 +43,8 @@ def ifConnected(method): def wrapper_decorator(self, *args, **kwargs): if self._connected: return method(self, *args, **kwargs) - else: - sLog.error("Not connected") - return S_ERROR("Not connected") + sLog.error("Not connected") + return S_ERROR("Not connected") return wrapper_decorator @@ -53,7 +58,6 @@ def generateDocs(data, withTimeStamp=True): :return: doc """ for doc in copy.deepcopy(data): - if withTimeStamp: if "timestamp" not in doc: sLog.warn("timestamp is not given") @@ -73,7 +77,7 @@ def generateDocs(data, withTimeStamp=True): sLog.error("Wrong timestamp", e) doc["timestamp"] = int(TimeUtilities.toEpochMilliSeconds()) - sLog.debug("yielding %s" % doc) + sLog.debug("yielding", doc) yield doc @@ -84,7 +88,7 @@ class ElasticSearchDB: :param str url: the url to the database for example: el.cern.ch:9200 :param str gDebugFile: is used to save the debug information to a file - :param int timeout: the default time out to Elasticsearch + :param int timeout: the default time out to OpenSearch :param int RESULT_SIZE: The number of data points which will be returned by the query. """ @@ -121,20 +125,20 @@ def __init__( :param str client_cert: Client certificate. """ - self.__indexPrefix = indexPrefix self._connected = False if user and password: sLog.debug("Specified username and password") + password = urlparse.quote_plus(password) if port: - self.__url = "https://%s:%s@%s:%d" % (user, password, host, port) + self.__url = f"://{user}:{password}@{host}:{port}" else: - self.__url = f"https://{user}:{password}@{host}" + self.__url = f"://{user}:{password}@{host}" else: sLog.debug("Username and password not specified") if port: - self.__url = "http://%s:%d" % (host, port) + self.__url = f"://{host}:{port}" else: - self.__url = "http://%s" % host + self.__url = f"://{host}" if port: sLog.verbose(f"Connecting to {host}:{port}, useSSL = {useSSL}") @@ -142,6 +146,7 @@ def __init__( sLog.verbose(f"Connecting to {host}, useSSL = {useSSL}") if useSSL: + self.__url = "https" + self.__url if ca_certs: casFile = ca_certs else: @@ -154,11 +159,12 @@ def __init__( else: casFile = retVal["Value"] - self.client = Elasticsearch( + self.client = OpenSearch( self.__url, timeout=self.__timeout, use_ssl=True, verify_certs=True, ca_certs=casFile ) elif useCRT: - self.client = Elasticsearch( + self.__url = "https" + self.__url + self.client = OpenSearch( self.__url, timeout=self.__timeout, use_ssl=True, @@ -168,7 +174,8 @@ def __init__( client_key=client_key, ) else: - self.client = Elasticsearch(self.__url, timeout=self.__timeout) + self.__url = "http" + self.__url + self.client = OpenSearch(self.__url, timeout=self.__timeout) # Before we use the database we try to connect # and retrieve the cluster name @@ -178,26 +185,45 @@ def __init__( # Returns True if the cluster is running, False otherwise result = self.client.info() self.clusterName = result.get("cluster_name", " ") # pylint: disable=no-member - sLog.info("Database info\n", json.dumps(result, indent=4)) + sLog.debug("Database info\n", json.dumps(result, indent=4)) self._connected = True else: - sLog.error("Cannot ping ElasticsearchDB!") - except ConnectionError as e: + sLog.error("Cannot ping OpensearchDB!") + except ElasticConnectionError as e: sLog.error(repr(e)) - def getIndexPrefix(self): - """ - It returns the DIRAC setup. - """ - return self.__indexPrefix + @ifConnected + def addIndexTemplate( + self, name: str, index_patterns: list, mapping: dict, priority: int = 1, settings: dict = None + ) -> dict: + """Adds an index template. + + :param self: self reference + :param str name: index name + :param list index_patterns: list of index patterns to match + :param dict mapping: it is the mapping of the index + """ + if settings is None: + settings = {"index": {"number_of_shards": 1, "number_of_replicas": 1}} + body = { + "index_patterns": index_patterns, + "priority": priority, + "template": {"settings": settings, "mappings": {"properties": mapping}}, + } + try: + res = self.client.indices.put_index_template(name=name, body=body) + return S_OK(res) + except Exception as e: # pylint: disable=broad-except + sLog.exception() + return S_ERROR(e) @ifConnected - def query(self, index, query): + def query(self, index: str, query): """Executes a query and returns its result (uses ES DSL language). :param self: self reference :param str index: index name - :param dict query: It is the query in ElasticSearch DSL language + :param dict query: It is the query in OpenSearch DSL language """ try: @@ -207,18 +233,17 @@ def query(self, index, query): return S_ERROR(re) @ifConnected - def update(self, index, query=None, updateByQuery=True, id=None): + def update(self, index: str, query=None, updateByQuery: bool = True, docID: str = None): """Executes an update of a document, and returns S_OK/S_ERROR - :param self: self reference - :param str index: index name - :param dict query: It is the query in ElasticSearch DSL language - :param bool updateByQuery: A bool to determine update by query or index values using index function. - :param int id: ID for the document to be created. + :param index: index name + :param query: It is the query in OpenSearch DSL language + :param updateByQuery: A bool to determine update by query or index values using index function. + :param docID: ID for the document to be created. """ - sLog.debug(f"Updating {index} with {query}, updateByQuery={updateByQuery}, id={id}") + sLog.debug(f"Updating {index} with {query}, updateByQuery={updateByQuery}, docID={docID}") if not index or not query: return S_ERROR("Missing index or query") @@ -227,21 +252,21 @@ def update(self, index, query=None, updateByQuery=True, id=None): if updateByQuery: esDSLQueryResult = self.client.update_by_query(index=index, body=query) else: - esDSLQueryResult = self.client.index(index=index, body=query, id=id) + esDSLQueryResult = self.client.index(index=index, body=query, id=docID) return S_OK(esDSLQueryResult) except RequestError as re: return S_ERROR(re) @ifConnected - def getDoc(self, index: str, id: str) -> dict: + def getDoc(self, index: str, docID: str) -> dict: """Retrieves a document in an index. :param index: name of the index - :param id: document ID + :param docID: document ID """ - sLog.debug(f"Retrieving document {id} in index {index}") + sLog.debug(f"Retrieving document {docID} in index {index}") try: - return S_OK(self.client.get(index, id)["_source"]) + return S_OK(self.client.get(index, docID)["_source"]) except NotFoundError: sLog.warn("Could not find the document in index", index) return S_OK({}) @@ -249,42 +274,68 @@ def getDoc(self, index: str, id: str) -> dict: return S_ERROR(re) @ifConnected - def updateDoc(self, index: str, id: str, body: dict) -> dict: + def getDocs(self, indexFunc, docIDs: list[str], vo: str) -> list[dict]: + """Efficiently retrieve many documents from an index. + + :param index: name of the index + :param docIDs: document IDs + """ + sLog.debug(f"Retrieving documents {docIDs}") + docs = [{"_index": indexFunc(docID, vo), "_id": docID} for docID in docIDs] + try: + response = self.client.mget({"docs": docs}) + except RequestError as re: + return S_ERROR(re) + else: + results = {int(x["_id"]): x["_source"] if x.get("found") else {} for x in response["docs"]} + return S_OK(results) + + @ifConnected + def updateDoc(self, index: str, docID: str, body) -> dict: """Update an existing document with a script or partial document :param index: name of the index - :param id: document ID + :param docID: document ID :param body: The request definition requires either `script` or partial `doc` """ - sLog.debug(f"Updating document {id} in index {index}") + sLog.debug(f"Updating document {docID} in index {index}") try: - return S_OK(self.client.update(index, id, body)) + self.client.update(index, docID, body) + except ConflictError: + # updates are rather "heavy" operations from ES point of view, needing seqNo to be updated. + # Not ideal, but we just wait and retry. + time.sleep(1) + self.client.update(index, docID, body, params={"retry_on_conflict": 3}) except RequestError as re: return S_ERROR(re) + return S_OK() @ifConnected - def deleteDoc(self, index: str, id: str): + def deleteDoc(self, index: str, docID: str): """Deletes a document in an index. :param index: name of the index - :param id: document ID + :param docID: document ID """ - sLog.debug(f"Deleting document {id} in index {index}") + sLog.debug(f"Deleting document {docID} in index {index}") try: - return S_OK(self.client.delete(index, id)) + return S_OK(self.client.delete(index, docID)) except RequestError as re: return S_ERROR(re) + except NotFoundError: + sLog.warn("Could not find document to be deleted", f"{docID} in index {index}") + return S_OK() @ifConnected - def existsDoc(self, index: str, id: str) -> bool: + def existsDoc(self, index: str, docID: str) -> bool: """Returns information about whether a document exists in an index. :param index: name of the index - :param id: document ID + :param docID: document ID """ - sLog.debug(f"Checking if document {id} in index {index} exists") - return self.client.exists(index, id) + sLog.debug(f"Checking if document {docID} in index {index} exists") + return self.client.exists(index, docID) @ifConnected def _Search(self, indexname): @@ -312,10 +363,10 @@ def getIndexes(self, indexName=None): It returns the available indexes... """ if not indexName: - indexName = self.__indexPrefix - sLog.debug("Getting indices alias of %s" % indexName) + indexName = "" + sLog.debug(f"Getting indices alias of {indexName}") # we only return indexes which belong to a specific prefix for example 'lhcb-production' or 'dirac-production etc. - return list(self.client.indices.get_alias("%s*" % indexName)) + return list(self.client.indices.get_alias(f"{indexName}*")) @ifConnected def getDocTypes(self, indexName): @@ -338,14 +389,14 @@ def getDocTypes(self, indexName): if not result[indexConfig].get("mappings"): # there is a case when the mapping exits and the value is None... # this is usually an empty index or a corrupted index. - sLog.warn("Index does not have mapping %s!" % indexConfig) + sLog.warn(f"Index does not have mapping {indexConfig}!") continue if result[indexConfig].get("mappings"): doctype = result[indexConfig]["mappings"] break # we suppose the mapping of all indexes are the same... if not doctype: - return S_ERROR("%s does not exists!" % indexName) + return S_ERROR(f"{indexName} does not exists!") return S_OK(doctype) @@ -357,7 +408,7 @@ def existingIndex(self, indexName): :param str indexName: the name of the index :returns: S_OK/S_ERROR if the request is successful """ - sLog.debug("Checking existance of index %s" % indexName) + sLog.debug(f"Checking existance of index {indexName}") try: return S_OK(self.client.indices.exists(indexName)) except TransportError as e: @@ -379,19 +430,14 @@ def createIndex(self, indexPrefix, mapping=None, period="day"): sLog.warn("The period is not provided, so using non-periodic indexes names") fullIndex = indexPrefix - res = self.existingIndex(fullIndex) - if not res["OK"]: - return res - elif res["Value"]: - return S_OK(fullIndex) - try: - sLog.info("Create index: ", fullIndex + str(mapping)) - self.client.indices.create(index=fullIndex, body={"mappings": mapping}) # ES7 - + if not mapping: + self.client.indices.create(index=fullIndex) + else: + self.client.indices.create(index=fullIndex, body={"mappings": mapping}) return S_OK(fullIndex) - except Exception as e: # pylint: disable=broad-except - sLog.error("Can not create the index:", repr(e)) + except Exception: # pylint: disable=broad-except + sLog.exception() return S_ERROR("Can not create the index") @ifConnected @@ -404,7 +450,7 @@ def deleteIndex(self, indexName): retVal = self.client.indices.delete(indexName) except NotFoundError: sLog.warn("Index does not exist", indexName) - return S_OK("Noting to delete") + return S_OK("Nothing to delete") except ValueError as e: return S_ERROR(DErrno.EVALUE, e) @@ -446,13 +492,13 @@ def bulk_index(self, indexPrefix, data=None, mapping=None, period="day", withTim """ :param str indexPrefix: index name. :param list data: contains a list of dictionary - :param dict mapping: the mapping used by elasticsearch + :param dict mapping: the mapping used by Opensearch :param str period: Accepts 'day' and 'month'. We can specify which kind of indexes will be created. :param bool withTimeStamp: add timestamp to data, if not there already. :returns: S_OK/S_ERROR """ - sLog.verbose("Bulk indexing", "%d records will be inserted" % len(data)) + sLog.verbose("Bulk indexing", f"{len(data)} records will be inserted in {indexPrefix}") if mapping is None: mapping = {} @@ -474,7 +520,7 @@ def bulk_index(self, indexPrefix, data=None, mapping=None, period="day", withTim res = bulk(client=self.client, index=indexName, actions=generateDocs(data, withTimeStamp)) except (BulkIndexError, RequestError) as e: sLog.exception() - return S_ERROR(e) + return S_ERROR(f"Failed to index by bulk {e!r}") if res[0] == len(data): # we have inserted all documents... @@ -535,7 +581,7 @@ def pingDB(self): connected = False try: connected = self.client.ping() - except ConnectionError as e: + except ElasticConnectionError as e: sLog.error("Cannot connect to the db", repr(e)) return S_OK(connected) @@ -552,7 +598,7 @@ def deleteByQuery(self, indexName, query): except Exception as inst: sLog.error("ERROR: Couldn't delete data") return S_ERROR(inst) - return S_OK("Successfully deleted data from index %s" % indexName) + return S_OK(f"Successfully deleted data from index {indexName}") @staticmethod def generateFullIndexName(indexName, period): @@ -571,6 +617,7 @@ def generateFullIndexName(indexName, period): # Do NOT use datetime.today() because it is not UTC todayUTC = datetime.utcnow().date() + suffix = None if period.lower() == "day": suffix = todayUTC.strftime("%Y-%m-%d") diff --git a/src/DIRAC/Core/Utilities/EventDispatcher.py b/src/DIRAC/Core/Utilities/EventDispatcher.py index 8d80ddd17b8..297fd29da97 100644 --- a/src/DIRAC/Core/Utilities/EventDispatcher.py +++ b/src/DIRAC/Core/Utilities/EventDispatcher.py @@ -20,7 +20,7 @@ def registerEvent(self, eventName): @gEventSync def addListener(self, eventName, functor): if eventName not in self.__events: - return S_ERROR("Event %s is not registered" % eventName) + return S_ERROR(f"Event {eventName} is not registered") if functor in self.__events[eventName]: return S_OK() self.__events[eventName].append(functor) @@ -29,7 +29,7 @@ def addListener(self, eventName, functor): @gEventSync def removeListener(self, eventName, functor): if eventName not in self.__events: - return S_ERROR("Event %s is not registered" % eventName) + return S_ERROR(f"Event {eventName} is not registered") if functor not in self.__events[eventName]: return S_OK() iPos = self.__events[eventName].find(functor) @@ -54,7 +54,7 @@ def __realTrigger(self, eventName, params): gEventSync.lock() try: if eventName not in self.__events: - return S_ERROR("Event %s is not registered" % eventName) + return S_ERROR(f"Event {eventName} is not registered") if eventName in self.__processingEvents: return S_OK(0) eventFunctors = list(self.__events[eventName]) diff --git a/src/DIRAC/Core/Utilities/ExecutorDispatcher.py b/src/DIRAC/Core/Utilities/ExecutorDispatcher.py index c3ddec81cab..d08389ccbbe 100644 --- a/src/DIRAC/Core/Utilities/ExecutorDispatcher.py +++ b/src/DIRAC/Core/Utilities/ExecutorDispatcher.py @@ -219,7 +219,7 @@ def getState(self): return qInfo def deleteTask(self, taskId): - self.__log.verbose("Deleting task %s from waiting queues" % taskId) + self.__log.verbose(f"Deleting task {taskId} from waiting queues") self.__lock.acquire() try: try: @@ -284,9 +284,9 @@ def __init__(self, taskId, taskObj): self.retries = 0 def __repr__(self): - rS = "" else: rS += ">" return rS @@ -358,7 +358,7 @@ def __doPeriodicStuff(self): eTypes = self.__execTypes def addExecutor(self, eId, eTypes, maxTasks=1): - self.__log.verbose("Adding new executor to the pool", "{}: {}".format(eId, ", ".join(eTypes))) + self.__log.verbose("Adding new executor to the pool", f"{eId}: {', '.join(eTypes)}") self.__executorsLock.acquire() try: if eId in self.__idMap: @@ -401,7 +401,7 @@ def removeExecutor(self, eId): try: self.__cbHolder.cbDisconectExecutor(eId) except Exception: - self.__log.exception("Exception while disconnecting agent %s" % eId) + self.__log.exception(f"Exception while disconnecting agent {eId}") for eType in eTypes: self.__fillExecutors(eType) @@ -429,9 +429,7 @@ def __freezeTask(self, taskId, errMsg, eType=False, freezeTime=60): if not isFrozen: self.removeTask(taskId) if self.__failedOnTooFrozen: - self.__cbHolder.cbTaskError( - taskId, eTask.taskObj, "Retried more than 10 times. Last error: %s" % errMsg - ) + self.__cbHolder.cbTaskError(taskId, eTask.taskObj, f"Retried more than 10 times. Last error: {errMsg}") return False return True @@ -467,7 +465,7 @@ def __unfreezeTasks(self, eType=False): try: eTask = self.__tasks[taskId] except KeyError: - self.__log.notice("Removing task %s from the freezer. Somebody has removed the task" % taskId) + self.__log.notice(f"Removing task {taskId} from the freezer. Somebody has removed the task") self.__taskFreezer.pop(iP) continue # Current taskId/eTask is the one to defrost @@ -482,17 +480,17 @@ def __unfreezeTasks(self, eType=False): self.__freezerLock.release() # Out of the lock zone to minimize zone of exclusion eTask.frozenTime += time.time() - eTask.frozenSince - self.__log.verbose("Unfreezed task %s" % taskId) + self.__log.verbose(f"Unfreezed task {taskId}") self.__dispatchTask(taskId, defrozeIfNeeded=False) def __addTaskIfNew(self, taskId, taskObj): self.__tasksLock.acquire() try: if taskId in self.__tasks: - self.__log.verbose("Task %s was already known" % taskId) + self.__log.verbose(f"Task {taskId} was already known") return False self.__tasks[taskId] = ExecutorDispatcher.ETask(taskId, taskObj) - self.__log.verbose("Added task %s" % taskId) + self.__log.verbose(f"Added task {taskId}") return True finally: self.__tasksLock.release() @@ -504,7 +502,7 @@ def getTask(self, taskId): return None def __dispatchTask(self, taskId, defrozeIfNeeded=True): - self.__log.verbose("Dispatching task %s" % taskId) + self.__log.verbose(f"Dispatching task {taskId}") # If task already in executor skip if self.__states.getExecutorOfTask(taskId): return S_OK() @@ -520,19 +518,19 @@ def __dispatchTask(self, taskId, defrozeIfNeeded=True): return result taskObj = self.getTask(taskId) self.removeTask(taskId) - self.__cbHolder.cbTaskError(taskId, taskObj, "Could not dispatch task: %s" % result["Message"]) + self.__cbHolder.cbTaskError(taskId, taskObj, f"Could not dispatch task: {result['Message']}") return S_ERROR("Could not add task. Dispatching task failed") eType = result["Value"] if not eType: - self.__log.verbose("No more executors for task %s" % taskId) + self.__log.verbose(f"No more executors for task {taskId}") return self.removeTask(taskId) self.__log.verbose(f"Next executor type is {eType} for task {taskId}") if eType not in self.__execTypes: if self.__freezeOnUnknownExecutor: self.__log.verbose(f"Executor type {eType} has not connected. Freezing task {taskId}") - self.__freezeTask(taskId, "Unknown executor %s type" % eType, eType=eType, freezeTime=0) + self.__freezeTask(taskId, f"Unknown executor {eType} type", eType=eType, freezeTime=0) return S_OK() self.__log.verbose(f"Executor type {eType} has not connected. Forgetting task {taskId}") return self.removeTask(taskId) @@ -574,7 +572,7 @@ def __getNextExecutor(self, taskId): eTask = self.__tasks[taskId] except KeyError: msg = "Task was deleted prematurely while being dispatched" - self.__log.error(msg, "%s" % taskId) + self.__log.error(msg, f"{taskId}") return S_ERROR(msg) try: result = self.__cbHolder.cbDispatch(taskId, eTask.taskObj, tuple(eTask.pathExecuted)) @@ -609,9 +607,9 @@ def removeTask(self, taskId): try: self.__tasks.pop(taskId) except KeyError: - self.__log.verbose("Task %s is already removed" % taskId) + self.__log.verbose(f"Task {taskId} is already removed") return S_OK() - self.__log.verbose("Removing task %s" % taskId) + self.__log.verbose(f"Removing task {taskId}") eId = self.__states.getExecutorOfTask(taskId) self.__queues.deleteTask(taskId) self.__states.removeTask(taskId) @@ -634,8 +632,8 @@ def __taskReceived(self, taskId, eId): try: eTask = self.__tasks[taskId] except KeyError: - errMsg = "Task %s is not known" % taskId - self.__log.error("Task is not known", "%s" % taskId) + errMsg = f"Task {taskId} is not known" + self.__log.error("Task is not known", f"{taskId}") return S_ERROR(errMsg) if not self.__states.removeTask(taskId, eId): self.__log.info(f"Executor {eId} says it's processed task {taskId} but it didn't have it") @@ -668,10 +666,10 @@ def freezeTask(self, eId, taskId, freezeTime, taskObj=False): try: self.__tasks[taskId].taskObj = taskObj except KeyError: - self.__log.error("Task seems to have been removed while being processed!", "%s" % taskId) + self.__log.error("Task seems to have been removed while being processed!", f"{taskId}") self.__sendTaskToExecutor(eId, eType) return S_OK() - self.__freezeTask(taskId, "Freeze request by %s executor" % eType, eType=eType, freezeTime=freezeTime) + self.__freezeTask(taskId, f"Freeze request by {eType} executor", eType=eType, freezeTime=freezeTime) self.__sendTaskToExecutor(eId, eType) return S_OK() @@ -700,7 +698,7 @@ def taskProcessed(self, eId, taskId, taskObj=False): self.__tasks[taskId].taskObj = taskObj self.__tasks[taskId].pathExecuted.append(eType) except KeyError: - self.__log.error("Task seems to have been removed while being processed!", "%s" % taskId) + self.__log.error("Task seems to have been removed while being processed!", f"{taskId}") self.__sendTaskToExecutor(eId, eType) return S_OK() self.__log.verbose(f"Executor {eId} processed task {taskId}") @@ -710,8 +708,8 @@ def taskProcessed(self, eId, taskId, taskObj=False): def retryTask(self, eId, taskId): if taskId not in self.__tasks: - errMsg = "Task %s is not known" % taskId - self.__log.error("Task is not known", "%s" % taskId) + errMsg = f"Task {taskId} is not known" + self.__log.error("Task is not known", f"{taskId}") return S_ERROR(errMsg) if not self.__states.removeTask(taskId, eId): self.__log.info(f"Executor {eId} says it's processed task {taskId} but it didn't have it") @@ -721,27 +719,27 @@ def retryTask(self, eId, taskId): try: self.__tasks[taskId].retries += 1 except KeyError: - self.__log.error("Task seems to have been removed while waiting for retry!", "%s" % taskId) + self.__log.error("Task seems to have been removed while waiting for retry!", f"{taskId}") return S_OK() return self.__dispatchTask(taskId) def __fillExecutors(self, eType, defrozeIfNeeded=True): if defrozeIfNeeded: - self.__log.verbose("Unfreezing tasks for %s" % eType) + self.__log.verbose(f"Unfreezing tasks for {eType}") self.__unfreezeTasks(eType) - self.__log.verbose("Filling %s executors" % eType) + self.__log.verbose(f"Filling {eType} executors") eId = self.__states.getIdleExecutor(eType) while eId: result = self.__sendTaskToExecutor(eId, eType) if not result["OK"]: - self.__log.error("Could not send task to executor", "%s" % result["Message"]) + self.__log.error("Could not send task to executor", f"{result['Message']}") else: if not result["Value"]: # No more tasks for eType break - self.__log.verbose("Task {} was sent to {}".format(result["Value"], eId)) + self.__log.verbose(f"Task {result['Value']} was sent to {eId}") eId = self.__states.getIdleExecutor(eType) - self.__log.verbose("No more idle executors for %s" % eType) + self.__log.verbose(f"No more idle executors for {eType}") def __sendTaskToExecutor(self, eId, eTypes=False, checkIdle=False): if checkIdle and self.__states.freeSlots(eId) == 0: @@ -749,7 +747,7 @@ def __sendTaskToExecutor(self, eId, eTypes=False, checkIdle=False): try: searchTypes = list(reversed(self.__idMap[eId])) except KeyError: - self.__log.verbose("Executor %s invalid/disconnected" % eId) + self.__log.verbose(f"Executor {eId} invalid/disconnected") return S_ERROR("Invalid executor") if eTypes: if not isinstance(eTypes, (list, tuple)): @@ -762,7 +760,7 @@ def __sendTaskToExecutor(self, eId, eTypes=False, checkIdle=False): searchTypes.append(eType) pData = self.__queues.popTask(searchTypes) if pData is None: - self.__log.verbose("No more tasks for %s" % eTypes) + self.__log.verbose(f"No more tasks for {eTypes}") return S_OK() taskId, eType = pData self.__log.verbose(f"Sending task {taskId} to {eType}={eId}") @@ -785,5 +783,5 @@ def __msgTaskToExecutor(self, taskId, eId, eType): self.__log.fatal(errMsg) raise ValueError(errMsg) if not result["OK"]: - self.__log.error("Failed to cbSendTask", "%r" % result) + self.__log.error("Failed to cbSendTask", f"{result!r}") raise RuntimeError(result) diff --git a/src/DIRAC/Core/Utilities/Extensions.py b/src/DIRAC/Core/Utilities/Extensions.py index 7deb87632b8..9a7d55331e9 100644 --- a/src/DIRAC/Core/Utilities/Extensions.py +++ b/src/DIRAC/Core/Utilities/Extensions.py @@ -1,13 +1,13 @@ """Helpers for working with extensions to DIRAC""" import argparse -from collections import defaultdict import fnmatch -from importlib.machinery import PathFinder import functools import importlib import os import pkgutil import sys +from collections import defaultdict +from importlib.machinery import PathFinder import importlib_metadata as metadata import importlib_resources @@ -139,7 +139,7 @@ def getExtensionMetadata(extensionName): def recurseImport(modName, parentModule=None, hideExceptions=False): - from DIRAC import S_OK, S_ERROR, gLogger + from DIRAC import S_ERROR, S_OK, gLogger if parentModule is not None: raise NotImplementedError(parentModule) @@ -152,7 +152,7 @@ def recurseImport(modName, parentModule=None, hideExceptions=False): # and not for example a missing dependency in the handler we are trying to import if isinstance(excp, ModuleNotFoundError) and modName.startswith(notFoundModule): return S_OK() - errMsg = "Can't load %s" % modName + errMsg = f"Can't load {modName}" if not hideExceptions: gLogger.exception(errMsg) return S_ERROR(errMsg) diff --git a/src/DIRAC/Core/Utilities/File.py b/src/DIRAC/Core/Utilities/File.py index 8dfacf4ae5e..8ef60ca375f 100755 --- a/src/DIRAC/Core/Utilities/File.py +++ b/src/DIRAC/Core/Utilities/File.py @@ -11,6 +11,9 @@ import sys import re import errno +import stat +import tempfile +from contextlib import contextmanager # Translation table of a given unit to Bytes # I know, it should be kB... @@ -24,12 +27,19 @@ } -def mkDir(path): - """Emulate 'mkdir -p path' (if path exists already, don't raise an exception)""" +def mkDir(path, mode=None): + """Emulate 'mkdir -p path' (if path exists already, don't raise an exception) + + :param str path: directory hierarchy to create + :param int mode: Use this mode as the mode for new directories, use python default if None. + """ try: if os.path.isdir(path): return - os.makedirs(path) + if mode is None: + os.makedirs(path) + else: + os.makedirs(path, mode) except OSError as osError: if osError.errno == errno.EEXIST and os.path.isdir(path): pass @@ -132,7 +142,7 @@ def checkGuid(guid): return False -def getSize(fileName): +def getSize(fileName: os.PathLike) -> int: """Get size of a file. :param string fileName: name of file to be checked @@ -246,6 +256,27 @@ def convertSizeUnits(size, srcUnit, dstUnit): return -sys.maxsize +@contextmanager +def secureOpenForWrite(filename=None, *, text=True): + """Securely open a file for writing. + + If filename is not provided, a file is created in tempfile.gettempdir(). + The file always created with mode 600. + + :param string filename: name of file to be opened + """ + if filename: + fd = os.open( + path=filename, + flags=os.O_WRONLY | os.O_CREAT | os.O_TRUNC, + mode=stat.S_IRUSR | stat.S_IWUSR, + ) + else: + fd, filename = tempfile.mkstemp(text=text) + with open(fd, "w" if text else "wb", encoding="utf-8" if text else None) as fd: + yield fd, filename + + if __name__ == "__main__": for p in sys.argv[1:]: print(f"{p} : {getGlobbedTotalSize(p)} bytes") diff --git a/src/DIRAC/Core/Utilities/Glue2.py b/src/DIRAC/Core/Utilities/Glue2.py index 26e8e7e670f..285207afe3c 100644 --- a/src/DIRAC/Core/Utilities/Glue2.py +++ b/src/DIRAC/Core/Utilities/Glue2.py @@ -58,10 +58,10 @@ def getGlue2CEInfo(vo, host=None): sLog.debug(f"Policy {policyID} does not point to computing information") continue sLog.verbose(f"{siteName} policy {policyID} pointing to {shareID} ") - sLog.debug("Policy values:\n%s" % pformat(policyValues)) - shareFilter += "(GLUE2ShareID=%s)" % shareID + sLog.debug(f"Policy values:\n{pformat(policyValues)}") + shareFilter += f"(GLUE2ShareID={shareID})" - filt = "(&(objectClass=GLUE2Share)(|%s))" % shareFilter + filt = f"(&(objectClass=GLUE2Share)(|{shareFilter}))" shareRes = ldapsearchBDII(filt=filt, attr=None, host=host, base="o=glue", selectionString="GLUE2") if not shareRes["OK"]: sLog.error("Could not get share information", shareRes["Message"]) @@ -73,16 +73,16 @@ def getGlue2CEInfo(vo, host=None): if "GLUE2ComputingShare" not in shareInfo["objectClass"]: sLog.debug(f"Share {shareID!r} is not a ComputingShare: \n{pformat(shareInfo)}") continue - sLog.debug("Found computing share:\n%s" % pformat(shareInfo)) + sLog.debug(f"Found computing share:\n{pformat(shareInfo)}") siteName = shareInfo["attr"]["dn"].split("GLUE2DomainID=")[1].split(",", 1)[0] shareInfoLists.setdefault(siteName, []).append(shareInfo["attr"]) siteInfo = __getGlue2ShareInfo(host, shareInfoLists) if not siteInfo["OK"]: - sLog.error("Could not get CE info for", "{}: {}".format(shareID, siteInfo["Message"])) + sLog.error("Could not get CE info for", f"{shareID}: {siteInfo['Message']}") return siteInfo siteDict = siteInfo["Value"] - sLog.debug("Found Sites:\n%s" % pformat(siteDict)) + sLog.debug(f"Found Sites:\n{pformat(siteDict)}") sitesWithoutShares = set(siteDict) - listOfSitesWithPolicies if sitesWithoutShares: sLog.error("Found some sites without any shares", pformat(sitesWithoutShares)) @@ -198,17 +198,8 @@ def __getGlue2ShareInfo(host, shareInfoLists): shareEndpoints = [shareEndpoints] for endpoint in shareEndpoints: ceType = endpoint.rsplit(".", 1)[1] - # get queue Name, in CREAM this is behind GLUE2entityOtherInfo... - if ceType == "CREAM": - for otherInfo in shareInfoDict["GLUE2EntityOtherInfo"]: - if otherInfo.startswith("CREAMCEId"): - queueName = otherInfo.split("/", 1)[1] - # creamCEs are EOL soon, ignore any info they have - if queueInfo.pop("NumberOfProcessors", 1) != 1: - sLog.verbose("Ignoring MaxSlotsPerJob option for CreamCE", endpoint) - # HTCondorCE, htcondorce - elif ceType.lower().endswith("htcondorce"): + if ceType.lower().endswith("htcondorce"): ceType = "HTCondorCE" queueName = "condor" @@ -224,14 +215,27 @@ def __getGlue2ShareInfo(host, shareInfoLists): ceInfo["Queues"] = existingQueues cesDict[ceName].update(ceInfo) - # ARC CEs do not have endpoints, we have to try something else to get the information about the queue etc. + # ARC CEs do not have share(?) endpoints, we have to try something else to get the information about the + # queue etc. try: if not shareEndpoints and shareInfoDict["GLUE2ShareID"].startswith("urn:ogf"): exeInfo = dict(exeInfo) # silence pylint about tuples - queueInfo["GlueCEImplementationName"] = "ARC" + computingInfo = shareInfoDict["GLUE2ComputingShareComputingEndpointForeignKey"] + computingInfo = computingInfo if isinstance(computingInfo, list) else [computingInfo] + for entry in computingInfo: + if "emies" in entry: + # has an entry like + # urn:ogf:ComputingEndpoint:ce01.tier2.hep.manchester.ac.uk:emies:https://ce01.tier2.hep.manchester.ac.uk:443/arex + queueInfo["GlueCEImplementationName"] = "AREX" + break + else: + sLog.error("No AREX for", siteName) + raise AttributeError() + + exeInfo = dict(exeInfo) # silence pylint about tuples managerName = exeInfo.pop("MANAGER", "").split(" ", 1)[0].rsplit(":", 1)[1] managerName = managerName.capitalize() if managerName == "condor" else managerName - queueName = "nordugrid-{}-{}".format(managerName, shareInfoDict["GLUE2ComputingShareMappingQueue"]) + queueName = f"nordugrid-{managerName}-{shareInfoDict['GLUE2ComputingShareMappingQueue']}" ceName = shareInfoDict["GLUE2ShareID"].split("ComputingShare:")[1].split(":")[0] cesDict.setdefault(ceName, {}) existingQueues = dict(cesDict[ceName].get("Queues", {})) @@ -256,13 +260,13 @@ def __getGlue2ExecutionEnvironmentInfo(host, executionEnvironments): for exeEnvs in breakListIntoChunks(executionEnvironments, 1000): exeFilter = "" for execEnv in exeEnvs: - exeFilter += "(GLUE2ResourceID=%s)" % execEnv - filt = "(&(objectClass=GLUE2ExecutionEnvironment)(|%s))" % exeFilter + exeFilter += f"(GLUE2ResourceID={execEnv})" + filt = f"(&(objectClass=GLUE2ExecutionEnvironment)(|{exeFilter}))" response = ldapsearchBDII(filt=filt, attr=None, host=host, base="o=glue", selectionString="GLUE2") if not response["OK"]: return response if not response["Value"]: - sLog.error("No information found for %s" % executionEnvironments) + sLog.error(f"No information found for {executionEnvironments}") continue listOfValues += response["Value"] if not listOfValues: diff --git a/src/DIRAC/Core/Utilities/Graphs/BarGraph.py b/src/DIRAC/Core/Utilities/Graphs/BarGraph.py index ca0003873cf..53a194ee6dd 100644 --- a/src/DIRAC/Core/Utilities/Graphs/BarGraph.py +++ b/src/DIRAC/Core/Utilities/Graphs/BarGraph.py @@ -29,7 +29,6 @@ class BarGraph(PlotBase): """ def __init__(self, data, ax, prefs, *args, **kw): - PlotBase.__init__(self, data, ax, prefs, *args, **kw) if "span" in self.prefs: self.width = self.prefs["span"] @@ -41,7 +40,6 @@ def __init__(self, data, ax, prefs, *args, **kw): self.width = (max(self.gdata.all_keys) - min(self.gdata.all_keys)) / (nKeys - 1) def draw(self): - PlotBase.draw(self) self.x_formatter_cb(self.ax) @@ -67,8 +65,8 @@ def draw(self): start_plot = 0 end_plot = 0 if "starttime" in self.prefs and "endtime" in self.prefs: - start_plot = date2num(datetime.datetime.fromtimestamp(to_timestamp(self.prefs["starttime"]))) - end_plot = date2num(datetime.datetime.fromtimestamp(to_timestamp(self.prefs["endtime"]))) + start_plot = date2num(datetime.datetime.utcfromtimestamp(to_timestamp(self.prefs["starttime"]))) + end_plot = date2num(datetime.datetime.utcfromtimestamp(to_timestamp(self.prefs["endtime"]))) nKeys = self.gdata.getNumberOfKeys() tmp_b = [] @@ -181,7 +179,6 @@ def x_formatter_cb(self, ax): xmin = 0 ax.set_xlim(xmin=xmin, xmax=len(ticks)) elif self.gdata.key_type == "time": - # ax.set_xlim( xmin=self.begin_num,xmax=self.end_num ) dl = PrettyDateLocator() df = PrettyDateFormatter(dl) diff --git a/src/DIRAC/Core/Utilities/Graphs/CurveGraph.py b/src/DIRAC/Core/Utilities/Graphs/CurveGraph.py index 07a57a34e08..6799f09c01c 100644 --- a/src/DIRAC/Core/Utilities/Graphs/CurveGraph.py +++ b/src/DIRAC/Core/Utilities/Graphs/CurveGraph.py @@ -23,11 +23,9 @@ class CurveGraph(PlotBase): """ def __init__(self, data, ax, prefs, *args, **kw): - PlotBase.__init__(self, data, ax, prefs, *args, **kw) def draw(self): - PlotBase.draw(self) self.x_formatter_cb(self.ax) @@ -37,8 +35,8 @@ def draw(self): start_plot = 0 end_plot = 0 if "starttime" in self.prefs and "endtime" in self.prefs: - start_plot = date2num(datetime.datetime.fromtimestamp(to_timestamp(self.prefs["starttime"]))) - end_plot = date2num(datetime.datetime.fromtimestamp(to_timestamp(self.prefs["endtime"]))) + start_plot = date2num(datetime.datetime.utcfromtimestamp(to_timestamp(self.prefs["starttime"]))) + end_plot = date2num(datetime.datetime.utcfromtimestamp(to_timestamp(self.prefs["endtime"]))) labels = self.gdata.getLabels() labels.reverse() diff --git a/src/DIRAC/Core/Utilities/Graphs/Graph.py b/src/DIRAC/Core/Utilities/Graphs/Graph.py index 8e958fe55fe..67e1ccd3769 100644 --- a/src/DIRAC/Core/Utilities/Graphs/Graph.py +++ b/src/DIRAC/Core/Utilities/Graphs/Graph.py @@ -6,26 +6,26 @@ """ import datetime -import time +import importlib import os - +import time from matplotlib.backends.backend_agg import FigureCanvasAgg from matplotlib.figure import Figure -from DIRAC.Core.Utilities.Graphs.GraphUtilities import pixelToPoint, evalPrefs, to_timestamp, add_time_to_title + +from DIRAC import gLogger from DIRAC.Core.Utilities.Graphs.GraphData import GraphData +from DIRAC.Core.Utilities.Graphs.GraphUtilities import add_time_to_title, evalPrefs, pixelToPoint, to_timestamp from DIRAC.Core.Utilities.Graphs.Legend import Legend -from DIRAC import gLogger DEBUG = 0 class Graph: - def __init__(self, *args, **kw): - super().__init__(*args, **kw) + """Base class for all other Graphs""" def layoutFigure(self, legend): - prefs = self.prefs + nsublines = left = bottom = None # Get the main Figure object # self.figure = Figure() @@ -182,9 +182,8 @@ def makeTextGraph(self, text="Empty image"): figure.text(0.5, 0.5, text, horizontalalignment="center", size=pixelToPoint(text_size, dpi)) def makeGraph(self, data, *args, **kw): - start = time.time() - + plot_type = None # Evaluate all the preferences self.prefs = evalPrefs(*args, **kw) prefs = self.prefs @@ -275,18 +274,13 @@ def makeGraph(self, data, *args, **kw): for i in range(nPlots): plot_type = plot_prefs[i]["plot_type"] try: - # TODO: Remove when we moved to python3 - exec("import %s" % plot_type) - except ImportError: - print("Trying to use python like import") - try: - exec("from . import %s" % plot_type) - except ImportError as x: - print(f"Failed to import graph type {plot_type}: {str(x)}") - return None + plotModule = importlib.import_module(f"DIRAC.Core.Utilities.Graphs.{plot_type}") + except ModuleNotFoundError as x: + print(f"Failed to import graph type {plot_type}: {str(x)}") + return None ax = plot_axes[i] - plot = eval(f"{plot_type}.{plot_type}(graphData[i],ax,plot_prefs[i])") + plot = getattr(plotModule, plot_type)(graphData[i], ax, plot_prefs[i]) plot.draw() if DEBUG: @@ -342,8 +336,8 @@ def drawWaterMark(self, imagePath=None): ax_wm.axis("off") ax_wm.set_frame_on(False) ax_wm.set_clip_on(False) - except Exception as e: - print(e) + except Exception: + gLogger.exception("Caught exception") def writeGraph(self, fname, fileFormat="PNG"): """Write out the resulting graph to a file with fname in a given format""" @@ -352,4 +346,4 @@ def writeGraph(self, fname, fileFormat="PNG"): if fileFormat.lower() == "png": self.canvas.print_png(fname) else: - gLogger.error("File format '%s' is not supported!" % fileFormat) + gLogger.error(f"File format '{fileFormat}' is not supported!") diff --git a/src/DIRAC/Core/Utilities/Graphs/GraphData.py b/src/DIRAC/Core/Utilities/Graphs/GraphData.py index 7c27535fc68..4abc4b37b61 100644 --- a/src/DIRAC/Core/Utilities/Graphs/GraphData.py +++ b/src/DIRAC/Core/Utilities/Graphs/GraphData.py @@ -53,7 +53,6 @@ def get_key_type(keys): class GraphData: def __init__(self, data={}): - self.truncated = 0 self.all_keys = [] self.labels = [] @@ -76,7 +75,6 @@ def setData(self, data): self.initialize() def initialize(self, key_type=None): - keys = list(self.data) if not keys: print("GraphData Error: empty data") @@ -110,13 +108,11 @@ def initialize(self, key_type=None): self.sortLabels() def expandKeys(self): - if not self.plotdata: for sub in self.subplots: self.subplots[sub].expandKeys(self.all_keys) def isSimplePlot(self): - return self.plotdata is not None def sortLabels(self, sort_type="max_value", reverse_order=False): @@ -198,7 +194,9 @@ def makeNumKeys(self): self.all_num_keys.append(next) next += 1 elif self.key_type == "time": - self.all_num_keys = [date2num(datetime.datetime.fromtimestamp(to_timestamp(key))) for key in self.all_keys] + self.all_num_keys = [ + date2num(datetime.datetime.utcfromtimestamp(to_timestamp(key))) for key in self.all_keys + ] elif self.key_type == "numeric": self.all_num_keys = [float(key) for key in self.all_keys] @@ -247,11 +245,9 @@ def getStringMap(self): return self.all_string_map def getNumberOfKeys(self): - return len(self.all_keys) def getNumberOfLabels(self): - if self.truncated: return self.truncated + 1 else: @@ -364,7 +360,6 @@ class PlotData: """PlotData class is a container for a one dimensional plot data""" def __init__(self, data, single=True, key_type=None): - self.key_type = "unknown" if not data: print("PlotData Error: empty data") @@ -390,7 +385,6 @@ def __init__(self, data, single=True, key_type=None): self.initialize() def initialize(self): - if self.key_type == "string": self.keys = self.sortKeys("weight") else: @@ -540,7 +534,6 @@ def parseData(self, key_type=None): self.keys = list(self.parsed_data) def makeCumulativePlot(self): - if not self.sorted_keys: self.sortKeys() @@ -559,30 +552,15 @@ def makeCumulativePlot(self): self.last_value = float(self.values[-1]) def getPlotData(self): - return self.parsed_data def getPlotErrors(self): - return self.parsed_errors def getPlotNumData(self): - return zip(self.num_keys, self.values, self.errors) - def getPlotDataForKeys(self, keys): - - result_pairs = [] - for key in keys: - if key in self.parsed_data: - result_pairs.append(key, self.parsed_data[key], self.parsed_errors[key]) - else: - result_pairs.append(key, None, 0.0) - - return result_pairs - def getPlotDataForNumKeys(self, num_keys, zeroes=False): - result_pairs = [] for num_key in num_keys: try: @@ -600,21 +578,16 @@ def getPlotDataForNumKeys(self, num_keys, zeroes=False): return result_pairs def getKeys(self): - return self.keys def getNumKeys(self): - return self.num_keys def getValues(self): - return self.values def getMaxValue(self): - return max(self.values) def getMinValue(self): - return min(self.values) diff --git a/src/DIRAC/Core/Utilities/Graphs/GraphUtilities.py b/src/DIRAC/Core/Utilities/Graphs/GraphUtilities.py index dffec7ee8e4..d75f9dc0b7b 100644 --- a/src/DIRAC/Core/Utilities/Graphs/GraphUtilities.py +++ b/src/DIRAC/Core/Utilities/Graphs/GraphUtilities.py @@ -70,9 +70,10 @@ def convert_to_datetime(dstring): else: results = eval(str(dstring), {"__builtins__": None, "time": time, "math": math}, {}) if isinstance(results, (int, float)): - results = datetime.datetime.fromtimestamp(int(results)) + # Use utcfromtimestamp for UTC time + results = datetime.datetime.utcfromtimestamp(int(results)) elif isinstance(results, datetime.datetime): - pass + results = results.astimezone(datetime.timezone.utc) # Ensure in UTC else: raise ValueError("Unknown datetime type!") except Exception: @@ -80,8 +81,8 @@ def convert_to_datetime(dstring): for dateformat in datestrings: try: t = time.strptime(dstring, dateformat) - timestamp = calendar.timegm(t) # -time.timezone - results = datetime.datetime.fromtimestamp(timestamp) + timestamp = calendar.timegm(t) # Convert to UTC timestamp + results = datetime.datetime.utcfromtimestamp(timestamp) break except Exception: pass @@ -89,18 +90,17 @@ def convert_to_datetime(dstring): try: dstring = dstring.split(".", 1)[0] t = time.strptime(dstring, dateformat) - timestamp = time.mktime(t) # -time.timezone - results = datetime.datetime.fromtimestamp(timestamp) + timestamp = calendar.timegm(t) # Convert to UTC timestamp + results = datetime.datetime.utcfromtimestamp(timestamp) except Exception: raise ValueError( "Unable to create time from string!\nExpecting " - "format of: '12/06/06 12:54:67'\nRecieved:%s" % orig_string + "format of: '12/06/06 12:54:67'\nReceived:%s" % orig_string ) return results def to_timestamp(val): - try: v = float(val) if v > 1000000000 and v < 1900000000: @@ -109,8 +109,8 @@ def to_timestamp(val): pass val = convert_to_datetime(val) - # return calendar.timegm( val.timetuple() ) - return time.mktime(val.timetuple()) + return calendar.timegm(val.timetuple()) + # return time.mktime(val.timetuple()) # If the graph has more than `hour_switch` minutes, we print @@ -146,6 +146,7 @@ def add_time_to_title(begin, end, metadata={}): representing 168 Hours, but needed the format to show the date as well as the time. """ + format_str = None if "span" in metadata: interval = metadata["span"] else: @@ -180,10 +181,10 @@ def add_time_to_title(begin, end, metadata={}): format_name = "Seconds" time_slice = 1 - begin_tuple = time.localtime(begin) - end_tuple = time.localtime(end) + begin_tuple = time.gmtime(begin) + end_tuple = time.gmtime(end) added_title = "%i %s from " % (int((end - begin) / time_slice), format_name) - added_title += time.strftime("%s to" % format_str, begin_tuple) + added_title += time.strftime(f"{format_str} to", begin_tuple) if time_slice < 86400: add_utc = " UTC" else: @@ -367,13 +368,10 @@ def get_locator(self, dmin, dmax): locator = RRuleLocator(rrule, self.tz) locator.set_axis(self.axis) - locator.set_view_interval(*self.axis.get_view_interval()) - locator.set_data_interval(*self.axis.get_data_interval()) return locator def pretty_float(num): - if num > 1000: return comma_format(int(num)) @@ -388,7 +386,7 @@ def pretty_float(num): try: retval = format % float(num) except Exception: - raise Exception("Unable to convert %s into a float." % (str(num))) + raise Exception(f"Unable to convert {str(num)} into a float.") return retval @@ -465,7 +463,6 @@ def makeDataFromCSV(csv): def darkenColor(color, factor=2): - c1 = int(color[1:3], 16) c2 = int(color[3:5], 16) c3 = int(color[5:7], 16) diff --git a/src/DIRAC/Core/Utilities/Graphs/Legend.py b/src/DIRAC/Core/Utilities/Graphs/Legend.py index fc5601c4f4c..00aa5a8a662 100644 --- a/src/DIRAC/Core/Utilities/Graphs/Legend.py +++ b/src/DIRAC/Core/Utilities/Graphs/Legend.py @@ -3,6 +3,7 @@ The DIRAC Graphs package is derived from the GraphTool plotting package of the CMS/Phedex Project by ... """ + from matplotlib.patches import Rectangle from matplotlib.text import Text from matplotlib.figure import Figure @@ -15,14 +16,13 @@ class Legend: def __init__(self, data=None, axes=None, *aw, **kw): - self.text_size = 0 self.column_width = 0 self.labels = {} if isinstance(data, dict): for label, ddict in data.items(): # self.labels[label] = pretty_float(max([ float(x) for x in ddict.values() if x ]) ) - self.labels[label] = "%.1f" % max(float(x) for x in ddict.values() if x) + self.labels[label] = f"{max(float(x) for x in ddict.values() if x):.1f}" elif isinstance(data, GraphData): self.labels = data.getLabels() else: @@ -45,16 +45,13 @@ def __init__(self, data=None, axes=None, *aw, **kw): self.__get_column_width() def dumpPrefs(self): - for key in self.prefs: print(key.rjust(20), ":", str(self.prefs[key]).ljust(40)) def setLabels(self, labels): - self.labels = labels def setAxes(self, axes): - self.ax = axes self.canvas = self.ax.figure.canvas self.ax.set_axis_off() @@ -67,6 +64,7 @@ def getLegendSize(self): legend_padding = float(self.prefs["legend_padding"]) legend_text_size = self.prefs.get("legend_text_size", self.prefs["text_size"]) legend_text_padding = self.prefs.get("legend_text_padding", self.prefs["text_padding"]) + legend_max_height = -1 if legend_position in ["right", "left"]: # One column in case of vertical legend legend_width = self.column_width + legend_padding @@ -86,7 +84,6 @@ def getLegendSize(self): return legend_width, legend_height, legend_max_height def __get_legend_text_size(self): - text_size = self.prefs["text_size"] text_padding = self.prefs["text_padding"] legend_text_size = self.prefs.get("legend_text_size", text_size) @@ -94,7 +91,6 @@ def __get_legend_text_size(self): return legend_text_size, legend_text_padding def __get_column_width(self): - max_length = 0 max_column_text = "" flag = self.prefs.get("legend_numbers", True) @@ -112,12 +108,12 @@ def __get_column_width(self): if isinstance(num, int): numString = str(num) else: - numString = "%.1f" % float(num) + numString = f"{float(num):.1f}" max_column_text = f"{str(label)} {numString}" if unit: max_column_text += "%" else: - max_column_text = "%s " % str(label) + max_column_text = f"{str(label)} " figure = Figure() canvas = FigureCanvasAgg(figure) @@ -135,7 +131,6 @@ def __get_column_width(self): ) def draw(self): - dpi = self.prefs["dpi"] ax_xsize = self.ax.get_window_extent().width ax_ysize = self.ax.get_window_extent().height @@ -163,9 +158,9 @@ def draw(self): percent_flag = self.prefs.get("legend_unit", "") if num_flag: if percent_flag == "%": - num = "%.1f" % num + "%" + num = f"{num:.1f}" + "%" else: - num = "%.1f" % num + num = f"{num:.1f}" else: num = None color = self.palette.getColor(label) diff --git a/src/DIRAC/Core/Utilities/Graphs/LineGraph.py b/src/DIRAC/Core/Utilities/Graphs/LineGraph.py index f7c7e1d55ed..c8868e0f843 100644 --- a/src/DIRAC/Core/Utilities/Graphs/LineGraph.py +++ b/src/DIRAC/Core/Utilities/Graphs/LineGraph.py @@ -25,11 +25,9 @@ class LineGraph(PlotBase): """ def __init__(self, data, ax, prefs, *args, **kw): - PlotBase.__init__(self, data, ax, prefs, *args, **kw) def draw(self): - PlotBase.draw(self) self.x_formatter_cb(self.ax) @@ -51,8 +49,8 @@ def draw(self): start_plot = 0 end_plot = 0 if "starttime" in self.prefs and "endtime" in self.prefs: - start_plot = date2num(datetime.datetime.fromtimestamp(to_timestamp(self.prefs["starttime"]))) - end_plot = date2num(datetime.datetime.fromtimestamp(to_timestamp(self.prefs["endtime"]))) + start_plot = date2num(datetime.datetime.utcfromtimestamp(to_timestamp(self.prefs["starttime"]))) + end_plot = date2num(datetime.datetime.utcfromtimestamp(to_timestamp(self.prefs["endtime"]))) self.polygons = [] seq_b = [(self.gdata.max_num_key, 0.0), (self.gdata.min_num_key, 0.0)] @@ -71,7 +69,6 @@ def draw(self): labels = [(color, 0.0)] for label, num in labels: - color = self.palette.getColor(label) ind = 0 tmp_x = [] diff --git a/src/DIRAC/Core/Utilities/Graphs/Palette.py b/src/DIRAC/Core/Utilities/Graphs/Palette.py index 460d039f929..a9e28205886 100644 --- a/src/DIRAC/Core/Utilities/Graphs/Palette.py +++ b/src/DIRAC/Core/Utilities/Graphs/Palette.py @@ -31,8 +31,6 @@ JobMinorStatus.INPUT_NOT_AVAILABLE: "#2822A6", JobMinorStatus.INPUT_DATA_RESOLUTION: "#FFBE94", JobMinorStatus.DOWNLOADING_INPUT_SANDBOX: "#586CFF", - JobMinorStatus.INPUT_CONTAINS_SLASHES: "#AB7800", - JobMinorStatus.INPUT_INCORRECT: "#6812D6", JobMinorStatus.JOB_WRAPPER_INITIALIZATION: "#FFFFCC", JobMinorStatus.JOB_EXCEEDED_WALL_CLOCK: "#FF33CC", JobMinorStatus.JOB_INSUFFICIENT_DISK: "#33FFCC", @@ -64,7 +62,6 @@ class Palette: def __init__(self, palette={}, colors=[]): - self.palette = country_palette self.palette.update(job_status_palette) self.palette.update(miscelaneous_pallette) @@ -80,14 +77,12 @@ def addPalette(self, palette): self.palette.update(palette) def getColor(self, label): - if label in self.palette: return self.palette[label] else: return self.generateColor(label) def generateColor(self, label): - myMD5 = hashlib.md5() myMD5.update(label.encode()) hexstring = myMD5.hexdigest() diff --git a/src/DIRAC/Core/Utilities/Graphs/PieGraph.py b/src/DIRAC/Core/Utilities/Graphs/PieGraph.py index 420c733bbb7..56b0e5a61b2 100644 --- a/src/DIRAC/Core/Utilities/Graphs/PieGraph.py +++ b/src/DIRAC/Core/Utilities/Graphs/PieGraph.py @@ -14,12 +14,10 @@ class PieGraph(PlotBase): def __init__(self, data, ax, prefs, *args, **kw): - PlotBase.__init__(self, data, ax, prefs, *args, **kw) self.pdata = data def pie(self, explode=None, colors=None, autopct=None, pctdistance=0.6, shadow=False): - start = time.time() labels = self.pdata.getLabels() if labels[0][0] == "NoLabels": @@ -155,18 +153,16 @@ def pie(self, explode=None, colors=None, autopct=None, pctdistance=0.6, shadow=F min_amount = 0.1 def getLegendData(self): - return self.legendData def draw(self): - self.ylabel = "" self.prefs["square_axis"] = True PlotBase.draw(self) def my_display(x): if x > 100 * self.min_amount: - return "%.1f" % x + "%" + return f"{x:.1f}" + "%" else: return "" diff --git a/src/DIRAC/Core/Utilities/Graphs/PlotBase.py b/src/DIRAC/Core/Utilities/Graphs/PlotBase.py index 62744fb69c1..de7fb719bdc 100644 --- a/src/DIRAC/Core/Utilities/Graphs/PlotBase.py +++ b/src/DIRAC/Core/Utilities/Graphs/PlotBase.py @@ -3,6 +3,9 @@ The DIRAC Graphs package is derived from the GraphTool plotting package of the CMS/Phedex Project by ... """ +# matplotlib dynamicaly defines all the get_xticklabels and the like, +# so we just ignore it +# pylint: disable=not-callable from DIRAC.Core.Utilities.Graphs.Palette import Palette from DIRAC.Core.Utilities.Graphs.GraphData import GraphData @@ -13,7 +16,6 @@ class PlotBase: def __init__(self, data=None, axes=None, *aw, **kw): - self.ax_contain = axes self.canvas = None self.figure = None @@ -31,12 +33,10 @@ def __init__(self, data=None, axes=None, *aw, **kw): self.gdata = data def dumpPrefs(self): - for key in self.prefs: print(key.rjust(20), ":", str(self.prefs[key]).ljust(40)) def setAxes(self, axes): - self.ax_contain = axes self.ax_contain.set_axis_off() self.figure = self.ax_contain.get_figure() @@ -44,7 +44,6 @@ def setAxes(self, axes): self.dpi = self.ax_contain.figure.get_dpi() def draw(self): - prefs = self.prefs dpi = self.ax_contain.figure.get_dpi() diff --git a/src/DIRAC/Core/Utilities/Graphs/QualityMapGraph.py b/src/DIRAC/Core/Utilities/Graphs/QualityMapGraph.py index 00278c7d4db..db6c34d4971 100644 --- a/src/DIRAC/Core/Utilities/Graphs/QualityMapGraph.py +++ b/src/DIRAC/Core/Utilities/Graphs/QualityMapGraph.py @@ -3,6 +3,7 @@ The DIRAC Graphs package is derived from the GraphTool plotting package of the CMS/Phedex Project by ... """ + import datetime from pylab import setp from matplotlib.colors import Normalize @@ -39,7 +40,6 @@ class QualityMapGraph(PlotBase): - """ The BarGraph class is a straightforward bar graph; given a dictionary of values, it takes the keys as the independent variable and the values @@ -47,7 +47,6 @@ class QualityMapGraph(PlotBase): """ def __init__(self, data, ax, prefs, *args, **kw): - PlotBase.__init__(self, data, ax, prefs, *args, **kw) if isinstance(data, dict): self.gdata = GraphData(data) @@ -99,7 +98,6 @@ def get_alpha(*args, **kw): self.mapper = mapper def draw(self): - PlotBase.draw(self) self.x_formatter_cb(self.ax) @@ -115,8 +113,8 @@ def draw(self): start_plot = 0 end_plot = 0 if "starttime" in self.prefs and "endtime" in self.prefs: - start_plot = date2num(datetime.datetime.fromtimestamp(to_timestamp(self.prefs["starttime"]))) - end_plot = date2num(datetime.datetime.fromtimestamp(to_timestamp(self.prefs["endtime"]))) + start_plot = date2num(datetime.datetime.utcfromtimestamp(to_timestamp(self.prefs["starttime"]))) + end_plot = date2num(datetime.datetime.utcfromtimestamp(to_timestamp(self.prefs["endtime"]))) labels = self.gdata.getLabels() nKeys = self.gdata.getNumberOfKeys() @@ -134,7 +132,6 @@ def draw(self): for label, _num in labels: labelNames.append(label) for key, value, _error in self.gdata.getPlotNumData(label): - if xmin is None or xmin > (key + offset): xmin = key + offset if xmax is None or xmax < (key + offset): @@ -165,16 +162,17 @@ def draw(self): self.ax.set_xlim(xmin=start_plot, xmax=end_plot) else: self.ax.set_xlim(xmin=min(tmp_x), xmax=max(tmp_x)) - self.ax.set_yticks([i + 0.5 for i in range(nLabel)]) - self.ax.set_yticklabels(labelNames) - setp(self.ax.get_xticklines(), markersize=0.0) - setp(self.ax.get_yticklines(), markersize=0.0) - - cax, kw = make_axes(self.ax, orientation="vertical", fraction=0.07) - cb = ColorbarBase( + self.ax.set_yticks([i + 0.5 for i in range(nLabel)]) # pylint: disable=not-callable + self.ax.set_yticklabels(labelNames) # pylint: disable=not-callable + setp(self.ax.get_xticklines(), markersize=0.0) # pylint: disable=not-callable + setp(self.ax.get_yticklines(), markersize=0.0) # pylint: disable=not-callable + + cax, _kw = make_axes(self.ax, orientation="vertical", fraction=0.07) # pylint: disable=unpacking-non-sequence + # ColorbarBase is used to generate the colors within the legend at the right of the plot + ColorbarBase( cax, cmap=self.cmap, norm=self.norms, boundaries=self.cbBoundaries, values=self.cbValues, ticks=self.cbTicks ) - cb.draw_all() + self.figure.draw_without_rendering() # cb = self.ax.colorbar( self.mapper, format="%d%%", # orientation='horizontal', fraction=0.04, pad=0.1, aspect=40 ) # setp( cb.outline, linewidth=.5 ) @@ -183,13 +181,11 @@ def draw(self): # setp( cb.ax.get_xticklabels(), fontname = self.prefs['font'] ) def getQualityColor(self, value): - if value is None or value < 0.0: return "#FFFFFF" return self.mapper.to_rgba(value) def getLegendData(self): - return None def x_formatter_cb(self, ax): @@ -209,7 +205,6 @@ def x_formatter_cb(self, ax): xmin = 0.0 ax.set_xlim(xmin=xmin, xmax=len(ticks)) elif self.gdata.key_type == "time": - # ax.set_xlim( xmin=self.begin_num,xmax=self.end_num ) dl = PrettyDateLocator() df = PrettyDateFormatter(dl) diff --git a/src/DIRAC/Core/Utilities/Graphs/__init__.py b/src/DIRAC/Core/Utilities/Graphs/__init__.py index d8bf696fabe..bef0dd671dd 100644 --- a/src/DIRAC/Core/Utilities/Graphs/__init__.py +++ b/src/DIRAC/Core/Utilities/Graphs/__init__.py @@ -113,10 +113,9 @@ def graph(data, fileName, *args, **kw): - prefs = evalPrefs(*args, **kw) graph_size = prefs.get("graph_size", "normal") - + defaults = None if graph_size == "normal": defaults = graph_normal_prefs elif graph_size == "small": diff --git a/src/DIRAC/Core/Utilities/Grid.py b/src/DIRAC/Core/Utilities/Grid.py index fbd6e9066b2..1f9a4d99f79 100644 --- a/src/DIRAC/Core/Utilities/Grid.py +++ b/src/DIRAC/Core/Utilities/Grid.py @@ -1,60 +1,9 @@ """ The Grid module contains several utilities for grid operations """ -import os -from DIRAC.Core.Utilities.Os import sourceEnv -from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager -from DIRAC.Core.Security.ProxyInfo import getProxyInfo -from DIRAC.ConfigurationSystem.Client.Helpers import Local -from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR -from DIRAC.Core.Utilities.Subprocess import systemCall, shellCall - - -def executeGridCommand(proxy, cmd, gridEnvScript=None): - """ - Execute cmd tuple after sourcing GridEnv - """ - currentEnv = dict(os.environ) - - if not gridEnvScript: - # if not passed as argument, use default from CS Helpers - gridEnvScript = Local.gridEnv() - - if gridEnvScript: - command = gridEnvScript.split() - ret = sourceEnv(10, command) - if not ret["OK"]: - return S_ERROR("Failed sourcing GridEnv: %s" % ret["Message"]) - gridEnv = ret["outputEnv"] - # - # Preserve some current settings if they are there - # - if "X509_VOMS_DIR" in currentEnv: - gridEnv["X509_VOMS_DIR"] = currentEnv["X509_VOMS_DIR"] - if "X509_CERT_DIR" in currentEnv: - gridEnv["X509_CERT_DIR"] = currentEnv["X509_CERT_DIR"] - else: - gridEnv = currentEnv - - if not proxy: - res = getProxyInfo() - if not res["OK"]: - return res - gridEnv["X509_USER_PROXY"] = res["Value"]["path"] - elif isinstance(proxy, str): - if os.path.exists(proxy): - gridEnv["X509_USER_PROXY"] = proxy - else: - return S_ERROR("Can not treat proxy passed as a string") - else: - ret = gProxyManager.dumpProxyToFile(proxy) - if not ret["OK"]: - return ret - gridEnv["X509_USER_PROXY"] = ret["Value"] - - result = systemCall(120, cmd, env=gridEnv) - return result +from DIRAC.Core.Utilities.ReturnValues import S_ERROR, S_OK +from DIRAC.Core.Utilities.Subprocess import shellCall def ldapsearchBDII(filt=None, attr=None, host=None, base=None, selectionString="Glue"): diff --git a/src/DIRAC/Core/Utilities/JDL.py b/src/DIRAC/Core/Utilities/JDL.py index d68bcbaf67a..935094f1ed9 100644 --- a/src/DIRAC/Core/Utilities/JDL.py +++ b/src/DIRAC/Core/Utilities/JDL.py @@ -1,6 +1,46 @@ +"""Transformation classes around the JDL format.""" + from diraccfg import CFG +from pydantic import ValidationError + from DIRAC import S_OK, S_ERROR from DIRAC.Core.Utilities import List +from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd +from DIRAC.WorkloadManagementSystem.Utilities.JobModel import BaseJobDescriptionModel + +ARGUMENTS = "Arguments" +BANNED_SITES = "BannedSites" +CPU_TIME = "CPUTime" +EXECUTABLE = "Executable" +EXECUTION_ENVIRONMENT = "ExecutionEnvironment" +GRID_CE = "GridCE" +INPUT_DATA = "InputData" +INPUT_DATA_POLICY = "InputDataPolicy" +INPUT_SANDBOX = "InputSandbox" +JOB_CONFIG_ARGS = "JobConfigArgs" +JOB_TYPE = "JobType" +JOB_GROUP = "JobGroup" +LOG_LEVEL = "LogLevel" +NUMBER_OF_PROCESSORS = "NumberOfProcessors" +MAX_NUMBER_OF_PROCESSORS = "MaxNumberOfProcessors" +MIN_NUMBER_OF_PROCESSORS = "MinNumberOfProcessors" +OUTPUT_DATA = "OutputData" +OUTPUT_PATH = "OutputPath" +OUTPUT_SE = "OutputSE" +PLATFORM = "Platform" +PRIORITY = "Priority" +STD_ERROR = "StdError" +STD_OUTPUT = "StdOutput" +OUTPUT_SANDBOX = "OutputSandbox" +JOB_NAME = "JobName" +SITE = "Site" +TAGS = "Tags" + +OWNER = "Owner" +OWNER_GROUP = "OwnerGroup" +VO = "VirtualOrganization" + +CREDENTIALS_FIELDS = {OWNER, OWNER_GROUP, VO} def loadJDLAsCFG(jdl): @@ -42,7 +82,7 @@ def assignValue(key, value, cfg): return S_ERROR("Invalid key name") value = value.strip() if not value: - return S_ERROR("No value for key %s" % key) + return S_ERROR(f"No value for key {key}") if value[0] == "{": if value[-1] != "}": return S_ERROR("Value '%s' seems a list but does not end in '}'" % (value)) @@ -50,7 +90,7 @@ def assignValue(key, value, cfg): for i in range(len(valList)): result = cleanValue(valList[i]) if not result["OK"]: - return S_ERROR("Var {} : {}".format(key, result["Message"])) + return S_ERROR(f"Var {key} : {result['Message']}") valList[i] = result["Value"] if valList[i] is None: return S_ERROR(f"List value '{value}' seems invalid for item {i}") @@ -58,10 +98,10 @@ def assignValue(key, value, cfg): else: result = cleanValue(value) if not result["OK"]: - return S_ERROR("Var {} : {}".format(key, result["Message"])) + return S_ERROR(f"Var {key} : {result['Message']}") nV = result["Value"] if nV is None: - return S_ERROR("Value '%s seems invalid" % (value)) + return S_ERROR(f"Value '{value} seems invalid") value = nV cfg.setOption(key, value) return S_OK() @@ -90,7 +130,7 @@ def assignValue(key, value, cfg): if not key: return S_ERROR("Invalid key in JDL") if value.strip(): - return S_ERROR("Key %s seems to have a value and open a sub JDL at the same time" % key) + return S_ERROR(f"Key {key} seems to have a value and open a sub JDL at the same time") result = loadJDLAsCFG(jdl[iPos:]) if not result["OK"]: return result @@ -128,17 +168,17 @@ def assignValue(key, value, cfg): def dumpCFGAsJDL(cfg, level=1, tab=" "): indent = tab * level - contents = ["%s[" % (tab * (level - 1))] + contents = [f"{tab * (level - 1)}["] sections = cfg.listSections() for key in cfg: if key in sections: contents.append(f"{indent}{key} =") - contents.append("%s;" % dumpCFGAsJDL(cfg[key], level + 1, tab)) + contents.append(f"{dumpCFGAsJDL(cfg[key], level + 1, tab)};") else: val = List.fromChar(cfg[key]) # Some attributes are never lists - if len(val) < 2 or key in ["Arguments", "Executable", "StdOutput", "StdError"]: + if len(val) < 2 or key in [ARGUMENTS, EXECUTABLE, STD_OUTPUT, STD_ERROR]: value = cfg[key] try: try_value = float(value) @@ -152,8 +192,168 @@ def dumpCFGAsJDL(cfg, level=1, tab=" "): try: value = float(val[iPos]) except Exception: - val[iPos] = '"%s"' % val[iPos] + val[iPos] = f'"{val[iPos]}"' contents.append(",\n".join([f"{tab * (level + 1)}{value}" for value in val])) contents.append("%s};" % indent) - contents.append("%s]" % (tab * (level - 1))) + contents.append(f"{tab * (level - 1)}]") return "\n".join(contents) + + +def jdlToBaseJobDescriptionModel(classAd: ClassAd): + """ + Converts a JDL string into a JSON string for data validation from the BaseJob model + This method allows compatibility with older Client versions that used the _toJDL method + """ + try: + jobDescription = BaseJobDescriptionModel( + executable=classAd.getAttributeString(EXECUTABLE), + ) + if classAd.lookupAttribute(ARGUMENTS): + jobDescription.arguments = classAd.getAttributeString(ARGUMENTS) + classAd.deleteAttribute(ARGUMENTS) + + if classAd.lookupAttribute(BANNED_SITES): + jobDescription.bannedSites = classAd.getListFromExpression(BANNED_SITES) + classAd.deleteAttribute(BANNED_SITES) + + if classAd.lookupAttribute(CPU_TIME): + jobDescription.cpuTime = classAd.getAttributeInt(CPU_TIME) + classAd.deleteAttribute(CPU_TIME) + + if classAd.lookupAttribute(EXECUTABLE): + jobDescription.executable = classAd.getAttributeString(EXECUTABLE) + classAd.deleteAttribute(EXECUTABLE) + + if classAd.lookupAttribute(EXECUTION_ENVIRONMENT): + executionEnvironment = classAd.getListFromExpression(EXECUTION_ENVIRONMENT) + if executionEnvironment: + jobDescription.executionEnvironment = {} + for element in executionEnvironment: + key, value = element.split("=") + if value.isdigit(): + value = int(value) + else: + try: + value = float(value) + except ValueError: + pass + jobDescription.executionEnvironment[key] = value + classAd.deleteAttribute(EXECUTION_ENVIRONMENT) + + if classAd.lookupAttribute(GRID_CE): + jobDescription.gridCE = classAd.getAttributeString(GRID_CE) + classAd.deleteAttribute(GRID_CE) + + if classAd.lookupAttribute(INPUT_DATA): + jobDescription.inputData = classAd.getListFromExpression(INPUT_DATA) + classAd.deleteAttribute(INPUT_DATA) + + if classAd.lookupAttribute(INPUT_DATA_POLICY): + jobDescription.inputDataPolicy = classAd.getAttributeString(INPUT_DATA_POLICY) + classAd.deleteAttribute(INPUT_DATA_POLICY) + + if classAd.lookupAttribute(INPUT_SANDBOX): + jobDescription.inputSandbox = classAd.getListFromExpression(INPUT_SANDBOX) + classAd.deleteAttribute(INPUT_SANDBOX) + + if classAd.lookupAttribute(JOB_CONFIG_ARGS): + jobDescription.jobConfigArgs = classAd.getAttributeString(JOB_CONFIG_ARGS) + classAd.deleteAttribute(JOB_CONFIG_ARGS) + + if classAd.lookupAttribute(JOB_GROUP): + jobDescription.jobGroup = classAd.getAttributeString(JOB_GROUP) + classAd.deleteAttribute(JOB_GROUP) + + if classAd.lookupAttribute(JOB_NAME): + jobDescription.jobName = classAd.getAttributeString(JOB_NAME) + classAd.deleteAttribute(JOB_NAME) + + if classAd.lookupAttribute(JOB_TYPE): + jobDescription.jobType = classAd.getAttributeString(JOB_TYPE) + classAd.deleteAttribute(JOB_TYPE) + + if classAd.lookupAttribute(LOG_LEVEL): + jobDescription.logLevel = classAd.getAttributeString(LOG_LEVEL) + classAd.deleteAttribute(LOG_LEVEL) + + if classAd.lookupAttribute(NUMBER_OF_PROCESSORS): + jobDescription.maxNumberOfProcessors = classAd.getAttributeInt(NUMBER_OF_PROCESSORS) + jobDescription.minNumberOfProcessors = classAd.getAttributeInt(NUMBER_OF_PROCESSORS) + classAd.deleteAttribute(NUMBER_OF_PROCESSORS) + classAd.deleteAttribute(MAX_NUMBER_OF_PROCESSORS) + classAd.deleteAttribute(MIN_NUMBER_OF_PROCESSORS) + else: + if classAd.lookupAttribute(MAX_NUMBER_OF_PROCESSORS): + jobDescription.maxNumberOfProcessors = classAd.getAttributeInt(MAX_NUMBER_OF_PROCESSORS) + classAd.deleteAttribute(MAX_NUMBER_OF_PROCESSORS) + if classAd.lookupAttribute(MIN_NUMBER_OF_PROCESSORS): + jobDescription.minNumberOfProcessors = classAd.getAttributeInt(MIN_NUMBER_OF_PROCESSORS) + classAd.deleteAttribute(MIN_NUMBER_OF_PROCESSORS) + + if classAd.lookupAttribute(OUTPUT_DATA): + jobDescription.outputData = set(classAd.getListFromExpression(OUTPUT_DATA)) + classAd.deleteAttribute(OUTPUT_DATA) + + if classAd.lookupAttribute(OUTPUT_SANDBOX): + jobDescription.outputSandbox = set(classAd.getListFromExpression(OUTPUT_SANDBOX)) + classAd.deleteAttribute(OUTPUT_SANDBOX) + + if classAd.lookupAttribute(OUTPUT_PATH): + jobDescription.outputPath = classAd.getAttributeString(OUTPUT_PATH) + classAd.deleteAttribute(OUTPUT_PATH) + + if classAd.lookupAttribute(OUTPUT_SE): + jobDescription.outputSE = classAd.getAttributeString(OUTPUT_SE) + classAd.deleteAttribute(OUTPUT_SE) + + if classAd.lookupAttribute(SITE): + jobDescription.sites = classAd.getListFromExpression(SITE) + classAd.deleteAttribute(SITE) + + if classAd.lookupAttribute(PLATFORM): + jobDescription.platform = classAd.getAttributeString(PLATFORM) + classAd.deleteAttribute(PLATFORM) + + if classAd.lookupAttribute(PRIORITY): + jobDescription.priority = classAd.getAttributeInt(PRIORITY) + classAd.deleteAttribute(PRIORITY) + + if classAd.lookupAttribute(STD_OUTPUT): + jobDescription.stdout = classAd.getAttributeString(STD_OUTPUT) + classAd.deleteAttribute(STD_OUTPUT) + + if classAd.lookupAttribute(STD_ERROR): + jobDescription.stderr = classAd.getAttributeString(STD_ERROR) + classAd.deleteAttribute(STD_ERROR) + + if classAd.lookupAttribute(TAGS): + jobDescription.tags = classAd.getListFromExpression(TAGS) + classAd.deleteAttribute(TAGS) + + # Remove credentials + for attribute in CREDENTIALS_FIELDS: + classAd.deleteAttribute(attribute) + + # Remove legacy attributes + for attribute in {"DIRACSetup", "OwnerDN"}: + classAd.deleteAttribute(attribute) + + for attribute in classAd.getAttributes(): + if not jobDescription.extraFields: + jobDescription.extraFields = {} + + value = classAd.getAttributeString(attribute) + if value.isdigit(): + value = int(value) + else: + try: + value = float(value) + except ValueError: + pass + + jobDescription.extraFields[attribute] = value + + except ValidationError as e: + return S_ERROR(f"Invalid JDL: {e}") + + return S_OK(jobDescription) diff --git a/src/DIRAC/Core/Utilities/List.py b/src/DIRAC/Core/Utilities/List.py index 276b09dbb81..63960eb8de1 100755 --- a/src/DIRAC/Core/Utilities/List.py +++ b/src/DIRAC/Core/Utilities/List.py @@ -63,7 +63,7 @@ def stringListToString(aList: list) -> str: :param aList: list to be serialized to string for making queries """ - return ",".join("'%s'" % x for x in aList) + return ",".join(f"'{x}'" for x in aList) def intListToString(aList: list) -> str: diff --git a/src/DIRAC/Core/Utilities/LockRing.py b/src/DIRAC/Core/Utilities/LockRing.py index 17688c24343..d60825352e8 100644 --- a/src/DIRAC/Core/Utilities/LockRing.py +++ b/src/DIRAC/Core/Utilities/LockRing.py @@ -47,14 +47,14 @@ def acquire(self, lockName): try: self.__locks[lockName].acquire() except ValueError: - return S_ERROR("No lock named %s" % lockName) + return S_ERROR(f"No lock named {lockName}") return S_OK() def release(self, lockName): try: self.__locks[lockName].release() except ValueError: - return S_ERROR("No lock named %s" % lockName) + return S_ERROR(f"No lock named {lockName}") return S_OK() def _openAll(self): diff --git a/src/DIRAC/Core/Utilities/MJF.py b/src/DIRAC/Core/Utilities/MJF.py deleted file mode 100644 index fdd2bc64a2c..00000000000 --- a/src/DIRAC/Core/Utilities/MJF.py +++ /dev/null @@ -1,198 +0,0 @@ -""" The MJF utility calculates the amount of wall clock time - left for a given batch system slot or VM. This is essential for the - 'Filling Mode' where several jobs may be executed in the same slot. - - Machine Job/Features are used following HSF-TN-2016-02 if available. - Otherwise values are filled in using the batch system and CS - information. -""" -import os -import ssl -import time -from urllib.request import urlopen - -import DIRAC - -from DIRAC import gLogger, gConfig - - -class MJF: - """Machine/Job Features methods""" - - mjfKeys = { - "MACHINEFEATURES": ["total_cpu", "hs06", "shutdowntime", "grace_secs"], - "JOBFEATURES": [ - "allocated_cpu", - "hs06_job", - "shutdowntime_job", - "grace_secs_job", - "jobstart_secs", - "job_id", - "wall_limit_secs", - "cpu_limit_secs", - "max_rss_bytes", - "max_swap_bytes", - "scratch_limit_bytes", - ], - } - - ############################################################################# - def __init__(self): - """Standard constructor""" - self.log = gLogger.getSubLogger(self.__class__.__name__) - - capath = DIRAC.Core.Security.Locations.getCAsLocation() - - if not capath: - raise Exception("Unable to find CA files location! Not in /etc/grid-security/certificates/ etc.") - - # Used by urllib when talking to HTTPS web servers - self.context = ssl.create_default_context(capath=capath) - - def updateConfig(self, pilotStartTime=None): - """Populate /LocalSite/MACHINEFEATURES and /LocalSite/JOBFEATURES with MJF values - This is run early in the job to update the configuration file that subsequent DIRAC - scripts read when they start. - """ - - if pilotStartTime: - gConfig.setOptionValue("/LocalSite/JOBFEATURES/jobstart_secs", str(pilotStartTime)) - - for mORj in ["MACHINEFEATURES", "JOBFEATURES"]: - for key in self.mjfKeys[mORj]: - value = self.__fetchMachineJobFeature(mORj, key) - - if value is not None: - gConfig.setOptionValue(f"/LocalSite/{mORj}/{key}", value) - - def getMachineFeature(self, key): - """Returns MACHINEFEATURES/key value saved in /LocalSite configuration by - updateConfigFile() unless MACHINEFEATURES/shutdowntime when we try to fetch - from the source URL itself again in case it changes. - """ - if key == "shutdowntime": - value = self.__fetchMachineJobFeature("MACHINEFEATURES", "shutdowntime") - # If unable to fetch shutdowntime, go back to any value in /LocalSite - # in case HTTP(S) server is down - if value is not None: - return value - - return gConfig.getValue("/LocalSite/MACHINEFEATURES/" + key, None) - - def getIntMachineFeature(self, key): - """Returns MACHINEFEATURES/key as an int or None if not an int or not present""" - value = self.getMachineFeature(key) - - try: - return int(value) - except ValueError: - return None - - def getJobFeature(self, key): - """Returns JOBFEATURES/key value saved in /LocalSite configuration by - updateConfigFile() unless JOBFEATURES/shutdowntime_job when we try to fetch - from the source URL itself again in case it changes. - """ - if key == "shutdowntime_job": - value = self.__fetchMachineJobFeature("JOBFEATURES", "shutdowntime_job") - # If unable to fetch shutdowntime_job, go back to any value in /LocalSite - # in case HTTP(S) server is down - if value is not None: - return value - - return gConfig.getValue("/LocalSite/JOBFEATURES/" + key, None) - - def getIntJobFeature(self, key): - """Returns JOBFEATURES/key as an int or None if not an int or not present""" - value = self.getJobFeature(key) - - try: - return int(value) - except ValueError: - return None - - def __fetchMachineJobFeature(self, mORj, key): - """Returns raw MJF value for a given key, perhaps by HTTP(S), perhaps from a local file - mORj must be MACHINEFEATURES or JOBFEATURES - If the value cannot be found, then return None. There are many legitimate ways for - a site not to provide some MJF values so we don't log errors, failures etc. - """ - if mORj != "MACHINEFEATURES" and mORj != "JOBFEATURES": - raise Exception("Must request MACHINEFEATURES or JOBFEATURES") - - if mORj not in os.environ: - return None - - url = os.environ[mORj] + "/" + key - - # Simple if a file - if url[0] == "/": - try: - with open(url) as fd: - return fd.read().strip() - except Exception: - return None - - # Otherwise make sure it's an HTTP(S) URL - if not url.startswith("http://") and not url.startswith("https://"): - return None - - # We could have used urlopen() for local files too, but we also - # need to check HTTP return code in case we get an HTML error page - # instead of a true key value. - try: - mjfUrl = urlopen(url=url, context=self.context) - # HTTP return codes other than 2xx mean failure - if int(mjfUrl.getcode() / 100) != 2: - return None - return mjfUrl.read().strip() - except Exception: - return None - finally: - try: - mjfUrl.close() - except UnboundLocalError: - pass - - def getWallClockSecondsLeft(self): - """Returns the number of seconds until either the wall clock limit - or the shutdowntime(_job) is reached. - """ - - now = int(time.time()) - secondsLeft = None - jobstartSecs = self.getIntJobFeature("jobstart_secs") - wallLimitSecs = self.getIntJobFeature("wall_limit_secs") - shutdowntimeJob = self.getIntJobFeature("shutdowntime_job") - shutdowntime = self.getIntMachineFeature("shutdowntime") - - # look for local shutdown file - try: - with open("/var/run/shutdown_time") as fd: - shutdowntimeLocal = int(fd.read().strip()) - except (OSError, ValueError): - shutdowntimeLocal = None - - if jobstartSecs is not None and wallLimitSecs is not None: - secondsLeft = jobstartSecs + wallLimitSecs - now - - if shutdowntimeJob is not None: - if secondsLeft is None: - secondsLeft = shutdowntimeJob - now - elif shutdowntimeJob - now < secondsLeft: - secondsLeft = shutdowntimeJob - now - - if shutdowntime is not None: - if secondsLeft is None: - secondsLeft = shutdowntime - now - elif shutdowntime - now < secondsLeft: - secondsLeft = shutdowntime - now - - if shutdowntimeLocal is not None: - if secondsLeft is None: - secondsLeft = shutdowntimeLocal - now - elif shutdowntimeLocal - now < secondsLeft: - secondsLeft = shutdowntimeLocal - now - - # Wall Clock time left or None if unknown - return secondsLeft diff --git a/src/DIRAC/Core/Utilities/Mail.py b/src/DIRAC/Core/Utilities/Mail.py index a796130f2ff..547aeceb608 100644 --- a/src/DIRAC/Core/Utilities/Mail.py +++ b/src/DIRAC/Core/Utilities/Mail.py @@ -65,10 +65,10 @@ def _create(self, addresses): with open(attachment, "rb") as fil: part = MIMEApplication(fil.read(), Name=os.path.basename(attachment)) - part["Content-Disposition"] = 'attachment; filename="%s"' % os.path.basename(attachment) + part["Content-Disposition"] = f'attachment; filename="{os.path.basename(attachment)}"' msg.attach(part) except OSError as e: - gLogger.exception("Could not attach %s" % attachment, lException=e) + gLogger.exception(f"Could not attach {attachment}", lException=e) return S_OK(msg) @@ -110,7 +110,7 @@ def _send(self, msg=None): smtp.ehlo_or_helo_if_needed() smtp.sendmail(self._fromAddress, addresses, msg.as_string()) except Exception as x: - return S_ERROR("Sending mail failed %s" % str(x)) + return S_ERROR(f"Sending mail failed {str(x)}") smtp.quit() return S_OK("The mail was successfully sent") diff --git a/src/DIRAC/Core/Utilities/MixedEncode.py b/src/DIRAC/Core/Utilities/MixedEncode.py index b7eae66ab66..74159d788f3 100644 --- a/src/DIRAC/Core/Utilities/MixedEncode.py +++ b/src/DIRAC/Core/Utilities/MixedEncode.py @@ -12,7 +12,7 @@ def encode(inData): :return: an encoded string """ - if os.getenv("DIRAC_USE_JSON_ENCODE", "NO").lower() in ("yes", "true"): + if os.getenv("DIRAC_USE_JSON_ENCODE", "Yes").lower() in ("yes", "true"): return JEncode.encode(inData) return DEncode.encode(inData) @@ -25,15 +25,13 @@ def decode(encodedData): :return: the decoded objects, encoded object length """ - if os.getenv("DIRAC_USE_JSON_DECODE", "Yes").lower() in ("yes", "true"): - try: - # 'null' is a special case. - # None is encoded as 'null' as JSON - # that DEncode would understand as None, since it starts with 'n' - # but the length of the decoded string will not be correct - if encodedData == "null": - raise Exception - return DEncode.decode(encodedData) - except Exception: - return JEncode.decode(encodedData) - return DEncode.decode(encodedData) + try: + # 'null' is a special case. + # None is encoded as 'null' as JSON + # that DEncode would understand as None, since it starts with 'n' + # but the length of the decoded string will not be correct + if encodedData == "null": + raise Exception + return DEncode.decode(encodedData) + except Exception: + return JEncode.decode(encodedData) diff --git a/src/DIRAC/Core/Utilities/ModuleFactory.py b/src/DIRAC/Core/Utilities/ModuleFactory.py deleted file mode 100755 index 623708a5def..00000000000 --- a/src/DIRAC/Core/Utilities/ModuleFactory.py +++ /dev/null @@ -1,47 +0,0 @@ -######################################################################## -# File : ModuleFactory.py -# Author : Stuart Paterson -######################################################################## -""" The Module Factory instantiates a given Module based on a given input - string and set of arguments to be passed. This allows for VO specific - module utilities to be used in various contexts. -""" -from DIRAC import S_OK, S_ERROR, gLogger - - -class ModuleFactory: - - ############################################################################# - def __init__(self): - """Standard constructor""" - self.log = gLogger - - ############################################################################# - def getModule(self, importString, argumentsDict): - """This method returns the Module instance given the import string and - arguments dictionary. - """ - try: - moduleName = importString.split(".")[-1] - modulePath = importString.replace(".%s" % (moduleName), "") - importModule = __import__(f"{modulePath}.{moduleName}", globals(), locals(), [moduleName]) - except Exception as x: - msg = f"ModuleFactory could not import {modulePath}.{moduleName}" - self.log.warn(x) - self.log.warn(msg) - return S_ERROR(msg) - - try: - # FIXME: should we use imp module? - moduleStr = "importModule.%s(argumentsDict)" % (moduleName) - moduleInstance = eval(moduleStr) - except Exception as x: - msg = "ModuleFactory could not instantiate %s()" % (moduleName) - self.log.warn(x) - self.log.warn(msg) - return S_ERROR(msg) - - return S_OK(moduleInstance) - - -# EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF# diff --git a/src/DIRAC/Core/Utilities/MySQL.py b/src/DIRAC/Core/Utilities/MySQL.py index 310ab68dad7..25f282ad985 100755 --- a/src/DIRAC/Core/Utilities/MySQL.py +++ b/src/DIRAC/Core/Utilities/MySQL.py @@ -29,7 +29,7 @@ Returns S_OK or S_ERROR. - _query( cmd, [conn] ) + _query( cmd, [conn=conn] ) Executes SQL command "cmd". Gets a connection from the Queue (or open a new one if none is available), @@ -39,7 +39,7 @@ Returns S_OK with fetchall() out in Value or S_ERROR upon failure. - _update( cmd, [conn] ) + _update( cmd, [conn=conn] ) Executes SQL command "cmd" and issue a commit Gets a connection from the Queue (or open a new one if none is available), @@ -180,7 +180,7 @@ def _checkFields(inFields, inValues): return S_OK() -def _quotedList(fieldList=None): +def _quotedList(fieldList=None, allowDate=False): """ Quote a list of MySQL Field Names with "`" Return a comma separated list of quoted Field Names @@ -192,7 +192,11 @@ def _quotedList(fieldList=None): quotedFields = [] try: for field in fieldList: - quotedFields.append("`%s`" % field.replace("`", "")) + if allowDate and field.startswith("date(") and field.endswith(")"): + field = field[len("date(") : -len(")")] + quotedFields.append(f"date(`{field.replace('`', '')}`)") + else: + quotedFields.append(f"`{field.replace('`', '')}`") except Exception: return None if not quotedFields: @@ -222,7 +226,7 @@ def captureOptimizerTraces(meth): cd ${DIRAC_MYSQL_OPTIMIZER_TRACES_PATH} c=0; for i in $(ls); do newFn=$(echo $i | sed -E "s/_trace_[0-9]+.[0-9]+_(.*)/_trace_${c}_\1/g"); mv $i $newFn; c=$(( c + 1 )); done - This tool is useful then to compare the files https://github.com/cosmicanant/recursive-diff + This tool is useful then to compare the files https://github.com/crusaderky/recursive_diff Note that this method is far from pretty: @@ -249,10 +253,10 @@ def captureOptimizerTraces(meth): @functools.wraps(meth) def innerMethod(self, *args, **kwargs): - # First, get a connection to the DB, and enable the tracing - connection = self.__connectionPool.get(self.__dbName) + connection = self._MySQL__connectionPool.get(self._MySQL__dbName)["Value"] connection.cursor().execute('SET optimizer_trace="enabled=on";') + # We also set some options that worked for my use case. # you may need to tune these parameters if you have huge traces # or more recursive calls. @@ -284,7 +288,7 @@ def innerMethod(self, *args, **kwargs): # optimizer_trace__.json jsonFn = os.path.join(optimizerTracingFolder, f"optimizer_trace_{time.time()}_{methHash}.json") - with open(jsonFn, "wt") as f: + with open(jsonFn, "w") as f: # Some calls do not generate a trace, like "show tables" if not queryTraces: json.dump({"EmptyTrace": args}, f) @@ -312,7 +316,7 @@ def innerMethod(self, *args, **kwargs): {"Query": trace_query, "Trace": json.loads(trace_analysis) if trace_analysis else None} ) - json.dump(jsonTraces, f) + json.dump(jsonTraces, f, indent=True) return res @@ -327,12 +331,11 @@ class ConnectionPool: Management of connections per thread """ - def __init__(self, host, user, passwd, port=3306, graceTime=600): + def __init__(self, host, user, passwd, port=3306): self.__host = host self.__user = user self.__passwd = passwd self.__port = port - self.__graceTime = graceTime self.__spares = collections.deque() self.__maxSpares = 10 self.__lastClean = 0 @@ -369,7 +372,7 @@ def __getWithRetry(self, dbName, totalRetries, retriesLeft): except MySQLdb.MySQLError as excp: if retriesLeft > 0: return self.__getWithRetry(dbName, totalRetries, retriesLeft - 1) - return S_ERROR(DErrno.EMYSQL, "Could not connect: %s" % excp) + return S_ERROR(DErrno.EMYSQL, f"Could not connect: {excp}") if not self.__ping(conn): try: @@ -429,9 +432,9 @@ def __pop(self, thid): try: data[0].close() except MySQLdb.ProgrammingError as exc: - gLogger.warn("ProgrammingError exception while closing MySQL connection: %s" % exc) + gLogger.warn(f"ProgrammingError exception while closing MySQL connection: {exc}") except Exception as exc: - gLogger.warn("Exception while closing MySQL connection: %s" % exc) + gLogger.warn(f"Exception while closing MySQL connection: {exc}") except KeyError: pass @@ -442,12 +445,6 @@ def clean(self, now=False): for thid in list(self.__assigned): if not thid.is_alive(): self.__pop(thid) - try: - data = self.__assigned[thid] - except KeyError: - continue - if now - data[2] > self.__graceTime: - self.__pop(thid) def transactionStart(self, dbName): result = self.get(dbName) @@ -457,7 +454,7 @@ def transactionStart(self, dbName): try: return S_OK(self.__execute(conn, "START TRANSACTION WITH CONSISTENT SNAPSHOT")) except MySQLdb.MySQLError as excp: - return S_ERROR(DErrno.EMYSQL, "Could not begin transaction: %s" % excp) + return S_ERROR(DErrno.EMYSQL, f"Could not begin transaction: {excp}") def transactionCommit(self, dbName): result = self.get(dbName) @@ -468,7 +465,7 @@ def transactionCommit(self, dbName): result = self.__execute(conn, "COMMIT") return S_OK(result) except MySQLdb.MySQLError as excp: - return S_ERROR(DErrno.EMYSQL, "Could not commit transaction: %s" % excp) + return S_ERROR(DErrno.EMYSQL, f"Could not commit transaction: {excp}") def transactionRollback(self, dbName): result = self.get(dbName) @@ -479,7 +476,7 @@ def transactionRollback(self, dbName): result = self.__execute(conn, "ROLLBACK") return S_OK(result) except MySQLdb.MySQLError as excp: - return S_ERROR(DErrno.EMYSQL, "Could not rollback transaction: %s" % excp) + return S_ERROR(DErrno.EMYSQL, f"Could not rollback transaction: {excp}") class MySQL: @@ -523,7 +520,7 @@ def __init__(self, hostName="localhost", userName="dirac", passwd="dirac", dbNam self.__initialized = True result = self._connect() if not result["OK"]: - gLogger.error("Cannot connect to the DB", " %s" % result["Message"]) + gLogger.error("Cannot connect to the DB", f" {result['Message']}") def __del__(self): global gInstancesCount @@ -532,25 +529,23 @@ def __del__(self): except Exception: pass - def _except(self, methodName, x, err, cmd="", print=True): + def _except(self, methodName, x, err, cmd="", debug=True): """ print MySQL error or exception return S_ERROR with Exception """ - try: raise x except MySQLdb.Error as e: - if print: + if debug: self.log.error(f"{methodName} ({self._safeCmd(cmd)}): {err}", "%d: %s" % (e.args[0], e.args[1])) return S_ERROR(DErrno.EMYSQL, "%s: ( %d: %s )" % (err, e.args[0], e.args[1])) except Exception as e: - if print: + if debug: self.log.error(f"{methodName} ({self._safeCmd(cmd)}): {err}", repr(e)) return S_ERROR(DErrno.EMYSQL, f"{err}: ({repr(e)})") def __isDateTime(self, dateString): - if dateString == "UTC_TIMESTAMP()": return True try: @@ -589,8 +584,8 @@ def __escapeString(self, myString, connection=None): return S_OK(myString) for func in ["TIMESTAMPDIFF", "TIMESTAMPADD"]: - if myString.strip().startswith("%s(" % func) and myString.strip().endswith(")"): - args = myString.strip()[:-1].replace("%s(" % func, "").strip().split(",") + if myString.strip().startswith(f"{func}(") and myString.strip().endswith(")"): + args = myString.strip()[:-1].replace(f"{func}(", "").strip().split(",") arg1, arg2, arg3 = (x.strip() for x in args) if arg1 in timeUnits: if self.__isDateTime(arg2) or arg2.isalnum(): @@ -601,7 +596,7 @@ def __escapeString(self, myString, connection=None): escape_string = connection.escape_string(myString.encode()).decode() # self.log.debug('__escape_string: returns', '"%s"' % escape_string) - return S_OK('"%s"' % escape_string) + return S_OK(f'"{escape_string}"') except Exception as x: return self._except("__escape_string", x, "Could not escape string", myString) @@ -626,7 +621,7 @@ def __checkTable(self, tableName, force=False): # the requested exist and table creation is not force, return with error return S_ERROR(DErrno.EMYSQL, "The requested table already exist") else: - cmd = "DROP TABLE %s" % table + cmd = f"DROP TABLE {table}" retDict = self._update(cmd) if not retDict["OK"]: return retDict @@ -719,7 +714,7 @@ def _query(self, cmd, *, conn=None, debug=True): return S_ERROR upon error """ - self.log.debug("_query: %s" % self._safeCmd(cmd)) + self.log.debug(f"_query: {self._safeCmd(cmd)}") if conn: connection = conn @@ -756,16 +751,18 @@ def _query(self, cmd, *, conn=None, debug=True): return retDict @captureOptimizerTraces - def _update(self, cmd, *, conn=None, debug=True): + def _update(self, cmd, *, args=None, conn=None, debug=True): """execute MySQL update command + :param args: parameters passed to cursor.execute(..., args=args) method. + :param conn: connection object. :param debug: print or not the errors - return S_OK with number of updated registers upon success - return S_ERROR upon error + :return: S_OK with number of updated registers upon success. + S_ERROR upon error. """ - self.log.debug("_update: %s" % self._safeCmd(cmd)) + self.log.debug(f"_update: {self._safeCmd(cmd)}") if conn: connection = conn else: @@ -776,7 +773,7 @@ def _update(self, cmd, *, conn=None, debug=True): try: cursor = connection.cursor() - res = cursor.execute(cmd) + res = cursor.execute(cmd, args=args) retDict = S_OK(res) if cursor.lastrowid: retDict["lastRowId"] = cursor.lastrowid @@ -790,6 +787,41 @@ def _update(self, cmd, *, conn=None, debug=True): return retDict + @captureOptimizerTraces + def _updatemany(self, cmd, data, *, conn=None, debug=True): + """execute MySQL updatemany command + + :param debug: print or not the errors + + :return: S_OK with number of updated registers upon success. + S_ERROR upon error. + """ + + self.log.debug(f"_updatemany: {self._safeCmd(cmd)}") + if conn: + connection = conn + else: + retDict = self._getConnection() + if not retDict["OK"]: + return retDict + connection = retDict["Value"] + + try: + cursor = connection.cursor() + res = cursor.executemany(cmd, data) + retDict = S_OK(res) + if cursor.lastrowid: + retDict["lastRowId"] = cursor.lastrowid + except Exception as x: + retDict = self._except("_updatemany", x, "Execution failed.", cmd, debug) + + try: + cursor.close() + except Exception: + pass + + return retDict + def _transaction(self, cmdList, conn=None): """dummy transaction support @@ -800,7 +832,7 @@ def _transaction(self, cmdList, conn=None): :return: S_OK( [ ( cmd1, ret1 ), ... ] ) or S_ERROR """ if not isinstance(cmdList, list): - return S_ERROR(DErrno.EMYSQL, "_transaction: wrong type (%s) for cmdList" % type(cmdList)) + return S_ERROR(DErrno.EMYSQL, f"_transaction: wrong type ({type(cmdList)}) for cmdList") # # get connection connection = conn @@ -839,7 +871,6 @@ def _createViews(self, viewsDict, force=False): # gLogger.debug(viewsDict) for viewName, viewDict in viewsDict.items(): - viewQuery = [f"CREATE OR REPLACE VIEW `{self.__dbName}`.`{viewName}` AS"] columns = ",".join([f"{colDef} AS {colName}" for colName, colDef in viewDict.get("Fields", {}).items()]) @@ -849,15 +880,15 @@ def _createViews(self, viewsDict, force=False): where = " AND ".join(viewDict.get("Clauses", [])) if where: - viewQuery.append("WHERE %s" % where) + viewQuery.append(f"WHERE {where}") groupBy = ",".join(viewDict.get("GroupBy", [])) if groupBy: - viewQuery.append("GROUP BY %s" % groupBy) + viewQuery.append(f"GROUP BY {groupBy}") orderBy = ",".join(viewDict.get("OrderBy", [])) if orderBy: - viewQuery.append("ORDER BY %s" % orderBy) + viewQuery.append(f"ORDER BY {orderBy}") viewQuery.append(";") viewQuery = " ".join(viewQuery) @@ -898,7 +929,7 @@ def _createTables(self, tableDict, force=False): index is the list of fields to be indexed. This indexes will declared unique. "Engine": use the given DB engine, InnoDB is the default if not present. - "Charset": use the given character set. Default is latin1 + "Charset": use the given character set. Default is utf8mb4 force: if True, requested tables are DROP if they exist. if False, returned with S_ERROR if table exist. @@ -920,7 +951,7 @@ def _createTables(self, tableDict, force=False): DErrno.EMYSQL, f"Table description is not a dictionary: {type(thisTable)}( {thisTable} )" ) if "Fields" not in thisTable: - return S_ERROR(DErrno.EMYSQL, "Missing `Fields` key in `%s` table dictionary" % table) + return S_ERROR(DErrno.EMYSQL, f"Missing `Fields` key in `{table}` table dictionary") tableCreationList = [[]] @@ -967,7 +998,7 @@ def _createTables(self, tableDict, force=False): tableCreationList[i].append(table) if tableList: - return S_ERROR(DErrno.EMYSQL, "Recursive Foreign Keys in %s" % ", ".join(tableList)) + return S_ERROR(DErrno.EMYSQL, f"Recursive Foreign Keys in {', '.join(tableList)}") for tableList in tableCreationList: for table in tableList: @@ -979,11 +1010,11 @@ def _createTables(self, tableDict, force=False): thisTable = tableDict[table] cmdList = [] for field in thisTable["Fields"].keys(): - cmdList.append("`{}` {}".format(field, thisTable["Fields"][field])) + cmdList.append(f"`{field}` {thisTable['Fields'][field]}") if "PrimaryKey" in thisTable: if isinstance(thisTable["PrimaryKey"], str): - cmdList.append("PRIMARY KEY ( `%s` )" % thisTable["PrimaryKey"]) + cmdList.append(f"PRIMARY KEY ( `{thisTable['PrimaryKey']}` )") else: cmdList.append( "PRIMARY KEY ( %s )" % ", ".join(["`%s`" % str(f) for f in thisTable["PrimaryKey"]]) @@ -1003,7 +1034,6 @@ def _createTables(self, tableDict, force=False): if "ForeignKeys" in thisTable: thisKeys = thisTable["ForeignKeys"] for key, auxTable in thisKeys.items(): - forTable = auxTable.split(".")[0] forKey = key if forTable != auxTable: @@ -1016,7 +1046,7 @@ def _createTables(self, tableDict, force=False): ) engine = thisTable.get("Engine", "InnoDB") - charset = thisTable.get("Charset", "latin1") + charset = thisTable.get("Charset", "utf8mb4") cmd = "CREATE TABLE `{}` (\n{}\n) ENGINE={} DEFAULT CHARSET={}".format( table, @@ -1125,7 +1155,7 @@ def getCounters( # self.log.debug('getCounters:', error) return S_ERROR(DErrno.EMYSQL, error) - attrNames = _quotedList(attrList) + attrNames = _quotedList(attrList, allowDate=True) if attrNames is None: error = "Invalid updateFields argument" # self.log.debug('getCounters:', error) @@ -1220,7 +1250,7 @@ def buildCondition( """ condition = "" conjunction = "WHERE" - + attrName = None if condDict is not None: for aName, attrValue in condDict.items(): if isinstance(aName, str): @@ -1345,7 +1375,7 @@ def buildCondition( orderList.append(orderAttr) if orderList: - condition = "{} ORDER BY {}".format(condition, ", ".join(orderList)) + condition = f"{condition} ORDER BY {', '.join(orderList)}" if limit: if offset: @@ -1594,14 +1624,14 @@ def insertFields(self, tableName, inFields=None, inValues=None, conn=None, inDic # self.log.debug('insertFields:', error) return S_ERROR(DErrno.EMYSQL, error) - inFieldString = "( %s )" % inFieldString + inFieldString = f"( {inFieldString} )" retDict = self._escapeValues(inValues) if not retDict["OK"]: # self.log.debug('insertFields:', retDict['Message']) return retDict inValueString = ", ".join(retDict["Value"]) - inValueString = "( %s )" % inValueString + inValueString = f"( {inValueString} )" # self.log.debug('insertFields:', 'inserting %s into table %s' # % (inFieldString, table)) @@ -1624,7 +1654,7 @@ def executeStoredProcedure(self, packageName, parameters, outputIds, *, conn=Non row = [] for oId in outputIds: resName = f"@_{packageName}_{oId}" - cursor.execute("SELECT %s" % resName) + cursor.execute(f"SELECT {resName}") row.append(cursor.fetchone()[0]) retDict = S_OK(row) except Exception as x: diff --git a/src/DIRAC/Core/Utilities/NTP.py b/src/DIRAC/Core/Utilities/NTP.py deleted file mode 100644 index ae4a6142a7a..00000000000 --- a/src/DIRAC/Core/Utilities/NTP.py +++ /dev/null @@ -1,40 +0,0 @@ -from socket import socket, AF_INET, SOCK_DGRAM -import struct -import time as time -import datetime -from DIRAC import S_OK, S_ERROR - -TIME1970 = 2208988800 -gDefaultNTPServers = ["pool.ntp.org"] - - -def getNTPUTCTime(serverList=None, retries=2): - data = "\x1b" + 47 * "\0" - if not serverList: - serverList = gDefaultNTPServers - for server in serverList: - client = socket(AF_INET, SOCK_DGRAM) - client.settimeout(1) - worked = False - while retries >= 0 and not worked: - try: - client.sendto(data, (server, 123)) - data, address = client.recvfrom(1024) - worked = True - except Exception: - retries -= 1 - if not worked: - continue - if data: - myTime = struct.unpack("!12I", data)[10] - myTime -= TIME1970 - return S_OK(datetime.datetime(*time.gmtime(myTime)[:6])) - return S_ERROR("Could not get NTP time") - - -def getClockDeviation(serverList=None): - result = getNTPUTCTime(serverList) - if not result["OK"]: - return result - td = datetime.datetime.utcnow() - result["Value"] - return S_OK(abs(td.days * 86400 + td.seconds)) diff --git a/src/DIRAC/Core/Utilities/Network.py b/src/DIRAC/Core/Utilities/Network.py index 7e4757c40de..0c5e81078aa 100755 --- a/src/DIRAC/Core/Utilities/Network.py +++ b/src/DIRAC/Core/Utilities/Network.py @@ -30,7 +30,7 @@ def getFQDN(): def splitURL(url): o = parse.urlparse(url) if o.scheme == "": - return S_ERROR("'%s' URL is missing protocol" % url) + return S_ERROR(f"'{url}' URL is missing protocol") path = o.path path = path.lstrip("/") return S_OK((o.scheme, o.hostname or "", o.port or 0, path)) diff --git a/src/DIRAC/Core/Utilities/ObjectLoader.py b/src/DIRAC/Core/Utilities/ObjectLoader.py index 21d0df33a6c..2747f21b414 100644 --- a/src/DIRAC/Core/Utilities/ObjectLoader.py +++ b/src/DIRAC/Core/Utilities/ObjectLoader.py @@ -1,9 +1,13 @@ -""" An utility to load modules and objects in DIRAC and extensions, being sure that the extensions are considered +""" +An utility to load modules and objects in DIRAC and extensions, +being sure that the extensions are considered """ import collections +from importlib import import_module import os import re import pkgutil +from typing import Any import DIRAC from DIRAC import gLogger, S_OK, S_ERROR @@ -21,25 +25,13 @@ class ObjectLoader(metaclass=DIRACSingleton): ol.loadObject('TransformationSystem.Client.TransformationClient') """ - def __init__(self, baseModules=False): - """init""" + def __init__(self, baseModules: list[str] = None) -> None: # We save the original arguments in case # we need to reinitialize the rootModules - # CAUTION: we cant do it after doing - # baseModules = ['DIRAC'] - # because then baseModules, self.baseModules, and __rootModules - # are the same and edited in place by __generateRootModules !! - # (Think of it, it's a binding to a list) - self.originalBaseModules = baseModules - self._init(baseModules) - - def _init(self, baseModules): - """Actually performs the initialization""" if not baseModules: baseModules = ["DIRAC"] - self.__rootModules = baseModules - self.__objs = {} - self.__generateRootModules(baseModules) + self.__baseModules = baseModules + self.__rootModules = self.__generateRootModules(baseModules) def reloadRootModules(self): """Retrigger the initialization of the rootModules. @@ -49,19 +41,15 @@ def reloadRootModules(self): the initialization after the CS has been fully initialized in LocalConfiguration.enableCS """ - # Load the original baseModule argument that was given - # to the constructor - baseModules = self.originalBaseModules - # and replay the init sequence - self._init(baseModules) + self.__rootModules = self.__generateRootModules(self.__baseModules) - def __rootImport(self, modName, hideExceptions=False): + def __rootImport(self, modName: str, hideExceptions: bool = False): """Auto search which root module has to be used""" for rootModule in self.__rootModules: impName = modName if rootModule: impName = f"{rootModule}.{impName}" - gLogger.debug("Trying to load %s" % impName) + gLogger.debug(f"Trying to load {impName}") result = recurseImport(impName, hideExceptions=hideExceptions) if not result["OK"]: return result @@ -69,27 +57,29 @@ def __rootImport(self, modName, hideExceptions=False): return S_OK((impName, result["Value"])) return S_OK() - def __generateRootModules(self, baseModules): + def __generateRootModules(self, baseModules: list[str]) -> list[str]: """Iterate over all the possible root modules""" - self.__rootModules = baseModules + rootModules = baseModules for rootModule in reversed(extensionsByPriority()): - if rootModule not in self.__rootModules: - self.__rootModules.append(rootModule) - self.__rootModules.append("") + if rootModule not in rootModules: + rootModules.append(rootModule) + rootModules.append("") # Reversing the order because we want first to look in the extension(s) - self.__rootModules.reverse() + rootModules.reverse() + + return rootModules - def loadModule(self, importString, hideExceptions=False): + def loadModule(self, importString: str, hideExceptions: bool = False): """Load a module from an import string""" result = self.__rootImport(importString, hideExceptions=hideExceptions) if not result["OK"]: return result if not result["Value"]: - return S_ERROR(DErrno.EIMPERR, "No module %s found" % importString) + return S_ERROR(DErrno.EIMPERR, f"No module {importString} found") return S_OK(result["Value"][1]) - def loadObject(self, importString, objName=False, hideExceptions=False): + def loadObject(self, importString: str, objName: str = "", hideExceptions: bool = False): """Load an object from inside a module""" if not objName: objName = importString.split(".")[-1] @@ -105,7 +95,9 @@ def loadObject(self, importString, objName=False, hideExceptions=False): except AttributeError: return S_ERROR(DErrno.EIMPERR, f"{importString} does not contain a {objName} object") - def getObjects(self, modulePath, reFilter=None, parentClass=None, recurse=False, continueOnError=False): + def getObjects( + self, modulePath: str, reFilter=None, parentClass=None, recurse: bool = False, continueOnError: bool = False + ): """Search for modules under a certain path modulePath is the import string needed to access the parent module. @@ -124,7 +116,7 @@ def getObjects(self, modulePath, reFilter=None, parentClass=None, recurse=False, impPath = modulePath if rootModule: impPath = f"{rootModule}.{impPath}" - gLogger.debug("Trying to load %s" % impPath) + gLogger.debug(f"Trying to load {impPath}") result = recurseImport(impPath) if not result["OK"]: @@ -175,9 +167,14 @@ def getObjects(self, modulePath, reFilter=None, parentClass=None, recurse=False, return S_OK(modules) -def loadObjects(path, reFilter=None, parentClass=None): +def loadObjects(path: str, reFilter=None, parentClass: object = None) -> dict[str, Any]: """ - :param str path: the path to the syetem for example: DIRAC/AccountingSystem + + Note: this does not work for editable install because it hardcodes + DIRAC.__file__ + It is better to use ObjectLoader().getObjects() + + :param str path: the path to the system for example: DIRAC/AccountingSystem :param object reFilter: regular expression used to found the class :param object parentClass: class instance :return: dictionary containing the name of the class and its instance @@ -192,33 +189,37 @@ def loadObjects(path, reFilter=None, parentClass=None): objDir = os.path.join(os.path.dirname(os.path.dirname(DIRAC.__file__)), parentModule, *pathList) if not os.path.isdir(objDir): continue + for objFile in os.listdir(objDir): if reFilter.match(objFile): pythonClassName = objFile[:-3] if pythonClassName not in objectsToLoad: - gLogger.info(f"Adding to load queue {parentModule}/{path}/{pythonClassName}") + gLogger.debug(f"Adding to load queue {parentModule}/{path}/{pythonClassName}") objectsToLoad[pythonClassName] = parentModule # Load them! loadedObjects = {} - for pythonClassName in objectsToLoad: - parentModule = objectsToLoad[pythonClassName] + for pythonClassName, parentModule in objectsToLoad.items(): try: # Where parentModule can be DIRAC, pathList is something like [ "AccountingSystem", "Client", "Types" ] # And the python class name is.. well, the python class name - objPythonPath = "{}.{}.{}".format(parentModule, ".".join(pathList), pythonClassName) - objModule = __import__(objPythonPath, globals(), locals(), pythonClassName) + objPythonPath = f"{parentModule}.{'.'.join(pathList)}.{pythonClassName}" + objModule = import_module(objPythonPath) + except ImportError as e: + gLogger.error(f"No module {objPythonPath} found", str(e)) + continue + try: objClass = getattr(objModule, pythonClassName) - except Exception as e: - gLogger.error("Can't load type", f"{parentModule}/{pythonClassName}: {str(e)}") + except AttributeError as e: + gLogger.error(f"{objPythonPath} does not contain a {pythonClassName} object", str(e)) continue if parentClass == objClass: continue if parentClass and not issubclass(objClass, parentClass): gLogger.warn(f"{objClass} is not a subclass of {parentClass}. Skipping") continue - gLogger.info("Loaded %s" % objPythonPath) + gLogger.debug(f"Loaded {objPythonPath}") loadedObjects[pythonClassName] = objClass return loadedObjects diff --git a/src/DIRAC/Core/Utilities/Os.py b/src/DIRAC/Core/Utilities/Os.py index fa43a5bbece..0ac0c74826d 100755 --- a/src/DIRAC/Core/Utilities/Os.py +++ b/src/DIRAC/Core/Utilities/Os.py @@ -3,10 +3,8 @@ by default on Error they return None """ import os -import shutil import DIRAC -from DIRAC.Core.Utilities.Decorators import deprecated from DIRAC.Core.Utilities.Subprocess import shellCall, systemCall from DIRAC.Core.Utilities import List @@ -36,49 +34,44 @@ def getDiskSpace(path=".", exclude=None): if not os.path.exists(path): return -1 - comm = "df -P -m %s " % path + comm = f"df -P -m {path} " if exclude: - comm += "-x %s " % exclude + comm += f"-x {exclude} " comm += "| tail -1" resultDF = shellCall(10, comm) - if resultDF["OK"] and not resultDF["Value"][0]: - output = resultDF["Value"][1] - if output.find(" /afs") >= 0: # AFS disk space - comm = "fs lq | tail -1" - resultAFS = shellCall(10, comm) - if resultAFS["OK"] and not resultAFS["Value"][0]: - output = resultAFS["Value"][1] - fields = output.split() - quota = int(fields[1]) - used = int(fields[2]) - space = (quota - used) / 1024 - return int(space) - else: - return -1 - else: + if not resultDF["OK"] or resultDF["Value"][0]: + return -1 + output = resultDF["Value"][1] + if output.find(" /afs") >= 0: # AFS disk space + comm = "fs lq | tail -1" + resultAFS = shellCall(10, comm) + if resultAFS["OK"] and not resultAFS["Value"][0]: + output = resultAFS["Value"][1] fields = output.split() - try: - value = int(fields[3]) - except Exception as error: - print("Exception during disk space evaluation:", str(error)) - value = -1 - return value - else: + quota = int(fields[1]) + used = int(fields[2]) + space = (quota - used) / 1024 + return int(space) return -1 + fields = output.split() + try: + value = int(fields[3]) + except Exception as error: + print("Exception during disk space evaluation:", str(error)) + value = -1 + return value def getDirectorySize(path): """Get the total size of the given directory in MB""" - comm = "du -s -m %s" % path + comm = f"du -s -m {path}" result = shellCall(10, comm) if not result["OK"] or result["Value"][0] != 0: return 0 - else: - output = result["Value"][1] - print(output) - size = int(output.split()[0]) - return size + output = result["Value"][1] + print(output) + return int(output.split()[0]) def sourceEnv(timeout, cmdTuple, inputEnv=None): @@ -89,30 +82,20 @@ def sourceEnv(timeout, cmdTuple, inputEnv=None): # add appropriate extension to first element of the tuple (the command) envAsDict = '&& python -c "import os,sys ; print >> sys.stderr, os.environ"' - # 1.- Choose the right version of the configuration file - if DIRAC.getPlatformTuple()[0] == "Windows": - cmdTuple[0] += ".bat" - else: - cmdTuple[0] += ".sh" + cmdTuple[0] += ".sh" # 2.- Check that it exists if not os.path.exists(cmdTuple[0]): - result = DIRAC.S_ERROR("Missing script: %s" % cmdTuple[0]) + result = DIRAC.S_ERROR(f"Missing script: {cmdTuple[0]}") result["stdout"] = "" - result["stderr"] = "Missing script: %s" % cmdTuple[0] + result["stderr"] = f"Missing script: {cmdTuple[0]}" return result # Source it in a platform dependent way: - # On windows the execution makes the environment to be inherit # On Linux or Darwin use bash and source the file. - if DIRAC.getPlatformTuple()[0] == "Windows": - # this needs to be tested - cmd = " ".join(cmdTuple) + envAsDict - ret = shellCall(timeout, [cmd], env=inputEnv) - else: - cmdTuple.insert(0, "source") - cmd = " ".join(cmdTuple) + envAsDict - ret = systemCall(timeout, ["/bin/bash", "-c", cmd], env=inputEnv) + cmdTuple.insert(0, "source") + cmd = " ".join(cmdTuple) + envAsDict + ret = systemCall(timeout, ["/bin/bash", "-c", cmd], env=inputEnv) # 3.- Now get back the result stdout = "" @@ -133,7 +116,7 @@ def sourceEnv(timeout, cmdTuple, inputEnv=None): else: # execution error stdout = cmd + "\n" + stdout - result = DIRAC.S_ERROR("Execution returns %s" % ret["Value"][0]) + result = DIRAC.S_ERROR(f"Execution returns {ret['Value'][0]}") else: # Timeout stdout = cmd @@ -145,8 +128,3 @@ def sourceEnv(timeout, cmdTuple, inputEnv=None): result["stderr"] = stderr return result - - -@deprecated("Will be removed in DIRAC 8.1", onlyOnce=True) -def which(executable): - return shutil.which(executable) diff --git a/src/DIRAC/Core/Utilities/Pfn.py b/src/DIRAC/Core/Utilities/Pfn.py index 2dd541a5482..617f9a8790c 100755 --- a/src/DIRAC/Core/Utilities/Pfn.py +++ b/src/DIRAC/Core/Utilities/Pfn.py @@ -12,11 +12,13 @@ import os from urllib import parse +from typing import Optional + # # from DIRAC from DIRAC import S_OK, S_ERROR, gLogger -def pfnunparse(pfnDict, srmSpecific=True): +def pfnunparse(pfnDict: dict[str, str], srmSpecific: Optional[bool] = True) -> str: """Wrapper for backward compatibility Redirect either to the old hand made style of unparsing the pfn, which works for srm, or to the standard one @@ -38,7 +40,7 @@ def srm_pfnunparse(pfnDict): # # make sure all keys are in allDict = dict.fromkeys(["Protocol", "Host", "Port", "WSUrl", "Path", "FileName"], "") if not isinstance(pfnDict, dict): - return S_ERROR("pfnunparse: wrong type for pfnDict argument, expected a dict, got %s" % type(pfnDict)) + return S_ERROR(f"pfnunparse: wrong type for pfnDict argument, expected a dict, got {type(pfnDict)}") allDict.update(pfnDict) pfnDict = allDict @@ -51,26 +53,26 @@ def srm_pfnunparse(pfnDict): if pfnDict["Host"]: if pfnDict["Port"]: # host:port - uri = "{}:{}".format(pfnDict["Host"], pfnDict["Port"]) + uri = f"{pfnDict['Host']}:{pfnDict['Port']}" if pfnDict["WSUrl"]: - if "?" in pfnDict["WSUrl"] and "=" in pfnDict["WSUrl"]: + if "?" in pfnDict["WSUrl"] and "=" in pfnDict["WSUrl"]: # pylint: disable=unsupported-membership-test # host/wsurl # host:port/wsurl - uri = "{}{}".format(uri, pfnDict["WSUrl"]) + uri = f"{uri}{pfnDict['WSUrl']}" else: # host/wsurl # host:port/wsurl - uri = "{}{}?=".format(uri, pfnDict["WSUrl"]) + uri = f"{uri}{pfnDict['WSUrl']}?=" if pfnDict["Protocol"]: if uri: # proto://host # proto://host:port # proto://host:port/wsurl - uri = "{}://{}".format(pfnDict["Protocol"], uri) + uri = f"{pfnDict['Protocol']}://{uri}" else: # proto: - uri = "%s:" % pfnDict["Protocol"] + uri = f"{pfnDict['Protocol']}:" pfn = f"{uri}{filePath}" @@ -92,7 +94,7 @@ def default_pfnunparse(pfnDict): try: if not isinstance(pfnDict, dict): - return S_ERROR("pfnunparse: wrong type for pfnDict argument, expected a dict, got %s" % type(pfnDict)) + return S_ERROR(f"pfnunparse: wrong type for pfnDict argument, expected a dict, got {type(pfnDict)}") allDict = dict.fromkeys(["Protocol", "Host", "Port", "Path", "FileName", "Options"], "") allDict.update(pfnDict) @@ -100,7 +102,7 @@ def default_pfnunparse(pfnDict): netloc = allDict["Host"] if allDict["Port"]: - netloc += ":%s" % allDict["Port"] + netloc += f":{allDict['Port']}" path = os.path.join(allDict["Path"], allDict["FileName"]) query = allDict["Options"] @@ -112,12 +114,12 @@ def default_pfnunparse(pfnDict): return S_OK(pfn) except Exception as e: # pylint: disable=broad-except - errStr = "Pfn.default_pfnunparse: Exception while unparsing pfn: %s" % pfnDict + errStr = f"Pfn.default_pfnunparse: Exception while unparsing pfn: {pfnDict}" gLogger.exception(errStr, lException=e) return S_ERROR(errStr) -def pfnparse(pfn, srmSpecific=True): +def pfnparse(pfn: str, srmSpecific: Optional[bool] = True) -> dict[str, str]: """Wrapper for backward compatibility Redirect either to the old hand made style of parsing the pfn, which works for srm, or to the standard one @@ -137,7 +139,7 @@ def srm_pfnparse(pfn): :param str pfn: pfn string """ if not pfn: - return S_ERROR("wrong 'pfn' argument value in function call, expected non-empty string, got %s" % str(pfn)) + return S_ERROR(f"wrong 'pfn' argument value in function call, expected non-empty string, got {str(pfn)}") pfnDict = dict.fromkeys(["Protocol", "Host", "Port", "WSUrl", "Path", "FileName"], "") try: if ":" not in pfn: @@ -206,10 +208,9 @@ def default_pfnparse(pfn): :param str pfn: pfn string """ if not pfn: - return S_ERROR("wrong 'pfn' argument value in function call, expected non-empty string, got %s" % str(pfn)) + return S_ERROR(f"wrong 'pfn' argument value in function call, expected non-empty string, got {str(pfn)}") pfnDict = dict.fromkeys(["Protocol", "Host", "Port", "WSUrl", "Path", "FileName"], "") try: - parsed = parse.urlparse(pfn) pfnDict["Protocol"] = parsed.scheme if ":" in parsed.netloc: diff --git a/src/DIRAC/Core/Utilities/Platform.py b/src/DIRAC/Core/Utilities/Platform.py index 1a42899c777..ff13369fdcd 100644 --- a/src/DIRAC/Core/Utilities/Platform.py +++ b/src/DIRAC/Core/Utilities/Platform.py @@ -44,9 +44,9 @@ def libc_ver(executable=sys.executable, lib="", version="", chunksize=2048): lib = b"libc" elif glibc: glibcversion_parts = glibcversion.split(b".") - for i in range(len(glibcversion_parts)): + for i, part in enumerate(glibcversion_parts): try: - glibcversion_parts[i] = int(glibcversion_parts[i]) + glibcversion_parts[i] = int(part) except ValueError: glibcversion_parts[i] = 0 if libcinit and not lib: @@ -105,9 +105,9 @@ def getPlatformString(): newest_lib = [0, 0, 0] for lib in libs: lib_parts = libc_ver(lib)[1].split(".") - for i in range(len(lib_parts)): + for i, part in enumerate(lib_parts): try: - lib_parts[i] = int(lib_parts[i]) + lib_parts[i] = int(part) except ValueError: lib_parts[i] = 0 # print "non integer version numbers" @@ -117,8 +117,6 @@ def getPlatformString(): platformTuple += ("glibc-" + ".".join(map(str, newest_lib)),) elif platformTuple[0] == "Darwin": platformTuple += (".".join(platform.mac_ver()[0].split(".")[:2]),) - elif platformTuple[0] == "Windows": - platformTuple += (platform.win32_ver()[0],) else: platformTuple += platform.release() diff --git a/src/DIRAC/Core/Utilities/Plotting/DataCache.py b/src/DIRAC/Core/Utilities/Plotting/DataCache.py index 890a0d75d3c..efc2b841e52 100644 --- a/src/DIRAC/Core/Utilities/Plotting/DataCache.py +++ b/src/DIRAC/Core/Utilities/Plotting/DataCache.py @@ -26,7 +26,7 @@ def setGraphsLocation(self, graphsDir): for graphName in os.listdir(self.graphsLocation): if graphName.find(".png") > 0: graphLocation = f"{self.graphsLocation}/{graphName}" - gLogger.verbose("Purging %s" % graphLocation) + gLogger.verbose(f"Purging {graphLocation}") os.unlink(graphLocation) def purgeExpired(self): @@ -60,9 +60,9 @@ def getReportPlot(self, reportRequest, reportHash, reportData, plotFunc): return retVal plotDict = retVal["Value"] if plotDict["plot"]: - plotDict["plot"] = "%s.png" % reportHash + plotDict["plot"] = f"{reportHash}.png" if plotDict["thumbnail"]: - plotDict["thumbnail"] = "%s.thb.png" % reportHash + plotDict["thumbnail"] = f"{reportHash}.thb.png" self.__graphCache.add(reportHash, self.__graphLifeTime, plotDict) return S_OK(plotDict) diff --git a/src/DIRAC/Core/Utilities/Plotting/FileCoding.py b/src/DIRAC/Core/Utilities/Plotting/FileCoding.py index bdc90f3a4a0..23a28944c59 100644 --- a/src/DIRAC/Core/Utilities/Plotting/FileCoding.py +++ b/src/DIRAC/Core/Utilities/Plotting/FileCoding.py @@ -16,22 +16,22 @@ def codeRequestInFileId(plotRequest, compressIfPossible=True): compress = compressIfPossible and gZCompressionEnabled thbStub = False if compress: - plotStub = "Z:%s" % base64.urlsafe_b64encode(zlib.compress(DEncode.encode(plotRequest), 9)).decode() + plotStub = f"Z:{base64.urlsafe_b64encode(zlib.compress(DEncode.encode(plotRequest), 9)).decode()}" elif not gForceRawEncoding: - plotStub = "S:%s" % base64.urlsafe_b64encode(DEncode.encode(plotRequest)) + plotStub = f"S:{base64.urlsafe_b64encode(DEncode.encode(plotRequest))}" else: - plotStub = "R:%s" % DEncode.encode(plotRequest) + plotStub = f"R:{DEncode.encode(plotRequest)}" # If thumbnail requested, use plot as thumbnail, and generate stub for plot without one extraArgs = plotRequest["extraArgs"] if "thumbnail" in extraArgs and extraArgs["thumbnail"]: thbStub = plotStub extraArgs["thumbnail"] = False if compress: - plotStub = "Z:%s" % base64.urlsafe_b64encode(zlib.compress(DEncode.encode(plotRequest), 9)).decode() + plotStub = f"Z:{base64.urlsafe_b64encode(zlib.compress(DEncode.encode(plotRequest), 9)).decode()}" elif not gForceRawEncoding: - plotStub = "S:%s" % base64.urlsafe_b64encode(DEncode.encode(plotRequest)).decode() + plotStub = f"S:{base64.urlsafe_b64encode(DEncode.encode(plotRequest)).decode()}" else: - plotStub = "R:%s" % DEncode.encode(plotRequest).decode() + plotStub = f"R:{DEncode.encode(plotRequest).decode()}" return S_OK({"plot": plotStub, "thumbnail": thbStub}) @@ -45,25 +45,25 @@ def extractRequestFromFileId(fileId): stub = base64.urlsafe_b64decode(stub.encode()) except Exception as e: gLogger.error("Oops! Plot request is not properly encoded!", str(e)) - return S_ERROR("Oops! Plot request is not properly encoded!: %s" % str(e)) + return S_ERROR(f"Oops! Plot request is not properly encoded!: {str(e)}") try: stub = zlib.decompress(stub) except Exception as e: gLogger.error("Oops! Plot request is invalid!", str(e)) - return S_ERROR("Oops! Plot request is invalid!: %s" % str(e)) + return S_ERROR(f"Oops! Plot request is invalid!: {str(e)}") elif compressType == "S": gLogger.info("Base64 request, decoding") try: stub = base64.urlsafe_b64decode(stub) except Exception as e: gLogger.error("Oops! Plot request is not properly encoded!", str(e)) - return S_ERROR("Oops! Plot request is not properly encoded!: %s" % str(e)) + return S_ERROR(f"Oops! Plot request is not properly encoded!: {str(e)}") elif compressType == "R": # Do nothing, it's already uncompressed pass else: gLogger.error("Oops! Stub type is unknown", compressType) - return S_ERROR("Oops! Stub type '%s' is unknown :P" % compressType) + return S_ERROR(f"Oops! Stub type '{compressType}' is unknown :P") plotRequest, stubLength = DEncode.decode(stub) if len(stub) != stubLength: gLogger.error("Oops! The stub is longer than the data :P") diff --git a/src/DIRAC/Core/Utilities/Plotting/ObjectLoader.py b/src/DIRAC/Core/Utilities/Plotting/ObjectLoader.py deleted file mode 100644 index a3c478a7c08..00000000000 --- a/src/DIRAC/Core/Utilities/Plotting/ObjectLoader.py +++ /dev/null @@ -1,4 +0,0 @@ -""" -It is used to load classes from a specific system. -""" -from DIRAC.Core.Utilities.ObjectLoader import loadObjects diff --git a/src/DIRAC/Core/Utilities/Plotting/Plots.py b/src/DIRAC/Core/Utilities/Plotting/Plots.py index 0804d35ba28..5f86a18e059 100644 --- a/src/DIRAC/Core/Utilities/Plotting/Plots.py +++ b/src/DIRAC/Core/Utilities/Plotting/Plots.py @@ -32,7 +32,7 @@ def generateNoDataPlot(fileName, data, metadata): """ try: with open(fileName, "wb") as fn: - text = "No data for this selection for the plot: %s" % metadata["title"] + text = f"No data for this selection for the plot: {metadata['title']}" textGraph(text, fn, metadata) except OSError as e: return S_ERROR(errno.EIO, e) diff --git a/src/DIRAC/Core/Utilities/Plotting/TypeLoader.py b/src/DIRAC/Core/Utilities/Plotting/TypeLoader.py index e87dc17c267..deacba161d6 100644 --- a/src/DIRAC/Core/Utilities/Plotting/TypeLoader.py +++ b/src/DIRAC/Core/Utilities/Plotting/TypeLoader.py @@ -1,9 +1,10 @@ """ Utility for loading plotting types. Works both for Accounting and Monitoring. """ + import re -from DIRAC.Core.Utilities.ObjectLoader import loadObjects +from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader from DIRAC.AccountingSystem.Client.Types.BaseAccountingType import BaseAccountingType from DIRAC.MonitoringSystem.Client.Types.BaseType import BaseType @@ -26,12 +27,11 @@ def __init__(self, plottingFamily="Accounting"): """c'tor""" self.__loaded = {} if plottingFamily == "Accounting": - self.__path = "AccountingSystem/Client/Types" + self.__path = "AccountingSystem.Client.Types" self.__parentCls = BaseAccountingType elif plottingFamily == "Monitoring": - self.__path = "MonitoringSystem/Client/Types" + self.__path = "MonitoringSystem.Client.Types" self.__parentCls = BaseType - self.__reFilter = re.compile(r".*[a-z1-9]\.py$") ######################################################################## def getTypes(self): @@ -39,5 +39,9 @@ def getTypes(self): It returns all monitoring classes """ if not self.__loaded: - self.__loaded = loadObjects(self.__path, self.__reFilter, self.__parentCls) + allObjects = ObjectLoader().getObjects(self.__path, parentClass=self.__parentCls)["Value"] + for _objectModule, objectClass in allObjects.items(): + if objectClass.__name__ not in self.__loaded and objectClass != self.__parentCls: + self.__loaded[objectClass.__name__] = objectClass + return self.__loaded diff --git a/src/DIRAC/Core/Utilities/PrettyPrint.py b/src/DIRAC/Core/Utilities/PrettyPrint.py index 2ca0cae1070..8874ce6b166 100644 --- a/src/DIRAC/Core/Utilities/PrettyPrint.py +++ b/src/DIRAC/Core/Utilities/PrettyPrint.py @@ -35,7 +35,6 @@ def printTable(fields, records, sortField="", numbering=True, printOut=True, col """ def __writeField(buffer, value, length, columnSeparator, lastColumn=False): - justification = None if isinstance(value, dict): justification = value.get("Just") @@ -88,7 +87,7 @@ def __writeField(buffer, value, length, columnSeparator, lastColumn=False): ll["Value"] = ll["Value"].strip() strippedList.append(ll) else: - out = "Wrong type for field value: %s" % type(ll) + out = f"Wrong type for field value: {type(ll)}" if printOut: print(out) return out @@ -104,7 +103,7 @@ def __writeField(buffer, value, length, columnSeparator, lastColumn=False): fieldValue.update({"Value": itemValue}) strippedRecord.append(fieldValue) else: - out = "Wrong type for field value: %s" % type(itemValue) + out = f"Wrong type for field value: {type(itemValue)}" if printOut: print(out) return out @@ -241,7 +240,7 @@ def printDict(dDict, printOut=False): if len(key) > keyLength: keyLength = len(key) for key in sorted(dDict): - line = "%s: " % key + line = f"{key}: " line = line.ljust(keyLength + 2) value = dDict[key] if isinstance(value, (list, tuple)): diff --git a/src/DIRAC/Core/Utilities/ProcessPool.py b/src/DIRAC/Core/Utilities/ProcessPool.py index b39833a80da..ad86fa65788 100644 --- a/src/DIRAC/Core/Utilities/ProcessPool.py +++ b/src/DIRAC/Core/Utilities/ProcessPool.py @@ -93,6 +93,7 @@ executing same type of callables in subprocesses and hence you are expecting the same type of results everywhere. """ + import errno import inspect import multiprocessing @@ -256,7 +257,6 @@ def run(self): # main loop while True: - # draining, stopEvent is set, exiting if self.__stopEvent.is_set(): return @@ -522,7 +522,7 @@ def process(self): taskObj = self.__taskFunction(*self.__taskArgs, **self.__taskKwArgs) # ## check if it is callable, raise TypeError if not if not callable(taskObj): - raise TypeError("__call__ operator not defined not in %s class" % taskObj.__class__.__name__) + raise TypeError(f"__call__ operator not defined not in {taskObj.__class__.__name__} class") # ## call it at least self.__taskResult = taskObj() except Exception as x: @@ -547,14 +547,14 @@ class ProcessPool: Pool depth - The :ProcessPool: is keeping required number of active workers all the time: slave workers are only created + The :ProcessPool: is keeping required number of active workers all the time: worker workers are only created when pendingQueue is being filled with tasks, not exceeding defined min and max limits. When pendingQueue is empty, active workers will be cleaned up by themselves, as each worker has got built in self-destroy mechanism after 10 idle loops. Processing and communication - The communication between :ProcessPool: instance and slaves is performed using two :multiprocessing.Queues: + The communication between :ProcessPool: instance and workers is performed using two :multiprocessing.Queues: * pendingQueue, used to push tasks to the workers, * resultsQueue for revert direction; @@ -896,10 +896,10 @@ def processResults(self): log.debug("Process results, queue size = %d" % self.__resultsQueueApproxSize) start = time.time() self.__cleanDeadProcesses() - log.debug("__cleanDeadProcesses", "t=%.2f" % (time.time() - start)) + log.debug("__cleanDeadProcesses", f"t={time.time() - start:.2f}") if not self.__pendingQueue.empty(): self.__spawnNeededWorkingProcesses() - log.debug("__spawnNeededWorkingProcesses", "t=%.2f" % (time.time() - start)) + log.debug("__spawnNeededWorkingProcesses", f"t={time.time() - start:.2f}") time.sleep(0.1) if self.__resultsQueue.empty(): if self.__resultsQueueApproxSize: @@ -912,20 +912,26 @@ def processResults(self): if processed == 0: log.debug("Process results, but queue is empty...") break - # get task - task = self.__resultsQueue.get() - log.debug("__resultsQueue.get", "t=%.2f" % (time.time() - start)) + # In principle, there should be a task right away. However, + # queue.empty can't be trusted (https://docs.python.org/3/library/queue.html#queue.Queue.empty) + try: + task = self.__resultsQueue.get(timeout=10) + except queue.Empty: + log.warn("Queue.empty lied to us again...") + return 0 + + log.debug("__resultsQueue.get", f"t={time.time() - start:.2f}") # execute callbacks try: task.doExceptionCallback() task.doCallback() - log.debug("doCallback", "t=%.2f" % (time.time() - start)) + log.debug("doCallback", f"t={time.time() - start:.2f}") if task.usePoolCallbacks(): if self.__poolExceptionCallback and task.exceptionRaised(): self.__poolExceptionCallback(task.getTaskID(), task.taskException()) if self.__poolCallback and task.taskResults(): self.__poolCallback(task.getTaskID(), task.taskResults()) - log.debug("__poolCallback", "t=%.2f" % (time.time() - start)) + log.debug("__poolCallback", f"t={time.time() - start:.2f}") except Exception as error: log.exception("Exception in callback", lException=error) pass diff --git a/src/DIRAC/Core/Utilities/Profiler.py b/src/DIRAC/Core/Utilities/Profiler.py index 9ab83086886..f6dbfa67b2b 100644 --- a/src/DIRAC/Core/Utilities/Profiler.py +++ b/src/DIRAC/Core/Utilities/Profiler.py @@ -16,14 +16,14 @@ def wrapper(*args, **kwargs): try: return func(*args, **kwargs) except psutil.ZombieProcess as e: - gLogger.error("Zombie process: %s" % e) - return S_ERROR(EEZOMBIE, "Zombie process: %s" % e) + gLogger.error(f"Zombie process: {e}") + return S_ERROR(EEZOMBIE, f"Zombie process: {e}") except psutil.NoSuchProcess as e: - gLogger.error("No such process: %s" % e) - return S_ERROR(errno.ESRCH, "No such process: %s" % e) + gLogger.error(f"No such process: {e}") + return S_ERROR(errno.ESRCH, f"No such process: {e}") except psutil.AccessDenied as e: - gLogger.error("Access denied: %s" % e) - return S_ERROR(errno.EPERM, "Access denied: %s" % e) + gLogger.error(f"Access denied: {e}") + return S_ERROR(errno.EPERM, f"Access denied: {e}") except Exception as e: # pylint: disable=broad-except gLogger.error(e) return S_ERROR(EEEXCEPTION, e) @@ -47,7 +47,7 @@ def __init__(self, pid=None): try: self.process = psutil.Process(int(pid)) except psutil.NoSuchProcess as e: - gLogger.error("No such process: %s" % e) + gLogger.error(f"No such process: {e}") def pid(self): """ @@ -139,7 +139,7 @@ def cpuUsageUser(self, withChildren=False, withTerminatedChildren=False): oldChildrenUser += child.cpu_times().children_user gLogger.debug("CPU user (process, old children)", f"({cpuUsageUser:.1f}s, {oldChildrenUser:.1f}s)") else: - gLogger.debug("CPU user", "%.1fs" % cpuUsageUser) + gLogger.debug("CPU user", f"{cpuUsageUser:.1f}s") return S_OK(cpuUsageUser + childrenUser + oldChildrenUser) @checkInvocation @@ -159,7 +159,7 @@ def cpuUsageSystem(self, withChildren=False, withTerminatedChildren=False): oldChildrenSystem += child.cpu_times().children_system gLogger.debug("CPU user (process, old children)", f"({cpuUsageSystem:.1f}s, {oldChildrenSystem:.1f}s)") else: - gLogger.debug("CPU user", "%.1fs" % cpuUsageSystem) + gLogger.debug("CPU user", f"{cpuUsageSystem:.1f}s") return S_OK(cpuUsageSystem + childrenSystem + oldChildrenSystem) def getAllProcessData(self, withChildren=False, withTerminatedChildren=False): diff --git a/src/DIRAC/Core/Utilities/PromptUser.py b/src/DIRAC/Core/Utilities/PromptUser.py index eb0c246d73b..b785a21e64a 100644 --- a/src/DIRAC/Core/Utilities/PromptUser.py +++ b/src/DIRAC/Core/Utilities/PromptUser.py @@ -16,11 +16,11 @@ def promptUser(message, choices=[], default="n", logger=None): return S_ERROR("The default value is not a valid choice") choiceString = "" if choices and default: - choiceString = "/".join(choices).replace(default, "[%s]" % default) + choiceString = "/".join(choices).replace(default, f"[{default}]") elif choices and (not default): choiceString = "/".join(choices) elif (not choices) and (default): - choiceString = "[%s]" % default + choiceString = f"[{default}]" while True: if choiceString: @@ -28,7 +28,7 @@ def promptUser(message, choices=[], default="n", logger=None): elif default: logger.notice(f"{message} {default} :") else: - logger.notice("%s :" % message) + logger.notice(f"{message} :") response = input("") if (not response) and (default): return S_OK(default) diff --git a/src/DIRAC/Core/Utilities/Proxy.py b/src/DIRAC/Core/Utilities/Proxy.py index 46e7b600d7f..f393c8c8172 100644 --- a/src/DIRAC/Core/Utilities/Proxy.py +++ b/src/DIRAC/Core/Utilities/Proxy.py @@ -34,6 +34,7 @@ def undecoratedFunction(foo='bar'): """ +import functools import os from DIRAC import gConfig, gLogger, S_ERROR, S_OK @@ -61,8 +62,8 @@ def executeWithUserProxy(fcn): :param bool executionLock: flag to execute with a lock for the time of user proxy application ( default False ) """ + @functools.wraps(fcn) def wrapped_fcn(*args, **kwargs): - userName = kwargs.pop("proxyUserName", "") userDN = kwargs.pop("proxyUserDN", "") userGroup = kwargs.pop("proxyUserGroup", "") @@ -71,7 +72,6 @@ def wrapped_fcn(*args, **kwargs): executionLockFlag = kwargs.pop("executionLock", False) if (userName or userDN) and userGroup: - proxyResults = _putProxy( userName=userName, userDN=userDN, @@ -118,7 +118,7 @@ def getProxy(userDNs, userGroup, vomsAttr, proxyFilePath): if not result["OK"]: gLogger.error( - "Can't download %sproxy " % ("VOMS" if vomsAttr else ""), + f"Can't download {'VOMS' if vomsAttr else ''}proxy ", f"of '{userDN}', group {userGroup} to file: " + result["Message"], ) else: @@ -149,7 +149,6 @@ def executeWithoutServerCertificate(fcn): """ def wrapped_fcn(*args, **kwargs): - # Get the lock and acquire it executionLock = LockRing().getLock("_UseUserProxy_", recursive=True) executionLock.acquire() @@ -161,10 +160,6 @@ def wrapped_fcn(*args, **kwargs): try: return fcn(*args, **kwargs) - except Exception as lException: # pylint: disable=broad-except - value = ",".join([str(arg) for arg in lException.args]) - exceptType = lException.__class__.__name__ - return S_ERROR(f"Exception - {exceptType}: {value}") finally: # Restore the default host certificate usage if necessary if useServerCertificate: diff --git a/src/DIRAC/Core/Utilities/RabbitMQAdmin.py b/src/DIRAC/Core/Utilities/RabbitMQAdmin.py deleted file mode 100644 index 69be71fc629..00000000000 --- a/src/DIRAC/Core/Utilities/RabbitMQAdmin.py +++ /dev/null @@ -1,153 +0,0 @@ -"""RabbitMQAdmin module serves for the management of the internal RabbitMQ - users database. It uses rabbitmqctl command. Only the user with the right - permissions can execute those commands. -""" -import re -from DIRAC import S_OK, S_ERROR -import errno -from DIRAC.Core.Utilities import Subprocess - - -def executeRabbitmqctl(arg, *argv): - """Executes RabbitMQ administration command. - It uses rabbitmqctl command line interface. - For every command the -q argument ("quit mode") - is used, since in some cases the output must be processed, - so we don't want any additional informations printed. - - Args: - arg(str): command recognized by the rabbitmqctl. - argv: optional list of string parameters. - - :rtype: S_OK or S_ERROR - :type argv: python:list - - """ - command = ["sudo", "/usr/sbin/rabbitmqctl", "-q", arg] + list(argv) - timeOut = 30 - result = Subprocess.systemCall(timeout=timeOut, cmdSeq=command) - if not result["OK"]: - return S_ERROR(errno.EPERM, "%r failed to launch" % command) - errorcode, cmd_out, cmd_err = result["Value"] - if errorcode: - # No idea what errno code should be used here. - # Maybe we should define some specific for rabbitmqctl - return S_ERROR( - errno.EPERM, f"{command!r} failed, status code: {errorcode} stdout: {cmd_out!r} stderr: {cmd_err!r}" - ) - return S_OK(cmd_out) - - -def addUserWithoutPassword(user): - """Adds user to the internal RabbitMQ database - and clears its password. - This should be done for all users, that - will be using SSL authentication. They do not - need any password. - """ - ret = addUser(user) - if not ret["OK"]: - return ret - return clearUserPassword(user) - - -def addUser(user, password="password"): - """Adds user to the internal RabbitMQ database - Function also sets user password. - User still cannot access to any resources, without - having permissions set. - """ - return executeRabbitmqctl("add_user", user, password) - - -def deleteUser(user): - """Removes the user from the internal RabbitMQ database.""" - return executeRabbitmqctl("delete_user", user) - - -def getAllUsers(): - """Returns all existing users in the internal RabbitMQ database. - - :returns: S_OK with a list of all users - :rtype: S_OK - """ - ret = executeRabbitmqctl("list_users") - if not ret["OK"]: - return ret - users = ret["Value"] - users = users.split("\n") - # the rabbitMQ user list is given in the format: - # user_name [usr_tag] - # I remove [usr_tag] part. - # Also only non-empty users are proceeded further. - # Empty users can appear, cause every new line was - # treated as a new user. - users = [re.sub(r"\\t\[\w*\]$", "", u) for u in users if u] - return S_OK(users) - - -def setUserPermission(user): - return executeRabbitmqctl("set_permissions", "-p", "/", user, '".*"', '".*"', '".*"') - - -def clearUserPassword(user): - """Clears users password for the internal RabbitMQ - database. User with no password cannot enter - the RabbitMQ website interface but still can - connect via SSL if given permission. - """ - return executeRabbitmqctl("clear_password", user) - - -def setUsersPermissions(users): - successful = {} - failed = {} - for u in users: - ret = setUserPermission(u) - if ret["OK"]: - successful[u] = ret["Value"] - else: - print("Problem with permissions:%s" % ret["Message"]) - failed[u] = "Permission not set because of:%s" % ret["Message"] - return S_OK({"Successful": successful, "Failed": failed}) - - -def addUsersWithoutPasswords(users): - successful = {} - failed = {} - for u in users: - ret = addUserWithoutPassword(u) - if ret["OK"]: - successful[u] = ret["Value"] - else: - print("Problem with adding user:%s" % ret["Message"]) - failed[u] = "User not added" - return S_OK({"Successful": successful, "Failed": failed}) - - -def addUsers(users): - """Adds users to the RabbitMQ internal database.""" - successful = {} - failed = {} - for u in users: - ret = addUser(u) - if ret["OK"]: - successful[u] = ret["Value"] - else: - print("Problem with adding user:%s" % ret["Message"]) - failed[u] = "User not added" - return S_OK({"Successful": successful, "Failed": failed}) - - -def deleteUsers(users): - """Deletes users from the RabbitMQ internal database.""" - successful = {} - failed = {} - for u in users: - ret = deleteUser(u) - if ret["OK"]: - successful[u] = ret["Value"] - else: - print("Problem with adding user:%s" % ret["Message"]) - failed[u] = "User not added" - return S_OK({"Successful": successful, "Failed": failed}) diff --git a/src/DIRAC/Core/Utilities/ReturnValues.py b/src/DIRAC/Core/Utilities/ReturnValues.py index 2420dbb85d7..6312df1d89a 100755 --- a/src/DIRAC/Core/Utilities/ReturnValues.py +++ b/src/DIRAC/Core/Utilities/ReturnValues.py @@ -34,7 +34,7 @@ class DErrorReturnType(TypedDict): OK: Literal[False] Message: str Errno: int - ExecInfo: NotRequired[tuple[Type[BaseException], BaseException, TracebackType]] + ExecInfo: NotRequired[tuple[type[BaseException], BaseException, TracebackType]] CallStack: NotRequired[list[str]] @@ -64,7 +64,7 @@ def S_ERROR(*args: Any, **kwargs: Any) -> DErrorReturnType: message = args[0] if result["Errno"]: - message = "{} ( {} : {})".format(strerror(result["Errno"]), result["Errno"], message) + message = f"{strerror(result['Errno'])} ( {result['Errno']} : {message})" result["Message"] = message if callStack is None: @@ -186,20 +186,21 @@ def returnSingleResult(dictRes: DReturnType[Any]) -> DReturnType[Any]: class SErrorException(Exception): """Exception class for use with `convertToReturnValue`""" - def __init__(self, result: Union[DErrorReturnType, str]): + def __init__(self, result: DErrorReturnType | str, errCode: int = 0): """Create a new exception return value If `result` is a `S_ERROR` return it directly else convert it to an - appropriate value using `S_ERROR(result)`. + appropriate value using `S_ERROR(errCode, result)`. :param result: The error to propagate + :param errCode: the error code to propagate """ if not isSError(result): - result = S_ERROR(result) + result = S_ERROR(errCode, result) self.result = cast(DErrorReturnType, result) -def returnValueOrRaise(result: DReturnType[T]) -> T: +def returnValueOrRaise(result: DReturnType[T], *, errorCode: int = 0) -> T: """Unwrap an S_OK/S_ERROR response into a value or Exception This method assists with using exceptions in DIRAC code by raising @@ -216,7 +217,7 @@ def returnValueOrRaise(result: DReturnType[T]) -> T: if "ExecInfo" in result: raise result["ExecInfo"][0] else: - raise SErrorException(result) + raise SErrorException(result, errorCode) return result["Value"] @@ -238,10 +239,10 @@ def wrapped(*args: P.args, **kwargs: P.kwargs) -> DReturnType[T]: except SErrorException as e: return e.result except Exception as e: - retval = S_ERROR(repr(e)) + retval = S_ERROR(f"{repr(e)}: {e}") # Replace CallStack with the one from the exception # Use cast as mypy doesn't understand that sys.exc_info can't return None in an exception block - retval["ExecInfo"] = cast(tuple[Type[BaseException], BaseException, TracebackType], sys.exc_info()) + retval["ExecInfo"] = cast(tuple[type[BaseException], BaseException, TracebackType], sys.exc_info()) exc_type, exc_value, exc_tb = retval["ExecInfo"] retval["CallStack"] = traceback.format_tb(exc_tb) return retval diff --git a/src/DIRAC/Core/Utilities/Shifter.py b/src/DIRAC/Core/Utilities/Shifter.py index a6255e88710..775956ad02e 100644 --- a/src/DIRAC/Core/Utilities/Shifter.py +++ b/src/DIRAC/Core/Utilities/Shifter.py @@ -3,12 +3,11 @@ """ import os -from DIRAC import S_OK, S_ERROR, gLogger +from DIRAC import S_ERROR, S_OK, gLogger +from DIRAC.ConfigurationSystem.Client.Helpers import Registry, cfgPath +from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.Core.Utilities.File import mkDir from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager -from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations -from DIRAC.ConfigurationSystem.Client.Helpers import cfgPath -from DIRAC.ConfigurationSystem.Client.Helpers import Registry def getShifterProxy(shifterType, fileName=False): @@ -24,7 +23,7 @@ def getShifterProxy(shifterType, fileName=False): opsHelper = Operations() userName = opsHelper.getValue(cfgPath("Shifter", shifterType, "User"), "") if not userName: - return S_ERROR("No shifter User defined for %s" % shifterType) + return S_ERROR(f"No shifter User defined for {shifterType}") result = Registry.getDNForUsername(userName) if not result["OK"]: return result diff --git a/src/DIRAC/Core/Utilities/SiteSEMapping.py b/src/DIRAC/Core/Utilities/SiteSEMapping.py index fea6a136952..d168b103ff9 100644 --- a/src/DIRAC/Core/Utilities/SiteSEMapping.py +++ b/src/DIRAC/Core/Utilities/SiteSEMapping.py @@ -43,7 +43,7 @@ def getSEHosts(seName): seParameters = getSEParameters(seName) if not seParameters["OK"]: - gLogger.warn("Could not get SE parameters", "SE: %s" % seName) + gLogger.warn("Could not get SE parameters", f"SE: {seName}") return seParameters return S_OK([parameters["Host"] for parameters in seParameters["Value"]]) @@ -64,16 +64,15 @@ def getStorageElementsHosts(seNames=None): seNames = DMSHelpers().getStorageElements() for seName in seNames: - try: seHost = getSEHosts(seName) if not seHost["OK"]: - gLogger.warn("Could not get SE Host", "SE: %s" % seName) + gLogger.warn("Could not get SE Host", f"SE: {seName}") continue if seHost["Value"]: seHosts.extend(seHost["Value"]) except Exception as excp: # pylint: disable=broad-except - gLogger.error("Failed to get SE %s information (SE skipped) " % seName) + gLogger.error(f"Failed to get SE {seName} information (SE skipped) ") gLogger.exception("Operation finished with exception: ", lException=excp) return S_OK(list(set(seHosts))) diff --git a/src/DIRAC/Core/Utilities/StateMachine.py b/src/DIRAC/Core/Utilities/StateMachine.py index ef95d14ea64..aece8b95427 100644 --- a/src/DIRAC/Core/Utilities/StateMachine.py +++ b/src/DIRAC/Core/Utilities/StateMachine.py @@ -141,7 +141,7 @@ def setState(self, candidateState, noWarn=False): self.state = result["Value"] # If the StateMachine does not accept the candidate, return error message else: - return S_ERROR("setState: %r is not a valid state" % candidateState) + return S_ERROR(f"setState: {candidateState!r} is not a valid state") return S_OK(self.state) @@ -176,7 +176,7 @@ def getNextState(self, candidateState): :return: S_OK(nextState) || S_ERROR """ if candidateState not in self.states: - return S_ERROR("getNextState: %r is not a valid state" % candidateState) + return S_ERROR(f"getNextState: {candidateState!r} is not a valid state") # FIXME: do we need this anymore ? if self.state is None: diff --git a/src/DIRAC/Core/Utilities/Subprocess.py b/src/DIRAC/Core/Utilities/Subprocess.py index 1880bde67cc..58072c12811 100644 --- a/src/DIRAC/Core/Utilities/Subprocess.py +++ b/src/DIRAC/Core/Utilities/Subprocess.py @@ -26,25 +26,22 @@ should be used to wrap third party python functions """ -from multiprocessing import Process, Manager -import threading -import time import os -import sys -import subprocess +import selectors import signal -import psutil +import subprocess +import sys +import threading +import time +from multiprocessing import Manager, Process -try: - import selectors -except ImportError: - import selectors2 as selectors +import psutil # Very Important: # Here we can not import directly from DIRAC, since this file it is imported # at initialization time therefore the full path is necessary # from DIRAC import S_OK, S_ERROR -from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR +from DIRAC.Core.Utilities.ReturnValues import S_ERROR, S_OK # from DIRAC import gLogger from DIRAC.FrameworkSystem.Client.Logger import gLogger @@ -121,7 +118,7 @@ def __call__(self, timeout=0): self.__watchdogThread = threading.Thread(target=self.watchdog) self.__watchdogThread.daemon = True self.__watchdogThread.start() - ret = {"OK": False, "Message": "Timeout after %s seconds" % timeout, "Value": (1, "", "")} + ret = {"OK": False, "Message": f"Timeout after {timeout} seconds", "Value": (1, "", "")} try: self.__executor.start() time.sleep(0.5) @@ -269,40 +266,20 @@ def __poll(self, pid): def killChild(self, recursive=True): """kill child process - FIXME: this can easily be rewritten just using this recipe: - https://psutil.readthedocs.io/en/latest/index.html#kill-process-tree - :param boolean recursive: flag to kill all descendants """ - if self.childPID < 1: - self.log.error("Could not kill child", "Child PID is %s" % self.childPID) - return -1 - os.kill(self.childPID, signal.SIGSTOP) - if recursive: - for gcpid in getChildrenPIDs(self.childPID, lambda cpid: os.kill(cpid, signal.SIGSTOP)): - try: - os.kill(gcpid, signal.SIGKILL) - self.__poll(gcpid) - except Exception: - pass - self.__killPid(self.childPID) - - # HACK to avoid python bug - # self.child.wait() - exitStatus = self.__poll(self.childPID) - i = 0 - while exitStatus is None and i < 1000: - i += 1 - time.sleep(0.000001) - exitStatus = self.__poll(self.childPID) - try: - exitStatus = os.waitpid(self.childPID, 0) - except os.error: - pass - self.childKilled = True - if exitStatus is None: - return exitStatus - return exitStatus[1] + + parent = psutil.Process(self.childPID) + children = parent.children(recursive=recursive) + children.append(parent) + for p in children: + try: + p.send_signal(signal.SIGTERM) + except psutil.NoSuchProcess: + pass + _gone, alive = psutil.wait_procs(children, timeout=10) + for p in alive: + p.kill() def pythonCall(self, function, *stArgs, **stKeyArgs): """call python function :function: with :stArgs: and :stKeyArgs:""" @@ -323,7 +300,7 @@ def pythonCall(self, function, *stArgs, **stKeyArgs): os.close(writeFD) readSeq = self.__selectFD([readFD]) if not readSeq: - return S_ERROR("Can't read from call %s" % (function.__name__)) + return S_ERROR(f"Can't read from call {function.__name__}") try: if len(readSeq) == 0: self.log.debug("Timeout limit reached for pythonCall", function.__name__) @@ -400,11 +377,11 @@ def __readFromFile(self, fd, baseLength): except Exception as x: self.log.exception("SUBPROCESS: readFromFile exception") try: - self.log.error("Error reading", "type(nB) =%s" % type(nB)) - self.log.error("Error reading", "nB =%s" % str(nB)) + self.log.error("Error reading", f"type(nB) ={type(nB)}") + self.log.error("Error reading", f"nB ={str(nB)}") except Exception: pass - return S_ERROR("Can not read from output: %s" % str(x)) + return S_ERROR(f"Can not read from output: {str(x)}") if len(dataString) + baseLength > self.bufferLimit: self.log.error("Maximum output buffer length reached") retDict = S_ERROR( @@ -425,12 +402,10 @@ def __readFromSystemCommandOutput(self, fd, bufferIndex): pass return S_OK() else: # buffer size limit reached killing process (see comment on __readFromFile) - exitStatus = self.killChild() - return self.__generateSystemCommandError( - exitStatus, "{} for '{}' call".format(retDict["Message"], self.cmdSeq) - ) + self.killChild() + return self.__generateSystemCommandError(1, f"{retDict['Message']} for '{self.cmdSeq}' call") - def systemCall(self, cmdSeq, callbackFunction=None, shell=False, env=None): + def systemCall(self, cmdSeq, callbackFunction=None, shell=False, env=None, preexec_fn=None): """system call (no shell) - execute :cmdSeq:""" if shell: @@ -450,6 +425,7 @@ def systemCall(self, cmdSeq, callbackFunction=None, shell=False, env=None): close_fds=closefd, env=env, universal_newlines=True, + preexec_fn=preexec_fn, ) self.childPID = self.child.pid except OSError as v: @@ -478,10 +454,10 @@ def systemCall(self, cmdSeq, callbackFunction=None, shell=False, env=None): return retDict if self.timeout and time.time() - initialTime > self.timeout: - exitStatus = self.killChild() + self.killChild() self.__readFromCommand() return self.__generateSystemCommandError( - exitStatus, "Timeout (%d seconds) for '%s' call" % (self.timeout, cmdSeq) + 1, "Timeout (%d seconds) for '%s' call" % (self.timeout, cmdSeq) ) time.sleep(0.01) exitStatus = self.__poll(self.child.pid) @@ -543,7 +519,7 @@ def __callLineCallback(self, bufferIndex): self.bufferList[bufferIndex][0] = self.bufferList[bufferIndex][0][nL:] self.bufferList[bufferIndex][1] = 0 except Exception: - self.log.exception("Exception while calling callback function", "%s" % self.callback.__name__) + self.log.exception("Exception while calling callback function", f"{self.callback.__name__}") self.log.showStack() return False @@ -551,7 +527,7 @@ def __callLineCallback(self, bufferIndex): return False -def systemCall(timeout, cmdSeq, callbackFunction=None, env=None, bufferLimit=52428800): +def systemCall(timeout, cmdSeq, callbackFunction=None, env=None, bufferLimit=52428800, preexec_fn=None): """ Use SubprocessExecutor class to execute cmdSeq (it can be a string or a sequence) with a timeout wrapper, it is executed directly without calling a shell @@ -561,13 +537,15 @@ def systemCall(timeout, cmdSeq, callbackFunction=None, env=None, bufferLimit=524 sysCall = Watchdog( spObject.systemCall, args=(cmdSeq,), - kwargs={"callbackFunction": callbackFunction, "env": env, "shell": False}, + kwargs={"callbackFunction": callbackFunction, "env": env, "shell": False, "preexec_fn": preexec_fn}, ) spObject.log.verbose("Subprocess Watchdog timeout set to %d" % timeout) result = sysCall(timeout + 1) else: spObject = Subprocess(timeout, bufferLimit=bufferLimit) - result = spObject.systemCall(cmdSeq, callbackFunction=callbackFunction, env=env, shell=False) + result = spObject.systemCall( + cmdSeq, callbackFunction=callbackFunction, env=env, shell=False, preexec_fn=preexec_fn + ) return result diff --git a/src/DIRAC/Core/Utilities/ThreadPool.py b/src/DIRAC/Core/Utilities/ThreadPool.py index cdef3683413..35044b4db7b 100755 --- a/src/DIRAC/Core/Utilities/ThreadPool.py +++ b/src/DIRAC/Core/Utilities/ThreadPool.py @@ -327,4 +327,4 @@ def generateWork(iWorkUnits): gINew = gIResult + random.randint(-3, 2) print(f"Processed {gIResult}, generating {gINew}..") generateWork(gINew) - print("Threads %s" % OTP.numWorkingThreads(), OTP.pendingJobs()) + print(f"Threads {OTP.numWorkingThreads()}", OTP.pendingJobs()) diff --git a/src/DIRAC/Core/Utilities/ThreadScheduler.py b/src/DIRAC/Core/Utilities/ThreadScheduler.py index 76d20a6b308..e1afdfb57ce 100644 --- a/src/DIRAC/Core/Utilities/ThreadScheduler.py +++ b/src/DIRAC/Core/Utilities/ThreadScheduler.py @@ -29,7 +29,7 @@ def disableCreateReactorThread(self): def addPeriodicTask(self, period, taskFunc, taskArgs=(), executions=0, elapsedTime=0): if not callable(taskFunc): - return S_ERROR("%s is not callable" % str(taskFunc)) + return S_ERROR(f"{str(taskFunc)} is not callable") period = max(period, self.__minPeriod) elapsedTime = min(elapsedTime, period - 1) md = hashlib.md5() @@ -41,7 +41,7 @@ def addPeriodicTask(self, period, taskFunc, taskArgs=(), executions=0, elapsedTi md.update(str(task).encode()) taskId = md.hexdigest() if taskId in self.__taskDict: - return S_ERROR("Task %s is already added" % taskId) + return S_ERROR(f"Task {taskId} is already added") if executions: task["executions"] = executions self.__taskDict[taskId] = task @@ -60,13 +60,13 @@ def setTaskPeriod(self, taskId, period): try: self.__taskDict[taskId]["period"] = period except KeyError: - return S_ERROR("Unknown task %s" % taskId) + return S_ERROR(f"Unknown task {taskId}") return S_OK() @gSchedulerLock def removeTask(self, taskId): if taskId not in self.__taskDict: - return S_ERROR("Task %s does not exist" % taskId) + return S_ERROR(f"Task {taskId} does not exist") del self.__taskDict[taskId] for i in range(len(self.__hood)): if self.__hood[i][0] == taskId: diff --git a/src/DIRAC/Core/Utilities/TimeUtilities.py b/src/DIRAC/Core/Utilities/TimeUtilities.py index 2369dbe3ac3..6dc969008c9 100755 --- a/src/DIRAC/Core/Utilities/TimeUtilities.py +++ b/src/DIRAC/Core/Utilities/TimeUtilities.py @@ -19,14 +19,12 @@ if a give datetime is in the defined interval. """ -import calendar -import time import datetime import sys +import time from DIRAC import gLogger - # Some useful constants for time operations microsecond = datetime.timedelta(microseconds=1) second = datetime.timedelta(seconds=1) @@ -66,12 +64,12 @@ def timed(*args, **kw): if args: try: if isinstance(args[1], (list, dict)): - argsLen = "arguments len: %d" % len(args[1]) + argsLen = f"arguments len: {len(args[1])}" except IndexError: if kw: try: if isinstance(list(list(kw.items())[0])[1], (list, dict)): - argsLen = "arguments len: %d" % len(list(list(kw.items())[0])[1]) + argsLen = f"arguments len: {len(list(list(kw.items())[0])[1])}" except IndexError: argsLen = "" @@ -152,7 +150,12 @@ def fromString(myDate=None): The format of the string it is assume to be that returned by toString method. See notice on toString method On Error, return None + + :param myDate: the date string to be converted + :type myDate: str or datetime.datetime """ + if isinstance(myDate, datetime.datetime): + return myDate if isinstance(myDate, str): if myDate.find(" ") > 0: dateTimeTuple = myDate.split(" ") diff --git a/src/DIRAC/Core/Utilities/test/Test_Encode.py b/src/DIRAC/Core/Utilities/test/Test_Encode.py index c77fc5458a8..cbce7c9b6ef 100644 --- a/src/DIRAC/Core/Utilities/test/Test_Encode.py +++ b/src/DIRAC/Core/Utilities/test/Test_Encode.py @@ -111,7 +111,7 @@ def test_everyBaseTypeIsTested(): "test_BaseType" """ for encodeFunc in g_dEncodeFunctions.values(): - testFuncName = ("test_BaseType_%s" % encodeFunc.__name__).replace("encode", "") + testFuncName = f"test_BaseType_{encodeFunc.__name__}".replace("encode", "") globals()[testFuncName] @@ -143,7 +143,6 @@ def base_enc_dec(request, monkeypatch): return request.param enc_dec_tuple, use_json_decode, use_json_encode = request.param - monkeypatch.setenv("DIRAC_USE_JSON_DECODE", use_json_decode) monkeypatch.setenv("DIRAC_USE_JSON_ENCODE", use_json_encode) return enc_dec_tuple diff --git a/src/DIRAC/Core/Utilities/test/Test_ExecutorDispatcher.py b/src/DIRAC/Core/Utilities/test/Test_ExecutorDispatcher.py index 156e7de8177..58c09a548c8 100644 --- a/src/DIRAC/Core/Utilities/test/Test_ExecutorDispatcher.py +++ b/src/DIRAC/Core/Utilities/test/Test_ExecutorDispatcher.py @@ -33,7 +33,7 @@ def test_execQueues(): """test of ExecutorQueues""" for y in range(2): for i in range(3): - assert eQ.pushTask("type%s" % y, f"t{y}{i}") == i + 1 + assert eQ.pushTask(f"type{y}", f"t{y}{i}") == i + 1 assert "DONE IN" res_internals = eQ._internals() assert res_internals["queues"] == {"type0": ["t00", "t01", "t02"], "type1": ["t10", "t11", "t12"]} @@ -73,7 +73,7 @@ def test_execQueues(): } assert eQ.getState() for i in range(3): - assert eQ.popTask("type1")[0] == "t1%s" % i + assert eQ.popTask("type1")[0] == f"t1{i}" res_internals = eQ._internals() assert res_internals["queues"] == {"type0": [], "type1": []} assert set(res_internals["lastUse"].keys()) == {"type0", "type1"} diff --git a/src/DIRAC/Core/Utilities/test/Test_File.py b/src/DIRAC/Core/Utilities/test/Test_File.py index 1891f2df5e2..7846398c84c 100644 --- a/src/DIRAC/Core/Utilities/test/Test_File.py +++ b/src/DIRAC/Core/Utilities/test/Test_File.py @@ -101,8 +101,6 @@ def testGetMD5ForFiles(): md5sum = getMD5ForFiles(filesList) reMD5 = re.compile("^[0-9a-fA-F]+$") assert reMD5.match(md5sum) is not None - # OK for python 2.7 - # self.assertRegexpMatches( md5sum, reMD5, "regexp doesn't match" ) @given(nb=floats(allow_nan=False, allow_infinity=False, min_value=1)) diff --git a/src/DIRAC/Core/Utilities/test/Test_JDL.py b/src/DIRAC/Core/Utilities/test/Test_JDL.py new file mode 100644 index 00000000000..b918bf3bb2e --- /dev/null +++ b/src/DIRAC/Core/Utilities/test/Test_JDL.py @@ -0,0 +1,102 @@ +""" Unit tests for JDL module""" + +# pylint: disable=protected-access, invalid-name + +from io import StringIO +from unittest.mock import patch + +import pytest + +from DIRAC import S_OK +from DIRAC.Core.Utilities.ClassAd.ClassAdLight import ClassAd +from DIRAC.Core.Utilities.JDL import jdlToBaseJobDescriptionModel +from DIRAC.Interfaces.API.Job import Job +from DIRAC.WorkloadManagementSystem.Utilities.JobModel import JobDescriptionModel + + +@pytest.fixture() +def jdl_monkey_business(monkeypatch): + monkeypatch.setattr("DIRAC.Core.Base.API.getSites", lambda: S_OK(["LCG.IN2P3.fr"])) + monkeypatch.setattr("DIRAC.WorkloadManagementSystem.Utilities.JobModel.getSites", lambda: S_OK(["LCG.IN2P3.fr"])) + monkeypatch.setattr("DIRAC.Interfaces.API.Job.getDIRACPlatforms", lambda: S_OK("x86_64-slc6-gcc49-opt")) + monkeypatch.setattr( + "DIRAC.WorkloadManagementSystem.Utilities.JobModel.getDIRACPlatforms", lambda: S_OK("x86_64-slc6-gcc49-opt") + ) + yield + + +def test_jdlToBaseJobDescriptionModel_valid(jdl_monkey_business): + """This test makes sure that a job object can be parsed by the jdlToBaseJobDescriptionModel method""" + # Arrange + job = Job() + job.setConfigArgs("configArgs") + job.setCPUTime(3600) + job.setExecutable("/bin/echo", arguments="arguments", logFile="logFile") + job.setName("JobName") + with patch( + "DIRAC.ConfigurationSystem.Client.Helpers.Operations.Operations.getValue", + return_value="DIRAC.WorkloadManagementSystem.Client.DownloadInputData", + ): + job.setInputDataPolicy("download") + job.setInputSandbox(["inputfile.opts"]) + job.setOutputSandbox(["inputfile.opts"]) + job.setInputData(["/lhcb/production/DC04/v2/DST/00000742_00003493_10.dst"]) + job.setParameterSequence("IntSequence", [1, 2, 3]) + job.setParameterSequence("StrSequence", ["a", "b", "c"]) + job.setParameterSequence("FloatSequence", [1.0, 2.0, 3.0]) + + job.setOutputData(["outputfile.root"], outputSE="IN2P3-disk", outputPath="/myjobs/1234") + job.setPlatform("x86_64-slc6-gcc49-opt") + job.setPriority(10) + + job.setDestination("LCG.IN2P3.fr") + job.setNumberOfProcessors(3) + with patch("DIRAC.Interfaces.API.Job.getCESiteMapping", return_value=S_OK({"some.ce.IN2P3.fr": "LCG.IN2P3.fr"})): + job.setDestinationCE("some.ce.IN2P3.fr") + job.setType("Test") + job.setTag(["WholeNode", "8GBMemory"]) + job.setJobGroup("1234abcd") + job.setLogLevel("DEBUG") + job.setConfigArgs("configArgs") + job.setExecutionEnv({"INTVAR": 1, "STRVAR": "a"}) + job._addJDLParameter("ExtraInt", 1) + job._addJDLParameter("ExtraFloat", 1.0) + job._addJDLParameter("ExtraString", "test") + # The 3 lines below are not a use case given that workflow parameters + # must be strings, ints, floats or booleans + # job._addJDLParameter("ExtraIntList", ";".join(["1", "2", "3"])) + # job._addJDLParameter("ExtraFloatList", ";".join(["1.0", "2.0", "3.0"])) + # job._addJDLParameter("ExtraStringList",";".join(["a", "b", "c"])) + + # We make sure that the job above is valid + assert not job.errorDict + + # Act + xml = job._toXML() + jdl = f"[{job._toJDL(jobDescriptionObject=StringIO(xml))}]" + + # Assert + res = jdlToBaseJobDescriptionModel(ClassAd(jdl)) + assert res["OK"], res["Message"] + + data = res["Value"].model_dump() + assert JobDescriptionModel(owner="owner", ownerGroup="ownerGroup", vo="lhcb", **data) + + +@pytest.mark.parametrize( + "jdl", + [ + """[]""", # No executable + """[Executable="";]""", # Empty executable + """Executable="executable";""", # Missing brackets + ], +) +def test_jdlToBaseJobDescriptionModel_invalid(jdl, jdl_monkey_business): + """This test makes sure that a job object without an executable raises an error""" + # Arrange + + # Act + res = jdlToBaseJobDescriptionModel(ClassAd(jdl)) + + # Assert + assert not res["OK"], res["Value"] diff --git a/src/DIRAC/Core/Utilities/test/Test_List.py b/src/DIRAC/Core/Utilities/test/Test_List.py index 361976202af..05031ea9257 100644 --- a/src/DIRAC/Core/Utilities/test/Test_List.py +++ b/src/DIRAC/Core/Utilities/test/Test_List.py @@ -13,6 +13,7 @@ # sut from DIRAC.Core.Utilities import List + ######################################################################## class ListTestCase(unittest.TestCase): """py:class ListTestCase diff --git a/src/DIRAC/Core/Utilities/test/Test_ProcessPool.py b/src/DIRAC/Core/Utilities/test/Test_ProcessPool.py index e340b4df32e..176c941a618 100644 --- a/src/DIRAC/Core/Utilities/test/Test_ProcessPool.py +++ b/src/DIRAC/Core/Utilities/test/Test_ProcessPool.py @@ -53,7 +53,7 @@ class CallableClass: """callable class to be executed in task""" def __init__(self, taskID, timeWait, raiseException=False): - self.log = gLogger.getSubLogger(self.__class__.__name__ + "/%s" % taskID) + self.log = gLogger.getSubLogger(self.__class__.__name__ + f"/{taskID}") self.taskID = taskID self.timeWait = timeWait self.raiseException = raiseException @@ -77,7 +77,7 @@ class LockedCallableClass: """callable and locked class""" def __init__(self, taskID, timeWait, raiseException=False): - self.log = gLogger.getSubLogger(self.__class__.__name__ + "/%s" % taskID) + self.log = gLogger.getSubLogger(self.__class__.__name__ + f"/{taskID}") self.taskID = taskID self.log.always(f"pid={os.getpid()} task={self.taskID} I'm locked") gLock.acquire() @@ -88,7 +88,7 @@ def __init__(self, taskID, timeWait, raiseException=False): def __call__(self): self.log.always("If you see this line, miracle had happened!") - self.log.always("will sleep for %s" % self.timeWait) + self.log.always(f"will sleep for {self.timeWait}") time.sleep(self.timeWait) if self.raiseException: raise Exception("testException") @@ -123,7 +123,7 @@ def test_TaskCallbacks_CallableClass(processPool): blocking=True, ) if result["OK"]: - print("CallableClass enqueued to task %s" % i) + print(f"CallableClass enqueued to task {i}") i += 1 else: continue @@ -150,7 +150,7 @@ def test_TaskCallbacks_CallableFunc(processPool): blocking=True, ) if result["OK"]: - print("CallableClass enqueued to task %s" % i) + print(f"CallableClass enqueued to task {i}") i += 1 else: continue @@ -194,7 +194,7 @@ def test_ProcessPoolCallbacks_CallableClass(processPoolWithCallbacks): blocking=True, ) if result["OK"]: - print("CallableClass enqueued to task %s" % i) + print(f"CallableClass enqueued to task {i}") i += 1 else: continue @@ -220,7 +220,7 @@ def test_ProcessPoolCallbacks_CallableFunc(processPoolWithCallbacks): blocking=True, ) if result["OK"]: - print("CallableFunc enqueued to task %s" % i) + print(f"CallableFunc enqueued to task {i}") i += 1 else: continue @@ -304,7 +304,7 @@ def test_TaskTimeOut_CallableFunc(processPoolWithCallbacks2): def test_TaskTimeOut_LockedClass(processPoolWithCallbacks2): """LockedCallableClass and task time out test""" for loop in range(2): - print("loop %s" % loop) + print(f"loop {loop}") i = 0 while i < 16: if processPoolWithCallbacks2.getFreeSlots() > 0: diff --git a/src/DIRAC/Core/Utilities/test/Test_Profiler.py b/src/DIRAC/Core/Utilities/test/Test_Profiler.py index 9e8a2f90612..aea819e70e4 100644 --- a/src/DIRAC/Core/Utilities/test/Test_Profiler.py +++ b/src/DIRAC/Core/Utilities/test/Test_Profiler.py @@ -5,7 +5,6 @@ from subprocess import Popen import pytest -from flaky import flaky import DIRAC from DIRAC.Core.Utilities.Profiler import Profiler @@ -78,7 +77,7 @@ def test_base(): assert resWC["Value"] >= res["Value"] -@flaky(max_runs=10, min_passes=2) +@pytest.mark.flaky(reruns=10) def test_cpuUsage(): mainProcess = Popen( [ diff --git a/src/DIRAC/Core/Utilities/test/Test_ReturnValues.py b/src/DIRAC/Core/Utilities/test/Test_ReturnValues.py index de086370ef1..40415b6dd39 100644 --- a/src/DIRAC/Core/Utilities/test/Test_ReturnValues.py +++ b/src/DIRAC/Core/Utilities/test/Test_ReturnValues.py @@ -1,6 +1,6 @@ import pytest -from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR, convertToReturnValue, returnValueOrRaise +from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR, SErrorException, convertToReturnValue, returnValueOrRaise def test_Ok(): @@ -40,6 +40,16 @@ def _sadFunction(): return {} +@convertToReturnValue +def _verySadFunction(): + raise SErrorException("I am very sad") + + +@convertToReturnValue +def _sadButPreciseFunction(): + raise SErrorException("I am sad, yet precise", errCode=123) + + def test_convertToReturnValue(): retVal = _happyFunction() assert retVal["OK"] is True @@ -51,3 +61,13 @@ def test_convertToReturnValue(): # Make sure the exception is re-raised with pytest.raises(CustomException): returnValueOrRaise(_sadFunction()) + + retVal = _verySadFunction() + assert retVal["OK"] is False + assert retVal["Errno"] == 0 + assert retVal["Message"] == "I am very sad" + + retVal = _sadButPreciseFunction() + assert retVal["OK"] is False + assert retVal["Errno"] == 123 + assert "I am sad, yet precise" in retVal["Message"] diff --git a/src/DIRAC/Core/Utilities/test/Test_Time.py b/src/DIRAC/Core/Utilities/test/Test_Time.py index 1ee9494d655..e4885e63183 100755 --- a/src/DIRAC/Core/Utilities/test/Test_Time.py +++ b/src/DIRAC/Core/Utilities/test/Test_Time.py @@ -59,7 +59,6 @@ def tearDown(self): class TimeSuccess(TimeTestCase): def test_timeThis(self): - self.assertIsNone(myMethod()) self.assertIsNone(myClass().myMethodInAClass()) self.assertIsNone(myBetterClass().myMethodInAClass()) diff --git a/src/DIRAC/Core/Utilities/test/Test_entrypoints.py b/src/DIRAC/Core/Utilities/test/Test_entrypoints.py new file mode 100644 index 00000000000..9341d7928fe --- /dev/null +++ b/src/DIRAC/Core/Utilities/test/Test_entrypoints.py @@ -0,0 +1,13 @@ +import importlib_metadata as metadata + + +def test_entrypoints(): + """Make sure all console_scripts defined by DIRAC are importable.""" + errors = [] + for ep in metadata.entry_points(group="console_scripts"): + if ep.module.startswith("DIRAC"): + try: + ep.load() + except ModuleNotFoundError as e: + errors.append(str(e)) + assert not errors, errors diff --git a/src/DIRAC/Core/Utilities/test/Test_gCFG.py b/src/DIRAC/Core/Utilities/test/Test_gCFG.py deleted file mode 100755 index 1269085eabd..00000000000 --- a/src/DIRAC/Core/Utilities/test/Test_gCFG.py +++ /dev/null @@ -1,57 +0,0 @@ -# FIXME: should be rewritten as real unittest -# import DIRAC -# from DIRAC.Core.Utilities.CFG import CFG -# -# DIRAC.gLogger.initialize('test_gConfig','/testSectionDebug') -# -# testconfig = '%s/DIRAC/ConfigurationSystem/test/test.cfg' % DIRAC.rootPath -# dumpconfig = '%s/DIRAC/ConfigurationSystem/test/dump.cfg' % DIRAC.rootPath -# -# cfg1 = CFG() -# cfg1.loadFromFile( testconfig ) -# -# with open( testconfig ) as fd: -# cfg1String = fd.read() -# -# cfg2 = CFG() -# cfg2.loadFromBuffer( cfg1.serialize() ) -# -# cfg3 = cfg1.mergeWith( cfg2 ) -# -# testList = [{ 'method' : DIRAC.gConfig.loadFile, -# 'arguments' : ( testconfig, ), -# 'output' : {'OK': True, 'Value': ''} -# }, -# { 'method' : DIRAC.gConfig.dumpLocalCFGToFile, -# 'arguments' : ( dumpconfig, ), -# 'output' : {'OK': True, 'Value': ''} -# }, -# { 'method' : cfg1.serialize, -# 'arguments' : ( ), -# 'output' : cfg1String -# }, -# { 'method' : cfg3.serialize, -# 'arguments' : ( ), -# 'output' : cfg1String -# }] -# -# testdict = { 'DIRAC.gConfig' : testList,} -# -# DIRAC.Tests.run( testdict, 'DIRAC.gConfig.files' ) -# -# -# -# # testList = [{ 'method' : DIRAC.gConfig.get, -# # 'arguments' : ( '/testSection/test', ), -# # 'output' : {'OK': True, 'Value': 'test'} -# # }, -# # # { 'method' : DIRAC.gConfig.get, -# # # 'arguments' : ( '/testSection/nonexisting','OK', ), -# # # 'output' : {'OK': True, 'Value': 'test'} -# # # }, -# # ] -# # -# # testdict = { 'DIRAC.gConfig' : testList,} -# # -# # -# # DIRAC.Tests.run( testdict, 'DIRAC.gConfig' ) diff --git a/src/DIRAC/Core/Workflow/Parameter.py b/src/DIRAC/Core/Workflow/Parameter.py index cdc711cdbb7..a2ec064a160 100755 --- a/src/DIRAC/Core/Workflow/Parameter.py +++ b/src/DIRAC/Core/Workflow/Parameter.py @@ -436,13 +436,13 @@ def linkUp(self, opt, prefix="", postfix="", objname="self"): for s in opt: par = self.find(s) if par is None: - print("ERROR ParameterCollection.linkUp() can not find parameter with the name=%s" % (s)) + print(f"ERROR ParameterCollection.linkUp() can not find parameter with the name={s}") else: - par.link(objname, prefix + p.getName() + postfix) + par.link(objname, prefix + par.getName() + postfix) elif isinstance(opt, str): par = self.find(opt) if par is None: - print("ERROR ParameterCollection.linkUp() can not find parameter with the name=%s" % (par)) + print(f"ERROR ParameterCollection.linkUp() can not find parameter with the name={par}") else: par.link(objname, prefix + par.getName() + postfix) else: @@ -474,13 +474,13 @@ def unlink(self, opt): for s in opt: par = self.find(s) if par is None: - print("ERROR ParameterCollection.unlink() can not find parameter with the name=%s" % (s)) + print(f"ERROR ParameterCollection.unlink() can not find parameter with the name={s}") else: par.unlink() elif isinstance(opt, str): par = self.find(opt) if par is None: - print("ERROR ParameterCollection.unlink() can not find parameter with the name=%s" % (s)) + print(f"ERROR ParameterCollection.unlink() can not find parameter with the name={opt}") else: par.unlink() else: @@ -498,7 +498,7 @@ def remove(self, name_or_ind): for s in name_or_ind: par = self.find(s) if par is None: - print("ERROR ParameterCollection.remove() can not find parameter with the name=%s" % (s)) + print(f"ERROR ParameterCollection.remove() can not find parameter with the name={s}") else: index = self.findIndex(s) if index > -1: @@ -580,7 +580,6 @@ def resolveGlobalVars(self, wf_parameters=None, step_parameters=None): substitute_vars = getSubstitute(v.value) while True: for substitute_var in substitute_vars: - # looking in the current scope v_other = self.find(substitute_var) @@ -596,7 +595,7 @@ def resolveGlobalVars(self, wf_parameters=None, step_parameters=None): if v_other is not None and not v_other.isLinked(): v.value = substitute(v.value, substitute_var, v_other.value) elif v_other is not None: - print("Leaving %s variable for dynamic resolution" % substitute_var) + print(f"Leaving {substitute_var} variable for dynamic resolution") skip_list.append(substitute_var) else: # if nothing helped tough! print("Can not resolve ", substitute_var, str(v)) diff --git a/src/DIRAC/Core/Workflow/Step.py b/src/DIRAC/Core/Workflow/Step.py index da545370c00..fab15b24a0a 100755 --- a/src/DIRAC/Core/Workflow/Step.py +++ b/src/DIRAC/Core/Workflow/Step.py @@ -158,7 +158,7 @@ def resolveGlobalVars(self, step_definitions, wf_parameters): inst.parameters.append( Parameter( "MODULE_NUMBER", - "%s" % module_instance_number, + f"{module_instance_number}", "string", "", "", @@ -292,7 +292,6 @@ def execute(self, step_exec_attr, definitions): step_exec_attr[parameter.getLinkedParameter()], ) else: - setattr( step_exec_modules[mod_inst_name], parameter.getName(), @@ -391,7 +390,7 @@ def execute(self, step_exec_attr, definitions): if "JobReport" in self.workflow_commons: if self.parent.workflowStatus["OK"]: self.workflow_commons["JobReport"].setApplicationStatus( - "Exception in %s module" % mod_inst_name + f"Exception in {mod_inst_name} module" ) self.stepStatus = S_ERROR(error_code, error_message) diff --git a/src/DIRAC/Core/Workflow/Utility.py b/src/DIRAC/Core/Workflow/Utility.py index 83473738c6e..806310b41c2 100644 --- a/src/DIRAC/Core/Workflow/Utility.py +++ b/src/DIRAC/Core/Workflow/Utility.py @@ -48,11 +48,9 @@ def resolveVariables(varDict): def dataFromOption(parameter): - result = [] if parameter.type.lower() == "option": - fields = parameter.value.split(",") for f in fields: @@ -70,7 +68,6 @@ def dataFromOption(parameter): def expandDatafileOption(option): - result = "" if not re.search(";;", option.value): @@ -81,7 +78,6 @@ def expandDatafileOption(option): fname, ftype = files[0] fnames = fname.split(";;") if len(fnames) > 1: - template = option.value.strip().replace("=", "", 1) template = template.replace("{", "") template = template.replace("}", "") diff --git a/src/DIRAC/Core/Workflow/Workflow.py b/src/DIRAC/Core/Workflow/Workflow.py index 65495613311..3a3f6bffd14 100755 --- a/src/DIRAC/Core/Workflow/Workflow.py +++ b/src/DIRAC/Core/Workflow/Workflow.py @@ -176,7 +176,7 @@ def resolveGlobalVars(self): inst.parameters.append( Parameter( "STEP_NUMBER", - "%s" % step_instance_number, + f"{step_instance_number}", "string", "", "", @@ -396,10 +396,10 @@ def execute(self): return S_OK(step_result) -from DIRAC.Core.Workflow.WorkflowReader import WorkflowXMLHandler - - def fromXMLString(xml_string, obj=None): + # prevent circular import in WorkflowReader + from DIRAC.Core.Workflow.WorkflowReader import WorkflowXMLHandler + # KGG !!! We need to reset Workflow if it exists handler = WorkflowXMLHandler(obj) xml.sax.parseString(xml_string, handler) @@ -407,6 +407,9 @@ def fromXMLString(xml_string, obj=None): def fromXMLFile(xml_file, obj=None): + # prevent circular import in WorkflowReader + from DIRAC.Core.Workflow.WorkflowReader import WorkflowXMLHandler + # KGG !!! We need to reset Workflow if it exists handler = WorkflowXMLHandler(obj) xml.sax.parse(xml_file, handler) diff --git a/src/DIRAC/Core/scripts/dirac_agent.py b/src/DIRAC/Core/scripts/dirac_agent.py index e0115af6dae..0eed6f67819 100755 --- a/src/DIRAC/Core/scripts/dirac_agent.py +++ b/src/DIRAC/Core/scripts/dirac_agent.py @@ -23,9 +23,7 @@ def main(): agentName = positionalArgs[0] localCfg.setConfigurationForAgent(agentName) - localCfg.addMandatoryEntry("/DIRAC/Setup") localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") - localCfg.addDefaultEntry("LogLevel", "INFO") localCfg.addDefaultEntry("LogColor", True) resultDict = localCfg.loadUserData() if not resultDict["OK"]: diff --git a/src/DIRAC/Core/scripts/dirac_apptainer_exec.py b/src/DIRAC/Core/scripts/dirac_apptainer_exec.py new file mode 100644 index 00000000000..0cb6ca567a9 --- /dev/null +++ b/src/DIRAC/Core/scripts/dirac_apptainer_exec.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python +""" Starts a DIRAC command inside an apptainer container. +""" + +import os +import sys +from pathlib import Path + +import DIRAC +from DIRAC import S_ERROR, gConfig, gLogger +from DIRAC.Core.Base.Script import Script +from DIRAC.Core.Security.Locations import getCAsLocation, getProxyLocation, getVOMSLocation +from DIRAC.Core.Utilities.Subprocess import systemCall + + +def generate_container_wrapper(dirac_env_var, diracos_env_var, etc_dir, rc_script, command, include_proxy=True): + lines = [ + "#!/bin/bash", + f"export DIRAC={dirac_env_var}", + f"export DIRACOS={diracos_env_var}", + ] + + if include_proxy: + lines.append("export X509_USER_PROXY=/etc/proxy") + + lines.extend( + [ + "export X509_CERT_DIR=/etc/grid-security/certificates", + "export X509_VOMS_DIR=/etc/grid-security/vomsdir", + "export X509_VOMSES=/etc/grid-security/vomses", + f"export DIRACSYSCONFIG={etc_dir}/dirac.cfg", + f"source {rc_script}", + command, + ] + ) + + return "\n".join(lines) + + +CONTAINER_DEFROOT = "" # Should add something like "/cvmfs/dirac.egi.eu/container/apptainer/alma9/x86_64" + + +@Script() +def main(): + command = sys.argv[1] + + user_image = None + Script.registerSwitch("i:", "image=", " apptainer image to use") + Script.parseCommandLine(ignoreErrors=False) + for switch in Script.getUnprocessedSwitches(): + if switch[0].lower() == "i" or switch[0].lower() == "image": + user_image = switch[1] + + dirac_env_var = os.environ.get("DIRAC", os.getcwd()) + diracos_env_var = os.environ.get("DIRACOS", os.getcwd()) + etc_dir = os.path.join(DIRAC.rootPath, "etc") + rc_script = os.path.join(os.path.realpath(sys.base_prefix), "diracosrc") + + include_proxy = True + proxy_location = getProxyLocation() + if not proxy_location: + include_proxy = False + + with open("dirac_container.sh", "w", encoding="utf-8") as fd: + script = generate_container_wrapper( + dirac_env_var, diracos_env_var, etc_dir, rc_script, command, include_proxy=include_proxy + ) + fd.write(script) + os.chmod("dirac_container.sh", 0o755) + + # Now let's construct the apptainer command + cmd = ["apptainer", "exec"] + cmd.extend(["--contain"]) # use minimal /dev and empty other directories (e.g. /tmp and $HOME) + cmd.extend(["--ipc"]) # run container in a new IPC namespace + cmd.extend(["--pid"]) # run container in a new PID namespace + cmd.extend(["--bind", f"{os.getcwd()}:/mnt"]) # bind current directory for dirac_container.sh + if proxy_location: + cmd.extend(["--bind", f"{proxy_location}:/etc/proxy"]) # bind proxy file + cmd.extend(["--bind", f"{getCAsLocation()}:/etc/grid-security/certificates"]) # X509_CERT_DIR + voms_location = Path(getVOMSLocation()) + cmd.extend(["--bind", f"{voms_location}:/etc/grid-security/vomsdir"]) # X509_VOMS_DIR + vomses_location = voms_location.parent / "vomses" + cmd.extend(["--bind", f"{vomses_location}:/etc/grid-security/vomses"]) # X509_VOMSES + cmd.extend(["--bind", "{0}:{0}:ro".format(etc_dir)]) # etc dir for dirac.cfg + cmd.extend(["--bind", "{0}:{0}:ro".format(os.path.join(os.path.realpath(sys.base_prefix)))]) # code dir + + rootImage = user_image or gConfig.getValue("/Resources/Computing/Singularity/ContainerRoot") or CONTAINER_DEFROOT + + if os.path.isdir(rootImage) or os.path.isfile(rootImage): + cmd.extend([rootImage, "/mnt/dirac_container.sh"]) + else: + # if we are here is because there's no image, or it is not accessible (e.g. not on CVMFS) + gLogger.error("Apptainer image to exec not found: ", rootImage) + return S_ERROR("Failed to find Apptainer image to exec") + + gLogger.debug(f"Execute Apptainer command: {' '.join(cmd)}") + result = systemCall(0, cmd) + if not result["OK"]: + gLogger.error(result["Message"]) + DIRAC.exit(1) + if result["Value"][0] != 0: + gLogger.error(result["Value"][2]) + DIRAC.exit(2) + gLogger.notice(result["Value"][1]) + + +if __name__ == "__main__": + main() diff --git a/src/DIRAC/Core/scripts/dirac_cert_convert.py b/src/DIRAC/Core/scripts/dirac_cert_convert.py index 7094f3ce53b..61314fd751a 100644 --- a/src/DIRAC/Core/scripts/dirac_cert_convert.py +++ b/src/DIRAC/Core/scripts/dirac_cert_convert.py @@ -6,10 +6,12 @@ import os import sys import shutil +import subprocess from datetime import datetime +from subprocess import PIPE, run, STDOUT +from tempfile import TemporaryDirectory from DIRAC import gLogger -from DIRAC.Core.Utilities.Subprocess import shellCall from DIRAC.Core.Base.Script import Script @@ -20,7 +22,7 @@ def main(): p12 = args[0] if not os.path.isfile(p12): - gLogger.fatal("%s does not exist." % p12) + gLogger.fatal(f"{p12} does not exist.") sys.exit(1) globus = os.path.join(os.environ["HOME"], ".globus") @@ -38,15 +40,19 @@ def main(): shutil.move(old, old + nowPrefix) # new OpenSSL version require OPENSSL_CONF to point to some accessible location', - gLogger.notice("Converting p12 key to pem format") - result = shellCall(900, f"export OPENSSL_CONF=/tmp && openssl pkcs12 -nocerts -in {p12} -out {key}") - # The last command was successful - if result["OK"] and result["Value"][0] == 0: - gLogger.notice("Converting p12 certificate to pem format") - result = shellCall(900, f"export OPENSSL_CONF=/tmp && openssl pkcs12 -clcerts -nokeys -in {p12} -out {cert}") + with TemporaryDirectory() as tmpdir: + env = os.environ | {"OPENSSL_CONF": tmpdir} + gLogger.notice("Converting p12 key to pem format") + cmd = ["openssl", "pkcs12", "-nocerts", "-in", p12, "-out", key] + res = run(cmd, env=env, check=False, timeout=900, text=True, stdout=PIPE, stderr=STDOUT) + # The last command was successful + if res.returncode == 0: + gLogger.notice("Converting p12 certificate to pem format") + cmd = ["openssl", "pkcs12", "-clcerts", "-nokeys", "-in", p12, "-out", cert] + res = run(cmd, env=env, check=False, timeout=900, text=True, stdout=PIPE, stderr=STDOUT) # Something went wrong - if not result["OK"] or result["Value"][0] != 0: - gLogger.fatal(result.get("Message", result["Value"][2])) + if res.returncode != 0: + gLogger.fatal(res.stdout) for old in [cert, key]: if os.path.isfile(old + nowPrefix): gLogger.notice(f"Restore {old} file from the {old + nowPrefix}") diff --git a/src/DIRAC/Core/scripts/dirac_configure.py b/src/DIRAC/Core/scripts/dirac_configure.py index f64477ad190..cb3f55a6e7d 100755 --- a/src/DIRAC/Core/scripts/dirac_configure.py +++ b/src/DIRAC/Core/scripts/dirac_configure.py @@ -41,17 +41,17 @@ --SkipCAChecks """ -import sys import os +import sys import warnings import urllib3 import DIRAC -from DIRAC.Core.Utilities.File import mkDir +from DIRAC.ConfigurationSystem.Client.Helpers import Registry, cfgInstallPath, cfgPath from DIRAC.Core.Base.Script import Script from DIRAC.Core.Security.ProxyInfo import getProxyInfo -from DIRAC.ConfigurationSystem.Client.Helpers import cfgInstallPath, cfgPath, Registry +from DIRAC.Core.Utilities.File import mkDir from DIRAC.Core.Utilities.SiteSEMapping import getSEsForSite from DIRAC.FrameworkSystem.Client.BundleDeliveryClient import BundleDeliveryClient @@ -77,6 +77,7 @@ def __init__(self): self.outputFile = "" self.skipVOMSDownload = False self.extensions = "" + self.legacyExchangeApiKey = "" def setGateway(self, optionValue): self.gatewayServer = optionValue @@ -98,9 +99,7 @@ def setAllServers(self, optionValue): self.includeAllServers = True def setSetup(self, optionValue): - self.setup = optionValue - DIRAC.gConfig.setOptionValue("/DIRAC/Setup", self.setup) - DIRAC.gConfig.setOptionValue(cfgInstallPath("Setup"), self.setup) + DIRAC.gLogger.warn("Ignoring Setup option as it is not used for DIRAC v9.0+") return DIRAC.S_OK() def setSiteName(self, optionValue): @@ -175,10 +174,22 @@ def setIssuer(self, optionValue): DIRAC.gConfig.setOptionValue("/DIRAC/Security/Authorization/issuer", self.issuer) return DIRAC.S_OK() + def setLegacyExchangeApiKey(self, optionValue): + self.legacyExchangeApiKey = optionValue + Script.localCfg.addDefaultEntry("/DiracX/LegacyExchangeApiKey", self.legacyExchangeApiKey) + DIRAC.gConfig.setOptionValue(cfgInstallPath("LegacyExchangeApiKey"), self.legacyExchangeApiKey) + return DIRAC.S_OK() + + def setDiracxUrl(self, optionValue): + self.diracxUrl = optionValue + Script.localCfg.addDefaultEntry("/DiracX/URL", self.diracxUrl) + DIRAC.gConfig.setOptionValue(cfgInstallPath("URL"), self.diracxUrl) + return DIRAC.S_OK() + def _runConfigurationWizard(setups, defaultSetup): """The implementation of the configuration wizard""" - from prompt_toolkit import prompt, print_formatted_text, HTML + from prompt_toolkit import HTML, print_formatted_text, prompt from prompt_toolkit.completion import FuzzyWordCompleter # It makes no sense to have suggestions if there is no default so adjust the message accordingly @@ -187,21 +198,21 @@ def _runConfigurationWizard(setups, defaultSetup): msg = "press tab for suggestions" if defaultSetup: msg = f"default {defaultSetup}, {msg}" - msg = " (%s)" % msg + msg = f" ({msg})" # Get the Setup setup = prompt( - HTML("Choose a DIRAC Setup%s:\n" % msg), + HTML(f"Choose a DIRAC Setup{msg}:\n"), completer=FuzzyWordCompleter(list(setups)), ) if defaultSetup and not setup: setup = defaultSetup if setup not in setups: - print_formatted_text(HTML("Unknown setup %s chosen" % setup)) + print_formatted_text(HTML(f"Unknown setup {setup} chosen")) confirm = prompt(HTML("Are you sure you want to continue? "), default="n") if confirm.lower() not in ["y", "yes"]: return None - # Get the URL to the master CS + # Get the URL to the controller CS csURL = prompt(HTML("Choose a configuration server URL (leave blank for default):\n")) if not csURL: csURL = setups[setup] @@ -210,8 +221,8 @@ def _runConfigurationWizard(setups, defaultSetup): print_formatted_text( HTML( "Configuration is:\n" - + " * Setup: %s\n" % setup - + " * Configuration server: %s\n" % csURL + + f" * Setup: {setup}\n" + + f" * Configuration server: {csURL}\n" ) ) confirm = prompt(HTML("Are you sure you want to continue? "), default="y") @@ -222,7 +233,9 @@ def _runConfigurationWizard(setups, defaultSetup): def runConfigurationWizard(params): """Interactively configure DIRAC using metadata from installed extensions""" import subprocess - from prompt_toolkit import prompt, print_formatted_text, HTML + + from prompt_toolkit import HTML, print_formatted_text, prompt + from DIRAC.Core.Utilities.Extensions import extensionsByPriority, getExtensionMetadata for extension in extensionsByPriority(): @@ -289,13 +302,14 @@ def main(): def login(params): - from prompt_toolkit import prompt, print_formatted_text, HTML - from DIRAC.Resources.IdProvider.IdProviderFactory import IdProviderFactory + from prompt_toolkit import HTML, print_formatted_text, prompt + from DIRAC.FrameworkSystem.private.authorization.utils.Tokens import ( - writeTokenDictToTokenFile, getTokenFileLocation, readTokenFromFile, + writeTokenDictToTokenFile, ) + from DIRAC.Resources.IdProvider.IdProviderFactory import IdProviderFactory # Init authorization client result = IdProviderFactory().getIdProvider("DIRACCLI", issuer=params.issuer, scope=" ") @@ -319,7 +333,7 @@ def login(params): for tokenType in ["access_token", "refresh_token"]: result = idpObj.revokeToken(oldToken[tokenType], tokenType) if result["OK"]: - DIRAC.gLogger.debug("%s is revoked from" % tokenType, tokenFile) + DIRAC.gLogger.debug(f"{tokenType} is revoked from", tokenFile) else: DIRAC.gLogger.warn(result["Message"]) @@ -328,7 +342,7 @@ def login(params): return result DIRAC.gLogger.debug(f"New token is saved to {result['Value']}.") - # Get server setups and master CS server URL + # Get server setups and controller CS server URL csURL = idpObj.get_metadata("configuration_server") setups = idpObj.get_metadata("setups") @@ -359,8 +373,12 @@ def runDiracConfigure(params): Script.registerSwitch("C:", "ConfigurationServer=", "Set as DIRAC configuration server", params.setServer) Script.registerSwitch("I", "IncludeAllServers", "include all Configuration Servers", params.setAllServers) Script.registerSwitch("n:", "SiteName=", "Set as DIRAC Site Name", params.setSiteName) - Script.registerSwitch("N:", "CEName=", "Determiner from ", params.setCEName) + Script.registerSwitch("N:", "CEName=", "Set as Computing Element name", params.setCEName) Script.registerSwitch("V:", "VO=", "Set the VO name", params.setVO) + Script.registerSwitch( + "K:", "LegacyExchangeApiKey=", "Set the Api Key to talk to DiracX", params.setLegacyExchangeApiKey + ) + Script.registerSwitch("", "DiracxUrl=", "Set the URL to talk to DiracX", params.setDiracxUrl) Script.registerSwitch("W:", "gateway=", "Configure as DIRAC Gateway for the site", params.setGateway) @@ -389,7 +407,7 @@ def runDiracConfigure(params): if params.issuer: result = login(params) if not result["OK"]: - DIRAC.gLogger.error("Authorization failed: %s" % result["Message"]) + DIRAC.gLogger.error(f"Authorization failed: {result['Message']}") DIRAC.exit(1) useTokens = result["Value"] if useTokens: @@ -464,17 +482,15 @@ def runDiracConfigure(params): if newExtensions: params.setExtensions(newExtensions) - DIRAC.gLogger.notice("Executing: %s " % (" ".join(sys.argv))) - DIRAC.gLogger.notice('Checking DIRAC installation at "%s"' % DIRAC.rootPath) + DIRAC.gLogger.notice(f"Executing: {' '.join(sys.argv)} ") + DIRAC.gLogger.notice(f'Checking DIRAC installation at "{DIRAC.rootPath}"') if params.update: if params.outputFile: - DIRAC.gLogger.notice("Will update the output file %s" % params.outputFile) + DIRAC.gLogger.notice(f"Will update the output file {params.outputFile}") else: - DIRAC.gLogger.notice("Will update %s" % DIRAC.gConfig.diracConfigFilePath) + DIRAC.gLogger.notice(f"Will update {DIRAC.gConfig.diracConfigFilePath}") - if params.setup: - DIRAC.gLogger.verbose("/DIRAC/Setup =", params.setup) if params.vo: DIRAC.gLogger.verbose("/DIRAC/VirtualOrganization =", params.vo) if params.configurationServer: @@ -517,7 +533,7 @@ def runDiracConfigure(params): Script.enableCS() try: dirName = os.path.join(DIRAC.rootPath, "etc", "grid-security", "certificates") - mkDir(dirName) + mkDir(dirName, 0o755) except Exception: DIRAC.gLogger.exception() DIRAC.gLogger.fatal("Fail to create directory:", dirName) @@ -527,8 +543,10 @@ def runDiracConfigure(params): result = bdc.syncCAs() if result["OK"]: result = bdc.syncCRLs() + if not result["OK"]: + DIRAC.gLogger.error("Failed to sync CAs and/or CRLs", result["Message"]) except Exception as e: - DIRAC.gLogger.error("Failed to sync CAs and CRLs: %s" % str(e)) + DIRAC.gLogger.error(f"Failed to sync CAs and CRLs: {str(e)}") Script.localCfg.deleteOption("/DIRAC/Security/SkipCAChecks") @@ -548,9 +566,9 @@ def runDiracConfigure(params): grids = gridSections["Value"] # try to get siteName from ceName or Local SE from siteName using Remote Configuration for grid in grids: - siteSections = DIRAC.gConfig.getSections("/Resources/Sites/%s/" % grid) + siteSections = DIRAC.gConfig.getSections(f"/Resources/Sites/{grid}/") if not siteSections["OK"]: - DIRAC.gLogger.warn("Could not get %s site list" % grid) + DIRAC.gLogger.warn(f"Could not get {grid} site list") sites = [] else: sites = siteSections["Value"] @@ -560,16 +578,16 @@ def runDiracConfigure(params): for site in sites: res = DIRAC.gConfig.getSections(f"/Resources/Sites/{grid}/{site}/CEs/", []) if not res["OK"]: - DIRAC.gLogger.warn("Could not get %s CEs list" % site) + DIRAC.gLogger.warn(f"Could not get {site} CEs list") if params.ceName in res["Value"]: params.siteName = site break if params.siteName: - DIRAC.gLogger.notice("Setting /LocalSite/Site = %s" % params.siteName) + DIRAC.gLogger.notice(f"Setting /LocalSite/Site = {params.siteName}") Script.localCfg.addDefaultEntry("/LocalSite/Site", params.siteName) DIRAC.__siteName = False if params.ceName: - DIRAC.gLogger.notice("Setting /LocalSite/GridCE = %s" % params.ceName) + DIRAC.gLogger.notice(f"Setting /LocalSite/GridCE = {params.ceName}") Script.localCfg.addDefaultEntry("/LocalSite/GridCE", params.ceName) if not params.localSE and params.siteName in sites: @@ -581,8 +599,8 @@ def runDiracConfigure(params): break if params.gatewayServer: - DIRAC.gLogger.verbose("/DIRAC/Gateways/%s =" % DIRAC.siteName(), params.gatewayServer) - Script.localCfg.addDefaultEntry("/DIRAC/Gateways/%s" % DIRAC.siteName(), params.gatewayServer) + DIRAC.gLogger.verbose(f"/DIRAC/Gateways/{DIRAC.siteName()} =", params.gatewayServer) + Script.localCfg.addDefaultEntry(f"/DIRAC/Gateways/{DIRAC.siteName()}", params.gatewayServer) # Create the local cfg if it is not yet there if not params.outputFile: @@ -593,6 +611,8 @@ def runDiracConfigure(params): mkDir(configDir) params.update = True DIRAC.gConfig.dumpLocalCFGToFile(params.outputFile) + elif not params.forceUpdate: + DIRAC.gLogger.notice(f"{params.outputFile} exists, not overwriting it. Or use the '--ForceUpdate' Flag") if params.includeAllServers: # We need user proxy or server certificate to continue in order to get all the CS URLs @@ -640,35 +660,35 @@ def runDiracConfigure(params): vomsDirPath = os.path.join(DIRAC.rootPath, "etc", "grid-security", "vomsdir", voName) vomsesDirPath = os.path.join(DIRAC.rootPath, "etc", "grid-security", "vomses") for path in (vomsDirPath, vomsesDirPath): - mkDir(path) + mkDir(path, 0o755) vomsesLines = [] for vomsHost in vomsDict[vo].get("Servers", {}): - hostFilePath = os.path.join(vomsDirPath, "%s.lsc" % vomsHost) + hostFilePath = os.path.join(vomsDirPath, f"{vomsHost}.lsc") try: DN = vomsDict[vo]["Servers"][vomsHost]["DN"] CA = vomsDict[vo]["Servers"][vomsHost]["CA"] port = vomsDict[vo]["Servers"][vomsHost]["Port"] if not DN or not CA or not port: - DIRAC.gLogger.error("DN = %s" % DN) - DIRAC.gLogger.error("CA = %s" % CA) - DIRAC.gLogger.error("Port = %s" % port) - DIRAC.gLogger.error("Missing Parameter for %s" % vomsHost) + DIRAC.gLogger.error(f"DN = {DN}") + DIRAC.gLogger.error(f"CA = {CA}") + DIRAC.gLogger.error(f"Port = {port}") + DIRAC.gLogger.error(f"Missing Parameter for {vomsHost}") continue - with open(hostFilePath, "wt") as fd: + with open(hostFilePath, "w") as fd: fd.write(f"{DN}\n{CA}\n") vomsesLines.append(f'"{voName}" "{vomsHost}" "{port}" "{DN}" "{voName}" "24"') - DIRAC.gLogger.notice("Created vomsdir file %s" % hostFilePath) + DIRAC.gLogger.notice(f"Created vomsdir file {hostFilePath}") except Exception: DIRAC.gLogger.exception("Could not generate vomsdir file for host", vomsHost) error = f"Could not generate vomsdir file for VO {voName}, host {vomsHost}" try: vomsesFilePath = os.path.join(vomsesDirPath, voName) - with open(vomsesFilePath, "wt") as fd: + with open(vomsesFilePath, "w") as fd: fd.write("%s\n" % "\n".join(vomsesLines)) - DIRAC.gLogger.notice("Created vomses file %s" % vomsesFilePath) + DIRAC.gLogger.notice(f"Created vomses file {vomsesFilePath}") except Exception: DIRAC.gLogger.exception("Could not generate vomses file") - error = "Could not generate vomses file for VO %s" % voName + error = f"Could not generate vomses file for VO {voName}" if params.useServerCert: Script.localCfg.deleteOption("/DIRAC/Security/UseServerCertificate") diff --git a/src/DIRAC/Core/scripts/dirac_executor.py b/src/DIRAC/Core/scripts/dirac_executor.py index a54a7ab87b8..8040cf2744d 100755 --- a/src/DIRAC/Core/scripts/dirac_executor.py +++ b/src/DIRAC/Core/scripts/dirac_executor.py @@ -27,9 +27,7 @@ def main(): mainName = "Framework/MultiExecutor" localCfg.setConfigurationForExecutor(mainName) - localCfg.addMandatoryEntry("/DIRAC/Setup") localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") - localCfg.addDefaultEntry("LogLevel", "INFO") localCfg.addDefaultEntry("LogColor", True) resultDict = localCfg.loadUserData() if not resultDict["OK"]: diff --git a/src/DIRAC/Core/scripts/dirac_info.py b/src/DIRAC/Core/scripts/dirac_info.py index 52b271c8659..8a9166872e7 100755 --- a/src/DIRAC/Core/scripts/dirac_info.py +++ b/src/DIRAC/Core/scripts/dirac_info.py @@ -11,7 +11,6 @@ Option Value ============================ - Setup Dirac-Production ConfigurationServer dips://ccdiracli08.in2p3.fr:9135/Configuration/Server Installation path /opt/dirac/versions/v7r2-pre33_1613239204 Installation type client @@ -52,7 +51,6 @@ def platform(arg): records = [] - records.append(("Setup", gConfig.getValue("/DIRAC/Setup", "Unknown"))) records.append( ( "AuthorizationServer", diff --git a/src/DIRAC/Core/scripts/dirac_install_db.py b/src/DIRAC/Core/scripts/dirac_install_db.py index c74347cfbe6..a032aa5709f 100755 --- a/src/DIRAC/Core/scripts/dirac_install_db.py +++ b/src/DIRAC/Core/scripts/dirac_install_db.py @@ -2,6 +2,8 @@ """ Create a new DB in the MySQL server """ +from DIRAC import exit as DIRACExit +from DIRAC import gConfig, gLogger from DIRAC.Core.Base.Script import Script @@ -12,24 +14,40 @@ def main(): _, args = Script.parseCommandLine() # Script imports - from DIRAC import gConfig + from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import useServerCertificate + from DIRAC.Core.Security.ProxyInfo import getProxyInfo from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller from DIRAC.FrameworkSystem.Utilities import MonitoringUtilities + user = "DIRAC" + gComponentInstaller.exitOnError = True gComponentInstaller.getMySQLPasswords() for db in args: result = gComponentInstaller.installDatabase(db) if not result["OK"]: - print("ERROR: failed to correctly install %s" % db, result["Message"]) - continue + gLogger.error(f"Failed to correctly install {db}:", result["Message"]) + DIRACExit(1) extension, system = result["Value"] - gComponentInstaller.addDatabaseOptionsToCS(gConfig, system, db, overwrite=True) + result = gComponentInstaller.addDatabaseOptionsToCS(gConfig, system, db, overwrite=True) + if not result["OK"]: + gLogger.error("Failed to add database options to CS:", result["Message"]) + DIRACExit(1) if db != "InstalledComponentsDB": - result = MonitoringUtilities.monitorInstallation("DB", system, db) + # get the user that installed the DB + if not useServerCertificate(): + result = getProxyInfo() + if not result["OK"]: + return result + proxyInfo = result["Value"] + if "username" in proxyInfo: + user = proxyInfo["username"] + + result = MonitoringUtilities.monitorInstallation("DB", system, db, user=user) if not result["OK"]: - print("ERROR: failed to register installation in database: %s" % result["Message"]) + gLogger.error("Failed to register installation in database:", result["Message"]) + DIRACExit(1) if __name__ == "__main__": diff --git a/src/DIRAC/Core/scripts/dirac_platform.py b/src/DIRAC/Core/scripts/dirac_platform.py index 81bcf0583c3..f2807490fb7 100755 --- a/src/DIRAC/Core/scripts/dirac_platform.py +++ b/src/DIRAC/Core/scripts/dirac_platform.py @@ -68,9 +68,9 @@ def libc_ver(executable=sys.executable, lib="", version="", chunksize=2048): lib = "libc" elif glibc: glibcversion_parts = glibcversion.split(".") - for i in range(len(glibcversion_parts)): + for i, part in enumerate(glibcversion_parts): try: - glibcversion_parts[i] = int(glibcversion_parts[i]) + glibcversion_parts[i] = int(part) except ValueError: glibcversion_parts[i] = 0 if libcinit and not lib: @@ -114,9 +114,9 @@ def getPlatformString(): newest_lib = [0, 0, 0] for lib in libs: lib_parts = libc_ver(lib)[1].split(".") - for i in range(len(lib_parts)): + for i, part in enumerate(lib_parts): try: - lib_parts[i] = int(lib_parts[i]) + lib_parts[i] = int(part) except ValueError: lib_parts[i] = 0 # print "non integer version numbers" @@ -126,8 +126,6 @@ def getPlatformString(): platformTuple += ("glibc-" + ".".join(map(str, newest_lib)),) elif platformTuple[0] == "Darwin": platformTuple += (".".join(platform.mac_ver()[0].split(".")[:2]),) - elif platformTuple[0] == "Windows": - platformTuple += (platform.win32_ver()[0],) else: platformTuple += platform.release() diff --git a/src/DIRAC/Core/scripts/dirac_service.py b/src/DIRAC/Core/scripts/dirac_service.py index 5d65a6c6c48..d6307cd34bd 100755 --- a/src/DIRAC/Core/scripts/dirac_service.py +++ b/src/DIRAC/Core/scripts/dirac_service.py @@ -25,10 +25,7 @@ def main(): serverName = positionalArgs[0] localCfg.setConfigurationForServer(serverName) localCfg.addMandatoryEntry("Port") - # localCfg.addMandatoryEntry( "HandlerPath" ) - localCfg.addMandatoryEntry("/DIRAC/Setup") localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") - localCfg.addDefaultEntry("LogLevel", "INFO") localCfg.addDefaultEntry("LogColor", True) resultDict = localCfg.loadUserData() if not resultDict["OK"]: diff --git a/src/DIRAC/Core/scripts/install_full.cfg b/src/DIRAC/Core/scripts/install_full.cfg index 8154d9be747..2f4c59e83df 100755 --- a/src/DIRAC/Core/scripts/install_full.cfg +++ b/src/DIRAC/Core/scripts/install_full.cfg @@ -20,8 +20,6 @@ LocalInstallation SiteName = lbcertifdirac7.cern.ch # Setup name Setup = DIRAC-Production - # Default name of system instances - InstanceName = Production # Flag to skip download of CAs, on the first Server of your installation you need to get CAs # installed by some external means SkipCADownload = yes @@ -73,7 +71,6 @@ LocalInstallation Databases += JobLoggingDB Databases += UserProfileDB Databases += TaskQueueDB - Databases += NotificationDB Databases += ReqDB Databases += FTSDB Databases += ProxyDB @@ -85,37 +82,36 @@ LocalInstallation # The list of Services to be installed, this is not an exhaustive list of available # services, consult the DIRAC administrator docs for other possibilities Services = Configuration/Server - Services += Framework/ComponentMonitoring + Services += Framework/TornadoComponentMonitoring Services += Framework/SystemAdministrator + Services += Accounting/DataStore + Services += Accounting/ReportGenerator Services += DataManagement/StorageElement - Services += DataManagement/FileCatalog - Services += DataManagement/StorageElementProxy - Services += DataManagement/FTSManager - Services += Framework/Monitoring - Services += Framework/Notification - Services += Framework/SecurityLogging - Services += Framework/UserProfileManager - Services += Framework/ProxyManager - Services += Framework/Plotting + Services += DataManagement/TornadoDataIntegrity + Services += DataManagement/TornadoFileCatalog + Services += DataManagement/TornadoFTS3Manager + Services += DataManagement/TornadoS3Gateway Services += Framework/BundleDelivery - Services += Monitoring/Monitoring - Services += WorkloadManagement/SandboxStore - Services += WorkloadManagement/Matcher - Services += WorkloadManagement/JobMonitoring - Services += WorkloadManagement/JobManager - Services += WorkloadManagement/JobStateUpdate - Services += WorkloadManagement/WMSAdministrator + Services += Framework/TornadoProxyManager + Services += Framework/TornadoTokenManager + Services += Framework/TornadoUserProfileManager + Services += Monitoring/TornadoMonitoring + Services += RequestManagement/TornadoReqManager + Services += ResourceStatus/TornadoPublisher + Services += ResourceStatus/TornadoResourceManagement + Services += ResourceStatus/TornadoResourceStatus + Services += StorageManagement/TornadoStorageManager + Services += Transformation/TornadoTransformationManager Services += WorkloadManagement/OptimizationMind - Services += RequestManagement/ReqManager - Services += Accounting/DataStore - Services += Accounting/ReportGenerator - Services += ResourceStatus/ResourceManagement - Services += ResourceStatus/ResourceStatus - Services += Transformation/TransformationManager + Services += WorkloadManagement/Matcher + Services += WorkloadManagement/SandboxStore + Services += WorkloadManagement/TornadoJobMonitoring + Services += WorkloadManagement/TornadoJobManager + Services += WorkloadManagement/TornadoJobStateUpdate + Services += WorkloadManagement/TornadoWMSAdministrator # The list of Agents to be installed - Agents = Framework/CAUpdateAgent - Agents += DataManagement/FTS3Agent + Agents = DataManagement/FTS3Agent Agents += WorkloadManagement/PilotStatusAgent Agents += WorkloadManagement/SiteDirector Agents += WorkloadManagement/JobCleaningAgent @@ -143,7 +139,7 @@ LocalInstallation Host = Port = } - # For ElasticSearch + # For OpenSearch NoSQLDatabases { User = diff --git a/src/DIRAC/Core/scripts/install_site.sh b/src/DIRAC/Core/scripts/install_site.sh deleted file mode 100755 index 8ee06327291..00000000000 --- a/src/DIRAC/Core/scripts/install_site.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash - -usage() { - echo Usage: - echo " install_site.sh [Options] ... CFG_file" - echo - echo "CFG_file - is the name of the installation configuration file which contains" - echo " all the instructions for the DIRAC installation. See DIRAC Administrator " - echo " Guide for the details" - echo "Options:" - echo " -d, --debug debug mode" - echo " -h, --help print this" -} - -while [ "${1}" ] -do - case "${1}" in - - -h | --help ) - usage - exit 0 - ;; - - -d | --debug ) - DEBUG='-o LogLevel=DEBUG' - ;; - - * ) - installCfg=${1} - ;; - - esac - shift -done - -if [[ -z "${installCfg}" ]]; then - usage - exit 1 -fi - -# Get the version of dirac-install requested - if none is requested, the version will come from integration -# -curl -L -o dirac-install https://raw.githubusercontent.com/DIRACGrid/management/master/dirac-install.py || exit -# -# define the target Dir -# -installDir=$(grep TargetPath "${installCfg}" | grep -v '#' | cut -d '=' -f 2 | sed -e 's/ //g') -# -mkdir -p "${installDir}" || exit -# - -python dirac-install -t server "${installCfg}" -source "${installDir}"/bashrc -dirac-configure --cfg "${installCfg}" "$DEBUG" -dirac-setup-site "${DEBUG}" diff --git a/src/DIRAC/DataManagementSystem/Agent/FTS3Agent.py b/src/DIRAC/DataManagementSystem/Agent/FTS3Agent.py index 954a4ed166c..2fef08f18cf 100644 --- a/src/DIRAC/DataManagementSystem/Agent/FTS3Agent.py +++ b/src/DIRAC/DataManagementSystem/Agent/FTS3Agent.py @@ -1,6 +1,4 @@ """ -.. versionadded:: v6r20 - FTS3Agent implementation. It is in charge of submitting and monitoring all the transfers. It can be duplicated. @@ -12,29 +10,35 @@ :caption: FTS3Agent options """ + +import datetime import errno +import os import time -# from threading import current_thread -from multiprocessing.pool import ThreadPool - # We use the dummy module because we use the ThreadPool from multiprocessing.dummy import current_process -from socket import gethostname - -from DIRAC import S_OK, S_ERROR +# from threading import current_thread +from multiprocessing.pool import ThreadPool +from socket import gethostname +from urllib import parse -from DIRAC.MonitoringSystem.Client.DataOperationSender import DataOperationSender +from DIRAC import S_ERROR, S_OK +from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations as opHelper +from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getDNForUsername +from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getFTS3ServerDict from DIRAC.Core.Base.AgentModule import AgentModule from DIRAC.Core.Utilities.DErrno import cmpError from DIRAC.Core.Utilities.DictCache import DictCache from DIRAC.Core.Utilities.TimeUtilities import fromString -from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getFTS3ServerDict -from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations as opHelper -from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getDNForUsername +from DIRAC.DataManagementSystem.Client.FTS3Job import FTS3Job +from DIRAC.DataManagementSystem.DB.FTS3DB import FTS3DB +from DIRAC.DataManagementSystem.private import FTS3Utilities from DIRAC.FrameworkSystem.Client.Logger import gLogger from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager +from DIRAC.MonitoringSystem.Client.DataOperationSender import DataOperationSender +from DIRAC.FrameworkSystem.Client.TokenManagerClient import gTokenManager from DIRAC.DataManagementSystem.private import FTS3Utilities from DIRAC.DataManagementSystem.DB.FTS3DB import FTS3DB from DIRAC.DataManagementSystem.Client.FTS3Job import FTS3Job @@ -45,13 +49,20 @@ AGENT_NAME = "DataManagement/FTS3Agent" # Lifetime in seconds of the proxy we download for submission -PROXY_LIFETIME = 43200 # 12 hours +# Because we force the redelegation if only a third is left, +# and we want to have a quiet night (~12h) +# let's make the lifetime 12*3 hours +PROXY_LIFETIME = 36 * 3600 # 36 hours # Instead of querying many jobs at once, # which maximizes the possibility of race condition # when running multiple agents, we rather do it in steps JOB_MONITORING_BATCH_SIZE = 20 +# We do not monitor a job more often +# than MONITORING_DELAY in minutes +MONITORING_DELAY = 10 + class FTS3Agent(AgentModule): """ @@ -95,6 +106,10 @@ def __readConf(self): self.maxDelete = self.am_getOption("DeleteLimitPerCycle", 100) # lifetime of the proxy we download to delegate to FTS self.proxyLifetime = self.am_getOption("ProxyLifetime", PROXY_LIFETIME) + self.jobMonitoringBatchSize = self.am_getOption("JobMonitoringBatchSize", JOB_MONITORING_BATCH_SIZE) + self.useTokens = self.am_getOption("UseTokens", False) + + self.jobMonitoringBatchSize = self.am_getOption("JobMonitoringBatchSize", JOB_MONITORING_BATCH_SIZE) return S_OK() @@ -108,6 +123,8 @@ def initialize(self): # name that will be used in DB for assignment tag self.assignmentTag = gethostname().split(".")[0] + self.workDirectory = self.am_getWorkDirectory() + res = self.__readConf() # We multiply by two because of the two threadPools @@ -163,7 +180,13 @@ def getFTS3Context(self, username, group, ftsServer, threadID): # We take the first DN returned userDN = res["Value"][0] - log.debug("UserDN %s" % userDN) + log.debug(f"UserDN {userDN}") + + # Chose a meaningful proxy name for easier debugging + srvName = parse.urlparse(ftsServer).netloc.split(":")[0] + proxyFile = os.path.join( + self.workDirectory, f"{int(time.time())}_{username}_{group}_{srvName}_{threadID}.pem" + ) # We dump the proxy to a file. # It has to have a lifetime of self.proxyLifetime @@ -171,13 +194,13 @@ def getFTS3Context(self, username, group, ftsServer, threadID): # we should make our cache a bit less than 2/3rd of the lifetime cacheTime = int(2 * self.proxyLifetime / 3) - 600 res = gProxyManager.downloadVOMSProxyToFile( - userDN, group, requiredTimeLeft=self.proxyLifetime, cacheTime=cacheTime + userDN, group, requiredTimeLeft=self.proxyLifetime, cacheTime=cacheTime, filePath=proxyFile ) if not res["OK"]: return res proxyFile = res["Value"] - log.debug("Proxy file %s" % proxyFile) + log.debug(f"Proxy file {proxyFile}") # We generate the context # In practice, the lifetime will be less than proxyLifetime @@ -208,7 +231,7 @@ def _monitorJob(self, ftsJob): # General try catch to avoid that the tread dies try: threadID = current_process().name - log = gLogger.getLocalSubLogger("_monitorJob/%s" % ftsJob.jobID) + log = gLogger.getLocalSubLogger(f"_monitorJob/{ftsJob.jobID}") res = self.getFTS3Context(ftsJob.username, ftsJob.userGroup, ftsJob.ftsServer, threadID=threadID) @@ -258,7 +281,7 @@ def _monitorJob(self, ftsJob): except Exception as e: log.exception("Exception while monitoring job", repr(e)) - return ftsJob, S_ERROR(0, "Exception %s" % repr(e)) + return ftsJob, S_ERROR(0, f"Exception {repr(e)}") @staticmethod def _monitorJobCallback(returnedValue): @@ -266,13 +289,16 @@ def _monitorJobCallback(returnedValue): :param returnedValue: value returned by the _monitorJob method (ftsJob, standard dirac return struct) """ - - ftsJob, res = returnedValue - log = gLogger.getLocalSubLogger("_monitorJobCallback/%s" % ftsJob.jobID) - if not res["OK"]: - log.error("Error updating job status", res) + if isinstance(returnedValue, tuple) and len(returnedValue) == 2: + ftsJob, res = returnedValue + log = gLogger.getLocalSubLogger(f"_monitorJobCallback/{ftsJob.jobID}") + if not res["OK"]: + log.error("Error updating job status", res) + else: + log.debug("Successfully updated job status") else: - log.debug("Successfully updated job status") + log = gLogger.getLocalSubLogger("_monitorJobCallback") + log.error("Invalid return value when monitoring job", f"{returnedValue!r}") def monitorJobsLoop(self): """* fetch the active FTSJobs from the DB @@ -282,26 +308,36 @@ def monitorJobsLoop(self): """ log = gLogger.getSubLogger("monitorJobs") - log.debug("Size of the context cache %s" % len(self._globalContextCache)) + log.debug(f"Size of the context cache {len(self._globalContextCache)}") # Find the number of loops nbOfLoops, mod = divmod(self.jobBulkSize, JOB_MONITORING_BATCH_SIZE) if mod: nbOfLoops += 1 + # Not only is it pointless to monitor right after submission + # but also we would end up fetching multiple time the same job otherwise + # as we call getActiveJobs by batch + lastMonitor = datetime.datetime.utcnow() - datetime.timedelta(minutes=MONITORING_DELAY) + log.debug("Getting active jobs") for loopId in range(nbOfLoops): - log.info("Getting next batch of jobs to monitor", f"{loopId}/{nbOfLoops}") # get jobs from DB - res = self.fts3db.getActiveJobs(limit=JOB_MONITORING_BATCH_SIZE, jobAssignmentTag=self.assignmentTag) + res = self.fts3db.getActiveJobs( + limit=self.jobMonitoringBatchSize, lastMonitor=lastMonitor, jobAssignmentTag=self.assignmentTag + ) if not res["OK"]: log.error("Could not retrieve ftsJobs from the DB", res) return res activeJobs = res["Value"] + if not activeJobs: + log.info("No more jobs to monitor") + break + log.info("Jobs queued for monitoring", len(activeJobs)) # We store here the AsyncResult object on which we are going to wait @@ -309,7 +345,7 @@ def monitorJobsLoop(self): # Starting the monitoring threads for ftsJob in activeJobs: - log.debug("Queuing executing of ftsJob %s" % ftsJob.jobID) + log.debug(f"Queuing executing of ftsJob {ftsJob.jobID}") # queue the execution of self._monitorJob( ftsJob ) in the thread pool # The returned value is passed to _monitorJobCallback applyAsyncResults.append( @@ -325,7 +361,7 @@ def monitorJobsLoop(self): # If we got less to monitor than what we asked, # stop looping - if len(activeJobs) < JOB_MONITORING_BATCH_SIZE: + if len(activeJobs) < self.jobMonitoringBatchSize: break # Commit records after each loop self.dataOpSender.concludeSending() @@ -340,13 +376,16 @@ def _treatOperationCallback(returnedValue): :param returnedValue: value returned by the _treatOperation method (ftsOperation, standard dirac return struct) """ - - operation, res = returnedValue - log = gLogger.getLocalSubLogger("_treatOperationCallback/%s" % operation.operationID) - if not res["OK"]: - log.error("Error treating operation", res) + if isinstance(returnedValue, tuple) and len(returnedValue) == 2: + operation, res = returnedValue + log = gLogger.getLocalSubLogger(f"_treatOperationCallback/{operation.operationID}") + if not res["OK"]: + log.error("Error treating operation", res) + else: + log.debug("Successfully treated operation") else: - log.debug("Successfully treated operation") + log = gLogger.getLocalSubLogger("_treatOperationCallback") + log.error("Invalid return value when treating operation", f"{returnedValue!r}") def _treatOperation(self, operation): """Treat one operation: @@ -359,12 +398,12 @@ def _treatOperation(self, operation): """ try: threadID = current_process().name - log = gLogger.getLocalSubLogger("treatOperation/%s" % operation.operationID) + log = gLogger.getLocalSubLogger(f"treatOperation/{operation.operationID}") # If the operation is totally processed # we perform the callback if operation.isTotallyProcessed(): - log.debug("FTS3Operation %s is totally processed" % operation.operationID) + log.debug(f"FTS3Operation {operation.operationID} is totally processed") res = operation.callback() if not res["OK"]: @@ -378,7 +417,7 @@ def _treatOperation(self, operation): return operation, res else: - log.debug("FTS3Operation %s is not totally processed yet" % operation.operationID) + log.debug(f"FTS3Operation {operation.operationID} is not totally processed yet") # This flag is set to False if we want to stop the ongoing processing # of an operation, typically when the matching RMS Request has been @@ -415,8 +454,21 @@ def _treatOperation(self, operation): operation.status = "Canceled" continueOperationProcessing = False - if continueOperationProcessing: + log.info("Canceling the associated FTS3 jobs") + + for ftsJob in operation.ftsJobs: + res = self.getFTS3Context( + ftsJob.username, ftsJob.userGroup, ftsJob.ftsServer, threadID=threadID + ) + if not res["OK"]: + log.error("Error getting context", res) + continue + + context = res["Value"] + # We ignore the return on purpose + ftsJob.cancel(context) + if continueOperationProcessing: res = operation.prepareNewJobs( maxFilesPerJob=self.maxFilesPerJob, maxAttemptsPerFile=self.maxAttemptsPerFile ) @@ -436,7 +488,7 @@ def _treatOperation(self, operation): continue ftsServer = res["Value"] - log.debug("Use %s server" % ftsServer) + log.debug(f"Use {ftsServer} server") ftsJob.ftsServer = ftsServer @@ -454,7 +506,22 @@ def _treatOperation(self, operation): log.error("Could not select TPC list", repr(e)) continue - res = ftsJob.submit(context=context, protocols=tpcProtocols) + # If we use token, get an access token with the + # fts scope in it + # The FTS3Job will decide to use it or not + fts_access_token = None + if self.useTokens: + res = gTokenManager.getToken( + userGroup=ftsJob.userGroup, + requiredTimeLeft=3600, + scope=["fts"], + ) + if not res["OK"]: + return operation, res + + fts_access_token = res["Value"]["access_token"] + + res = ftsJob.submit(context=context, protocols=tpcProtocols, fts_access_token=fts_access_token) if not res["OK"]: log.error("Could not submit FTS3Job", f"FTS3Operation {operation.operationID} : {res}") @@ -468,7 +535,7 @@ def _treatOperation(self, operation): % (operation.operationID, len(submittedFileIds)) ) - # new jobs are put in the DB at the same time + # new jobs are put in the DB at the same time res = self.fts3db.persistOperation(operation) if not res["OK"]: @@ -478,7 +545,7 @@ def _treatOperation(self, operation): except Exception as e: log.exception("Exception in the thread", repr(e)) - return operation, S_ERROR("Exception %s" % repr(e)) + return operation, S_ERROR(f"Exception {repr(e)}") def treatOperationsLoop(self): """* Fetch all the FTSOperations which are not finished @@ -489,7 +556,7 @@ def treatOperationsLoop(self): log = gLogger.getSubLogger("treatOperations") - log.debug("Size of the context cache %s" % len(self._globalContextCache)) + log.debug(f"Size of the context cache {len(self._globalContextCache)}") log.info("Getting non finished operations") @@ -503,12 +570,12 @@ def treatOperationsLoop(self): incompleteOperations = res["Value"] - log.info("Treating %s incomplete operations" % len(incompleteOperations)) + log.info(f"Treating {len(incompleteOperations)} incomplete operations") applyAsyncResults = [] for operation in incompleteOperations: - log.debug("Queuing executing of operation %s" % operation.operationID) + log.debug(f"Queuing executing of operation {operation.operationID}") # queue the execution of self._treatOperation( operation ) in the thread pool # The returned value is passed to _treatOperationCallback applyAsyncResults.append( @@ -541,7 +608,7 @@ def kickOperations(self): return res kickedOperations = res["Value"] - log.info("Kicked %s stuck operations" % kickedOperations) + log.info(f"Kicked {kickedOperations} stuck operations") return S_OK() @@ -558,7 +625,7 @@ def kickJobs(self): return res kickedJobs = res["Value"] - log.info("Kicked %s stuck jobs" % kickedJobs) + log.info(f"Kicked {kickedJobs} stuck jobs") return S_OK() @@ -575,7 +642,7 @@ def deleteOperations(self): return res deletedOperations = res["Value"] - log.info("Deleted %s final operations" % deletedOperations) + log.info(f"Deleted {deletedOperations} final operations") return S_OK() @@ -652,7 +719,6 @@ def endExecution(self): return self.dataOpSender.concludeSending() def __sendAccounting(self, ftsJob): - self.dataOpSender.sendData( ftsJob.accountingDict, commitFlag=True, diff --git a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/ArchiveFiles.py b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/ArchiveFiles.py index 06bd6cb1c1a..257e0417f3a 100644 --- a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/ArchiveFiles.py +++ b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/ArchiveFiles.py @@ -57,7 +57,7 @@ def _run(self): self._checkArchiveLFN() for parameter, value in self.parameterDict.items(): self.log.info(f"Parameters: {parameter} = {value}") - self.log.info("Cache folder: %r" % self.cacheFolder) + self.log.info(f"Cache folder: {self.cacheFolder!r}") self.waitingFiles = self.getWaitingFilesList() self.lfns = [opFile.LFN for opFile in self.waitingFiles] self._checkReplicas() @@ -71,9 +71,9 @@ def _checkArchiveLFN(self): """Make sure the archive LFN does not exist yet.""" archiveLFN = self.parameterDict["ArchiveLFN"] exists = returnSingleResult(self.fc.isFile(archiveLFN)) - self.log.debug("Checking for Tarball existence %r" % exists) + self.log.debug(f"Checking for Tarball existence {exists!r}") if exists["OK"] and exists["Value"]: - raise RuntimeError("Tarball %r already exists" % archiveLFN) + raise RuntimeError(f"Tarball {archiveLFN!r} already exists") def _checkReplicas(self): """Make sure the source files are at the sourceSE.""" @@ -90,7 +90,7 @@ def _checkReplicas(self): if sourceSE in replInfo: atSource.append(lfn) else: - self.log.warn("LFN {!r} not found at source, only at: {}".format(lfn, ",".join(replInfo.keys()))) + self.log.warn(f"LFN {lfn!r} not found at source, only at: {','.join(replInfo.keys())}") notAt.append(lfn) for lfn, errorMessage in resReplica["Value"]["Failed"].items(): @@ -100,10 +100,10 @@ def _checkReplicas(self): failed.append(lfn) if failed: - self.log.error("LFNs failed to get replica info:", "%r" % " ".join(failed)) + self.log.error("LFNs failed to get replica info:", f"{' '.join(failed)!r}") raise RuntimeError("Failed to get some replica information") if notAt: - self.log.error("LFNs not at sourceSE:", "%r" % " ".join(notAt)) + self.log.error("LFNs not at sourceSE:", f"{' '.join(notAt)!r}") raise RuntimeError("Some replicas are not at the source") def _downloadFiles(self): @@ -117,7 +117,7 @@ def _downloadFiles(self): attempts = 0 destFolder = os.path.join(self.cacheFolder, os.path.dirname(lfn)[1:]) - self.log.debug("Local Cache Folder: %s" % destFolder) + self.log.debug(f"Local Cache Folder: {destFolder}") if not os.path.exists(destFolder): os.makedirs(destFolder) while True: @@ -138,7 +138,7 @@ def _downloadFiles(self): break if attempts > 10: self.log.error("Completely failed to download file:", errorString) - raise RuntimeError("Completely failed to download file: %s" % errorString) + raise RuntimeError(f"Completely failed to download file: {errorString}") return def _checkFilePermissions(self): @@ -171,12 +171,12 @@ def _tarFiles(self): def _uploadTarBall(self): """Upload the tarball to specified LFN.""" lfn = self.parameterDict["ArchiveLFN"] - self.log.info("Uploading tarball to %r" % lfn) + self.log.info(f"Uploading tarball to {lfn!r}") localFile = os.path.basename(lfn) tarballSE = self.parameterDict["TarballSE"] upload = returnSingleResult(self.dm.putAndRegister(lfn, localFile, tarballSE)) if not upload["OK"]: - raise RuntimeError("Failed to upload tarball: %s" % upload["Message"]) + raise RuntimeError(f"Failed to upload tarball: {upload['Message']}") self.log.verbose("Uploading finished") def _registerDescendent(self): @@ -215,8 +215,8 @@ def _cleanup(self): if "ArchiveLFN" in self.parameterDict: os.remove(os.path.basename(self.parameterDict["ArchiveLFN"])) except OSError as e: - self.log.debug("Error when removing tarball: %s" % str(e)) + self.log.debug(f"Error when removing tarball: {str(e)}") try: shutil.rmtree(self.cacheFolder, ignore_errors=True) except OSError as e: - self.log.debug("Error when removing cacheFolder: %s" % str(e)) + self.log.debug(f"Error when removing cacheFolder: {str(e)}") diff --git a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/CheckMigration.py b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/CheckMigration.py index 705a5b0925e..17a037af6f0 100644 --- a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/CheckMigration.py +++ b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/CheckMigration.py @@ -45,9 +45,9 @@ def _run(self): migrated = True and migrated continue metaData = returnSingleResult(se.getFileMetadata(opFile.LFN)) - self.log.debug("MetaData: %s" % pformat(metaData)) + self.log.debug(f"MetaData: {pformat(metaData)}") if not metaData["OK"]: - self.log.error("Failed to get metadata:", "{}: {}".format(opFile.LFN, metaData["Message"])) + self.log.error("Failed to get metadata:", f"{opFile.LFN}: {metaData['Message']}") migrated = False continue migrated = metaData["Value"].get("Migrated", 0) == 1 and migrated diff --git a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/DMSRequestOperationsBase.py b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/DMSRequestOperationsBase.py index d079fd935a8..5fc92ebadf2 100644 --- a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/DMSRequestOperationsBase.py +++ b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/DMSRequestOperationsBase.py @@ -61,17 +61,17 @@ def checkSEsRSS(self, checkSEs=None, access="WriteAccess", failIfBanned=True): # If Some SE are always banned, we fail the request if alwaysBannedSEs: - self.operation.Error = "%s always banned" % alwaysBannedSEs + self.operation.Error = f"{alwaysBannedSEs} always banned" if failIfBanned: self.log.info("Some storages are always banned, failing the request", alwaysBannedSEs) for opFile in self.operation: - opFile.Error = "%s always banned" % alwaysBannedSEs + opFile.Error = f"{alwaysBannedSEs} always banned" opFile.Status = "Failed" # If it is temporary, we wait an hour else: self.log.info("Banning is temporary, next attempt in an hour") - self.operation.Error = "%s currently banned" % bannedSEs + self.operation.Error = f"{bannedSEs} currently banned" self.request.delayNextExecution(60) return S_OK(bannedSEs) diff --git a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/MoveReplica.py b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/MoveReplica.py index 924cff187b3..558e6eef33a 100644 --- a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/MoveReplica.py +++ b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/MoveReplica.py @@ -59,7 +59,7 @@ def __call__(self): return bannedSource if bannedSource["Value"]: - self.operation.Error = "SourceSE %s is banned for reading" % sourceSE + self.operation.Error = f"SourceSE {sourceSE} is banned for reading" self.log.info(self.operation.Error) return S_OK(self.operation.Error) @@ -73,7 +73,7 @@ def __call__(self): return bannedTargets if bannedTargets["Value"]: - self.operation.Error = "%s targets are banned for writing" % ",".join(bannedTargets["Value"]) + self.operation.Error = f"{','.join(bannedTargets['Value'])} targets are banned for writing" return S_OK(self.operation.Error) # Can continue now @@ -91,7 +91,7 @@ def __call__(self): return bannedTargets if bannedTargets["Value"]: - return S_OK("%s targets are banned for removal" % ",".join(bannedTargets["Value"])) + return S_OK(f"{','.join(bannedTargets['Value'])} targets are banned for removal") # Can continue now self.log.verbose("No targets banned for removal") @@ -145,18 +145,17 @@ def __checkReplicas(self): self.rmsMonitoringReporter.addRecord(self.createRMSRecord("Failed", len(noReplicas))) self.rmsMonitoringReporter.commit() for lfn in noReplicas.keys(): - self.log.error("File %s doesn't exist" % lfn) + self.log.error(f"File {lfn} doesn't exist") waitingFiles[lfn].Status = "Failed" for lfn, reps in allReplicas.items(): if targetSESet.issubset(set(reps)): - self.log.info("file %s has been replicated to all targets" % lfn) + self.log.info(f"file {lfn} has been replicated to all targets") waitingFiles[lfn].Status = "Done" return S_OK() def dmRemoval(self, toRemoveDict, targetSEs): - if self.rmsMonitoring: self.rmsMonitoringReporter.addRecord(self.createRMSRecord("Attempted", len(toRemoveDict))) self.rmsMonitoringReporter.commit() @@ -167,7 +166,7 @@ def dmRemoval(self, toRemoveDict, targetSEs): # # loop over targetSEs for targetSE in targetSEs: - self.log.info("removing replicas at %s" % targetSE) + self.log.info(f"removing replicas at {targetSE}") # # 1st step - bulk removal bulkRemoval = self.bulkRemoval(toRemoveDict, targetSE) @@ -212,7 +211,7 @@ def dmRemoval(self, toRemoveDict, targetSEs): opFile.Status = "Done" if failed: - self.operation.Error = "failed to remove %s replicas" % failed + self.operation.Error = f"failed to remove {failed} replicas" if self.rmsMonitoring: self.rmsMonitoringReporter.commit() @@ -256,15 +255,15 @@ def dmTransfer(self, opFile): allReplicasCorrupted = replicas["AllReplicasCorrupted"] if noReplicas: - self.log.error("Unable to replicate", "File %s doesn't exist" % (lfn)) + self.log.error("Unable to replicate", f"File {lfn} doesn't exist") opFile.Error = "No replicas found" opFile.Status = "Failed" elif missingAllReplicas: - self.log.error("Unable to replicate", "%s, all replicas are missing" % (lfn)) + self.log.error("Unable to replicate", f"{lfn}, all replicas are missing") opFile.Error = "Missing all replicas" opFile.Status = "Failed" elif allReplicasCorrupted: - self.log.error("Unable to replicate", "%s, all replicas are corrupted" % (lfn)) + self.log.error("Unable to replicate", f"{lfn}, all replicas are corrupted") opFile.Error = "All replicas corrupted" opFile.Status = "Failed" elif someReplicasCorrupted: @@ -312,12 +311,10 @@ def dmTransfer(self, opFile): prString = f"file {lfn} replicated at {targetSE} in {repTime} s." if "register" in res["Value"]["Successful"][lfn]: - regTime = res["Value"]["Successful"][lfn]["register"] - prString += " and registered in %s s." % regTime + prString += f" and registered in {regTime} s." self.log.info(prString) else: - prString += " but failed to register" self.log.warn(prString) @@ -330,20 +327,18 @@ def dmTransfer(self, opFile): opFile.Error = "Failed to replicate" else: - reason = res["Value"]["Failed"][lfn] self.log.error("Failed to replicate and register", f"File {lfn} at {targetSE}: {reason}") opFile.Error = reason else: - - opFile.Error = "DataManager error: %s" % res["Message"] + opFile.Error = f"DataManager error: {res['Message']}" self.log.error("DataManager error", res["Message"]) if not opFile.Error: if self.rmsMonitoring: self.rmsMonitoringReporter.addRecord(self.createRMSRecord("Successful", 1)) if len(self.operation.targetSEList) > 1: - self.log.info("file %s has been replicated to all targetSEs" % lfn) + self.log.info(f"file {lfn} has been replicated to all targetSEs") else: if self.rmsMonitoring: self.rmsMonitoringReporter.addRecord(self.createRMSRecord("Failed", 1)) @@ -384,7 +379,7 @@ def singleRemoval(self, opFile, targetSE): proxyFile = None if "Write access not permitted for this credential" in opFile.Error: # # not a DataManger? set status to failed and return - if "DataManager" in self.shifter: + if "DataManager" in self.shifter: # pylint: disable =unsupported-membership-test # # you're a data manager - save current proxy and get a new one for LFN and retry saveProxy = os.environ["X509_USER_PROXY"] try: diff --git a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/PhysicalRemoval.py b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/PhysicalRemoval.py index 4160abb62fd..a3d22496357 100644 --- a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/PhysicalRemoval.py +++ b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/PhysicalRemoval.py @@ -66,7 +66,7 @@ def __call__(self): return bannedTargets if bannedTargets["Value"]: - return S_OK("%s targets are banned for removal" % ",".join(bannedTargets["Value"])) + return S_OK(f"{','.join(bannedTargets['Value'])} targets are banned for removal") # # get waiting files waitingFiles = self.getWaitingFilesList() @@ -85,8 +85,7 @@ def __call__(self): removalStatus[lfn] = dict.fromkeys(targetSEs, "") for targetSE in targetSEs: - - self.log.info("removing files from %s" % targetSE) + self.log.info(f"removing files from {targetSE}") # # 1st - bulk removal bulkRemoval = self.bulkRemoval(toRemoveDict, targetSE) @@ -133,7 +132,7 @@ def __call__(self): opFile.Status = "Done" if failed: - self.operation.Error = "failed to remove %s files" % failed + self.operation.Error = f"failed to remove {failed} files" if self.rmsMonitoring: self.rmsMonitoringReporter.commit() @@ -155,7 +154,7 @@ def singleRemoval(self, opFile, targetSE): proxyFile = None if "Write access not permitted for this credential" in opFile.Error: # # not a DataManger? set status to failed and return - if "DataManager" not in self.shifter: + if "DataManager" not in self.shifter: # pylint: disable =unsupported-membership-test opFile.Status = "Failed" elif not opFile.LFN: opFile.Error = "LFN not set" diff --git a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/PutAndRegister.py b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/PutAndRegister.py index 531bbc3471d..dea84920d2d 100644 --- a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/PutAndRegister.py +++ b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/PutAndRegister.py @@ -66,10 +66,9 @@ def __call__(self): targetSEs = self.operation.targetSEList if len(targetSEs) != 1: - self.log.error("Wrong value for TargetSE list, should contain only one target!", "%s" % targetSEs) + self.log.error("Wrong value for TargetSE list, should contain only one target!", f"{targetSEs}") self.operation.Error = "Wrong parameters: TargetSE should contain only one targetSE" for opFile in self.operation: - opFile.Status = "Failed" opFile.Error = "Wrong parameters: TargetSE should contain only one targetSE" @@ -78,7 +77,7 @@ def __call__(self): self.rmsMonitoringReporter.addRecord(self.createRMSRecord(status, len(self.operation))) self.rmsMonitoringReporter.commit() - return S_ERROR("TargetSE should contain only one target, got %s" % targetSEs) + return S_ERROR(f"TargetSE should contain only one target, got {targetSEs}") targetSE = targetSEs[0] bannedTargets = self.checkSEsRSS(targetSE) @@ -90,7 +89,7 @@ def __call__(self): return bannedTargets if bannedTargets["Value"]: - return S_OK("%s targets are banned for writing" % ",".join(bannedTargets["Value"])) + return S_OK(f"{','.join(bannedTargets['Value'])} targets are banned for writing") # # get waiting files waitingFiles = self.getWaitingFilesList() @@ -102,7 +101,7 @@ def __call__(self): for opFile in waitingFiles: # # get LFN lfn = opFile.LFN - self.log.info("processing file %s" % lfn) + self.log.info(f"processing file {lfn}") pfn = opFile.PFN guid = opFile.GUID checksum = opFile.Checksum @@ -138,9 +137,7 @@ def __call__(self): putAndRegister = putAndRegister["Successful"] if lfn in putAndRegister: - if "put" not in putAndRegister[lfn]: - if self.rmsMonitoring: self.rmsMonitoringReporter.addRecord(self.createRMSRecord("Failed", 1)) # self.dataLoggingClient().addFileRecord( lfn, "PutFail", targetSE, "", "PutAndRegister" ) @@ -152,13 +149,12 @@ def __call__(self): continue if "register" not in putAndRegister[lfn]: - if self.rmsMonitoring: self.rmsMonitoringReporter.addRecord(self.createRMSRecord("Failed", 1)) # self.dataLoggingClient().addFileRecord( lfn, "Put", targetSE, "", "PutAndRegister" ) # self.dataLoggingClient().addFileRecord( lfn, "RegisterFail", targetSE, "", "PutAndRegister" ) - self.log.info("put of {} to {} took {} seconds".format(lfn, targetSE, putAndRegister[lfn]["put"])) + self.log.info(f"put of {lfn} to {targetSE} took {putAndRegister[lfn]['put']} seconds") self.log.error("Register of lfn to SE failed", f"{lfn} to {targetSE}") opFile.Error = f"failed to register {lfn} at {targetSE}" diff --git a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/ReTransfer.py b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/ReTransfer.py index 2ef1c9a4d47..6f166611696 100644 --- a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/ReTransfer.py +++ b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/ReTransfer.py @@ -71,7 +71,7 @@ def __call__(self): return bannedTargets if bannedTargets["Value"]: - return S_OK("%s targets are banned for writing" % ",".join(bannedTargets["Value"])) + return S_OK(f"{','.join(bannedTargets['Value'])} targets are banned for writing") # # get waiting files waitingFiles = self.getWaitingFilesList() @@ -82,7 +82,7 @@ def __call__(self): self.rmsMonitoringReporter.addRecord(self.createRMSRecord("Attempted", len(toRetransfer))) if len(targetSEs) != 1: - error = "only one TargetSE allowed, got %d" % len(targetSEs) + error = f"only one TargetSE allowed, got {len(targetSEs)}" for opFile in toRetransfer.values(): opFile.Error = error opFile.Status = "Failed" @@ -96,7 +96,6 @@ def __call__(self): se = StorageElement(targetSE) for opFile in toRetransfer.values(): - reTransfer = se.retransferOnlineFile(opFile.LFN) if not reTransfer["OK"]: opFile.Error = reTransfer["Message"] @@ -116,7 +115,7 @@ def __call__(self): continue opFile.Status = "Done" - self.log.info("%s retransfer done" % opFile.LFN) + self.log.info(f"{opFile.LFN} retransfer done") if self.rmsMonitoring: self.rmsMonitoringReporter.addRecord(self.createRMSRecord("Successful", 1)) diff --git a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/RegisterFile.py b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/RegisterFile.py index a1592f96361..e7dac46a196 100644 --- a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/RegisterFile.py +++ b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/RegisterFile.py @@ -70,7 +70,6 @@ def __call__(self): # # loop over files for opFile in waitingFiles: - # # get LFN lfn = opFile.LFN # # and others @@ -85,7 +84,6 @@ def __call__(self): registerFile = dm.registerFile(fileTuple) # # check results if not registerFile["OK"] or lfn in registerFile["Value"]["Failed"]: - if self.rmsMonitoring: self.rmsMonitoringReporter.addRecord(self.createRMSRecord("Failed", 1)) # self.dataLoggingClient().addFileRecord( @@ -107,14 +105,13 @@ def __call__(self): failedFiles += 1 else: - if self.rmsMonitoring: self.rmsMonitoringReporter.addRecord(self.createRMSRecord("Successful", 1)) # self.dataLoggingClient().addFileRecord( # lfn, "Register", ','.join(catalogs) if catalogs else "all catalogs", "", "RegisterFile") self.log.verbose( - "file {} has been registered at {}".format(lfn, ",".join(catalogs) if catalogs else "all catalogs") + f"file {lfn} has been registered at {','.join(catalogs) if catalogs else 'all catalogs'}" ) opFile.Status = "Done" @@ -123,7 +120,7 @@ def __call__(self): # # final check if failedFiles: - self.log.warn("all files processed, %s files failed to register" % failedFiles) + self.log.warn(f"all files processed, {failedFiles} files failed to register") self.operation.Error = "some files failed to register" return S_ERROR(self.operation.Error) diff --git a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/RegisterReplica.py b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/RegisterReplica.py index 53d3ffc3fd4..5f56e16d6de 100644 --- a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/RegisterReplica.py +++ b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/RegisterReplica.py @@ -55,15 +55,17 @@ def __call__(self): # # loop over files registerOperations = {} successReplicas = 0 - for opFile in waitingFiles: + targetSE = self.operation.targetSEList[0] + replicaTuples = [(opFile.LFN, opFile.PFN, targetSE) for opFile in waitingFiles] + + registerReplica = self.dm.registerReplica(replicaTuples, catalogs) + + for opFile in waitingFiles: # # get LFN lfn = opFile.LFN # # and others - targetSE = self.operation.targetSEList[0] - replicaTuple = (lfn, opFile.PFN, targetSE) - # # call ReplicaManager - registerReplica = self.dm.registerReplica(replicaTuple, catalogs) + # # check results if not registerReplica["OK"] or lfn in registerReplica["Value"]["Failed"]: # There have been some errors @@ -131,7 +133,7 @@ def __call__(self): # # if we have new replications to take place, put them at the end if registerOperations: - self.log.info("adding %d operations to the request" % len(registerOperations)) + self.log.info(f"adding {len(registerOperations)} operations to the request") for operation in registerOperations.values(): self.operation._parent.addOperation(operation) diff --git a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/RemoveFile.py b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/RemoveFile.py index fef2be5e3dc..ecc603b18c4 100644 --- a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/RemoveFile.py +++ b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/RemoveFile.py @@ -117,13 +117,13 @@ def __call__(self): if not toRemoveDict: # If there are no files that can be removed, exit, else try once to remove them anyway - return S_OK("%s targets are always banned for removal" % ",".join(sorted(bannedTargets))) + return S_OK(f"{','.join(sorted(bannedTargets))} targets are always banned for removal") if toRemoveDict: if self.rmsMonitoring: self.rmsMonitoringReporter.addRecord(self.createRMSRecord("Attempted", len(toRemoveDict))) # # 1st step - bulk removal - self.log.debug("bulk removal of %s files" % len(toRemoveDict)) + self.log.debug(f"bulk removal of {len(toRemoveDict)} files") bulkRemoval = self.bulkRemoval(toRemoveDict) if not bulkRemoval["OK"]: self.log.error("Bulk file removal failed", bulkRemoval["Message"]) @@ -135,14 +135,14 @@ def __call__(self): # # 2nd step - single file removal for lfn, opFile in toRemoveDict.items(): - self.log.info("removing single file %s" % lfn) + self.log.info(f"removing single file {lfn}") singleRemoval = self.singleRemoval(opFile) if not singleRemoval["OK"]: self.log.error("Error removing single file", singleRemoval["Message"]) if self.rmsMonitoring: self.rmsMonitoringReporter.addRecord(self.createRMSRecord("Failed", 1)) else: - self.log.info("file %s has been removed" % lfn) + self.log.info(f"file {lfn} has been removed") if self.rmsMonitoring: self.rmsMonitoringReporter.addRecord(self.createRMSRecord("Successful", 1)) @@ -151,13 +151,13 @@ def __call__(self): (lfn, opFile) for (lfn, opFile) in toRemoveDict.items() if opFile.Status in ("Failed", "Waiting") ] if failedFiles: - self.operation.Error = "failed to remove %d files" % len(failedFiles) + self.operation.Error = f"failed to remove {len(failedFiles)} files" if self.rmsMonitoring: self.rmsMonitoringReporter.commit() if bannedTargets: - return S_OK("%s targets are banned for removal" % ",".join(sorted(bannedTargets))) + return S_OK(f"{','.join(sorted(bannedTargets))} targets are banned for removal") return S_OK() def bulkRemoval(self, toRemoveDict): @@ -180,7 +180,6 @@ def bulkRemoval(self, toRemoveDict): if lfn in bulkRemoval["Successful"]: opFile.Status = "Done" elif lfn in bulkRemoval["Failed"]: - error = bulkRemoval["Failed"][lfn] if isinstance(error, dict): error = ";".join([f"{k}-{v}" for k, v in error.items()]) @@ -200,16 +199,16 @@ def singleRemoval(self, opFile): # # try to remove with owner proxy proxyFile = None if "Write access not permitted for this credential" in opFile.Error: - if "DataManager" in self.shifter: + if "DataManager" in self.shifter: # pylint: disable =unsupported-membership-test # # you're a data manager - get proxy for LFN and retry saveProxy = os.environ["X509_USER_PROXY"] try: fileProxy = self.getProxyForLFN(opFile.LFN) if not fileProxy["OK"]: - opFile.Error = "Error getting owner's proxy : %s" % fileProxy["Message"] + opFile.Error = f"Error getting owner's proxy : {fileProxy['Message']}" else: proxyFile = fileProxy["Value"] - self.log.info("Trying to remove file with owner's proxy (file %s)" % proxyFile) + self.log.info(f"Trying to remove file with owner's proxy (file {proxyFile})") removeFile = self.dm.removeFile(opFile.LFN, force=True) self.log.always(str(removeFile)) diff --git a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/RemoveReplica.py b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/RemoveReplica.py index e2e8dc090bc..ed529f780d0 100644 --- a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/RemoveReplica.py +++ b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/RemoveReplica.py @@ -70,7 +70,7 @@ def __call__(self): return bannedTargets if bannedTargets["Value"]: - return S_OK("%s targets are banned for removal" % ",".join(bannedTargets["Value"])) + return S_OK(f"{','.join(bannedTargets['Value'])} targets are banned for removal") # # get waiting files waitingFiles = self.getWaitingFilesList() @@ -89,8 +89,7 @@ def __call__(self): # # loop over targetSEs for targetSE in targetSEs: - - self.log.info("Removing replicas at %s" % targetSE) + self.log.info(f"Removing replicas at {targetSE}") # # 1st step - bulk removal bulkRemoval = self._bulkRemoval(toRemoveDict, targetSE) @@ -136,7 +135,7 @@ def __call__(self): opFile.Status = "Done" if failed: - self.operation.Error = "failed to remove %s replicas" % failed + self.operation.Error = f"failed to remove {failed} replicas" if self.rmsMonitoring: self.rmsMonitoringReporter.commit() @@ -179,7 +178,7 @@ def _removeWithOwnerProxy(self, opFile, targetSE): """ if "Write access not permitted for this credential" in opFile.Error: proxyFile = None - if "DataManager" in self.shifter: + if "DataManager" in self.shifter: # pylint: disable =unsupported-membership-test # # you're a data manager - save current proxy and get a new one for LFN and retry saveProxy = os.environ["X509_USER_PROXY"] try: diff --git a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/ReplicateAndRegister.py b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/ReplicateAndRegister.py index a214a8001b4..062fe7181e2 100644 --- a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/ReplicateAndRegister.py +++ b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/ReplicateAndRegister.py @@ -46,8 +46,15 @@ from DIRAC.MonitoringSystem.Client.MonitoringReporter import MonitoringReporter -def filterReplicas(opFile, logger=None, dataManager=None, opSources=None): - """filter out banned/invalid source SEs""" +def filterReplicas(opFile, logger=None, dataManager=None, opSources=None, activeReplicas=None): + """filter out banned/invalid source SEs + + :param list opSources: list of SE names to which limit the possible sources + :param dict activeReplicas: the result of dm.getActiveReplicas(*)["Value"]. Used as a cache + + :returns: Valid list of SEs valid as source + + """ if logger is None: logger = gLogger @@ -57,19 +64,21 @@ def filterReplicas(opFile, logger=None, dataManager=None, opSources=None): log = logger.getSubLogger("filterReplicas") result = defaultdict(list) - replicas = dataManager.getActiveReplicas(opFile.LFN, getUrl=False, preferDisk=True) - if not replicas["OK"]: - log.error("Failed to get active replicas", replicas["Message"]) - return replicas + if not activeReplicas: + res = dataManager.getActiveReplicas(opFile.LFN, getUrl=False, preferDisk=True) + if not res["OK"]: + log.error("Failed to get active replicas", res["Message"]) + return res + activeReplicas = res["Value"] + reNotExists = re.compile(r".*such file.*") - replicas = replicas["Value"] - failed = replicas["Failed"].get(opFile.LFN, "") + failed = activeReplicas["Failed"].get(opFile.LFN, "") if reNotExists.match(failed.lower()): opFile.Status = "Failed" opFile.Error = failed return S_ERROR(failed) - replicas = replicas["Successful"].get(opFile.LFN, {}) + replicas = activeReplicas["Successful"].get(opFile.LFN, {}) # If user set sourceSEs, only consider those replicas if opSources: @@ -86,7 +95,7 @@ def filterReplicas(opFile, logger=None, dataManager=None, opSources=None): else: # There are replicas but we cannot get metadata because the replica is not active result["NoActiveReplicas"] += list(allReplicas) - log.verbose("File has no%s replica in File Catalog" % ("" if noReplicas else " active"), opFile.LFN) + log.verbose(f"File has no{'' if noReplicas else ' active'} replica in File Catalog", opFile.LFN) else: return allReplicas @@ -175,6 +184,35 @@ def __call__(self): if self.rmsMonitoring: self.rmsMonitoringReporter = MonitoringReporter(monitoringType="RMSMonitoring") + sourceSE = self.operation.SourceSE if self.operation.SourceSE else None + if sourceSE: + # check sourceSE for read + bannedSource = self.checkSEsRSS(sourceSE, "ReadAccess") + if not bannedSource["OK"]: + if self.rmsMonitoring: + for status in ["Attempted", "Failed"]: + self.rmsMonitoringReporter.addRecord(self.createRMSRecord(status, len(self.operation))) + self.rmsMonitoringReporter.commit() + return bannedSource + + if bannedSource["Value"]: + self.operation.Error = f"SourceSE {sourceSE} is banned for reading" + self.log.info(self.operation.Error) + return S_OK(self.operation.Error) + + # check targetSEs for write + bannedTargets = self.checkSEsRSS() + if not bannedTargets["OK"]: + if self.rmsMonitoring: + for status in ["Attempted", "Failed"]: + self.rmsMonitoringReporter.addRecord(self.createRMSRecord(status, len(self.operation))) + self.rmsMonitoringReporter.commit() + return bannedTargets + + if bannedTargets["Value"]: + self.operation.Error = f"{','.join(bannedTargets['Value'])} targets are banned for writing" + return S_OK(self.operation.Error) + # # check replicas first checkReplicas = self.__checkReplicas() if not checkReplicas["OK"]: @@ -222,7 +260,7 @@ def _addMetadataToFiles(self, toSchedule): {'lfn1': opFile, 'lfn2': opFile} """ if toSchedule: - self.log.info("found %s files to schedule, getting metadata from FC" % len(toSchedule)) + self.log.info(f"found {len(toSchedule)} files to schedule, getting metadata from FC") else: self.log.verbose("No files to schedule") return S_OK([]) @@ -253,9 +291,15 @@ def _addMetadataToFiles(self, toSchedule): return S_OK(filesToSchedule) - def _filterReplicas(self, opFile): + def _filterReplicas(self, opFile, activeReplicas): """filter out banned/invalid source SEs""" - return filterReplicas(opFile, logger=self.log, dataManager=self.dm, opSources=self.operation.sourceSEList) + return filterReplicas( + opFile, + logger=self.log, + dataManager=self.dm, + opSources=self.operation.sourceSEList, + activeReplicas=activeReplicas, + ) def _checkExistingFTS3Operations(self): """ @@ -342,13 +386,22 @@ def fts3Transfer(self): if self.rmsMonitoring: self.rmsMonitoringReporter.addRecord(self.createRMSRecord("Attempted", len(self.getWaitingFilesList()))) - for opFile in self.getWaitingFilesList(): + waitingFiles = self.getWaitingFilesList() + + allLFNs = [opFile.LFN for opFile in waitingFiles] + res = self.dm.getActiveReplicas(allLFNs, getUrl=False, preferDisk=True) + if not res["OK"]: + self.log.error("Failed to get active replicas", res["Message"]) + return res + allActiveReplicas = res["Value"] + + for opFile in waitingFiles: rmsFilesIds[opFile.FileID] = opFile opFile.Error = "" # # check replicas - replicas = self._filterReplicas(opFile) + replicas = self._filterReplicas(opFile, allActiveReplicas) if not replicas["OK"]: continue replicas = replicas["Value"] @@ -362,7 +415,7 @@ def fts3Transfer(self): if validReplicas: validTargets = list(set(self.operation.targetSEList) - set(validReplicas)) if not validTargets: - self.log.info("file %s is already present at all targets" % opFile.LFN) + self.log.info(f"file {opFile.LFN} is already present at all targets") opFile.Status = "Done" else: toSchedule[opFile.LFN] = [opFile, validTargets] @@ -373,27 +426,31 @@ def fts3Transfer(self): if noMetaReplicas: self.log.warn( "unable to schedule file", - "'{}': couldn't get metadata at {}".format(opFile.LFN, ",".join(noMetaReplicas)), + f"'{opFile.LFN}': couldn't get metadata at {','.join(noMetaReplicas)}", ) opFile.Error = "Couldn't get metadata" elif noReplicas: - self.log.error( - "Unable to schedule transfer", - "File {} doesn't exist at {}".format(opFile.LFN, ",".join(noReplicas)), - ) + if None in noReplicas: + self.log.error( + "Unable to schedule transfer", + f"File {opFile.LFN} doesn't have any replicas, which should never happen", + ) + else: + self.log.error( + "Unable to schedule transfer", + f"File {opFile.LFN} doesn't exist at {','.join(noReplicas)}", + ) opFile.Error = "No replicas found" opFile.Status = "Failed" elif badReplicas: self.log.error( "Unable to schedule transfer", - "File {}, all replicas have a bad checksum at {}".format(opFile.LFN, ",".join(badReplicas)), + f"File {opFile.LFN}, all replicas have a bad checksum at {','.join(badReplicas)}", ) opFile.Error = "All replicas have a bad checksum" opFile.Status = "Failed" elif noPFN: - self.log.warn( - "unable to schedule {}, could not get a PFN at {}".format(opFile.LFN, ",".join(noPFN)) - ) + self.log.warn(f"unable to schedule {opFile.LFN}, could not get a PFN at {','.join(noPFN)}") if self.rmsMonitoring: self.rmsMonitoringReporter.commit() @@ -412,13 +469,7 @@ def fts3Transfer(self): fts3Files.append(ftsFile) if fts3Files: - res = Registry.getUsernameForDN(self.request.OwnerDN) - if not res["OK"]: - self.log.error("Cannot get username for DN", "{} {}".format(self.request.OwnerDN, res["Message"])) - return res - - username = res["Value"] - fts3Operation = FTS3TransferOperation.fromRMSObjects(self.request, self.operation, username) + fts3Operation = FTS3TransferOperation.fromRMSObjects(self.request, self.operation) fts3Operation.ftsFiles = fts3Files try: @@ -436,9 +487,9 @@ def fts3Transfer(self): # might have nothing to schedule ftsSchedule = ftsSchedule["Value"] - self.log.info("Scheduled with FTS3Operation id %s" % ftsSchedule) + self.log.info(f"Scheduled with FTS3Operation id {ftsSchedule}") - self.log.info("%d files have been scheduled to FTS3" % len(fts3Files)) + self.log.info(f"{len(fts3Files)} files have been scheduled to FTS3") if self.rmsMonitoring: self.rmsMonitoringReporter.addRecord(self.createRMSRecord("Successful", len(fts3Files))) @@ -446,7 +497,7 @@ def fts3Transfer(self): for ftsFile in fts3Files: opFile = rmsFilesIds[ftsFile.rmsFileID] opFile.Status = "Scheduled" - self.log.debug("%s has been scheduled for FTS" % opFile.LFN) + self.log.debug(f"{opFile.LFN} has been scheduled for FTS") else: self.log.info("No files to schedule after metadata checks") @@ -458,36 +509,6 @@ def fts3Transfer(self): def dmTransfer(self, fromFTS=False): """replicate and register using dataManager""" - # # get waiting files. If none just return - # # source SE - sourceSE = self.operation.SourceSE if self.operation.SourceSE else None - if sourceSE: - # # check source se for read - bannedSource = self.checkSEsRSS(sourceSE, "ReadAccess") - if not bannedSource["OK"]: - if self.rmsMonitoring: - for status in ["Attempted", "Failed"]: - self.rmsMonitoringReporter.addRecord(self.createRMSRecord(status, len(self.operation))) - self.rmsMonitoringReporter.commit() - return bannedSource - - if bannedSource["Value"]: - self.operation.Error = "SourceSE %s is banned for reading" % sourceSE - self.log.info(self.operation.Error) - return S_OK(self.operation.Error) - - # # check targetSEs for write - bannedTargets = self.checkSEsRSS() - if not bannedTargets["OK"]: - if self.rmsMonitoring: - for status in ["Attempted", "Failed"]: - self.rmsMonitoringReporter.addRecord(self.createRMSRecord(status, len(self.operation))) - self.rmsMonitoringReporter.commit() - return bannedTargets - - if bannedTargets["Value"]: - self.operation.Error = "%s targets are banned for writing" % ",".join(bannedTargets["Value"]) - return S_OK(self.operation.Error) # Can continue now self.log.verbose("No targets banned for writing") @@ -497,7 +518,12 @@ def dmTransfer(self, fromFTS=False): return S_OK() # # loop over files if fromFTS: - self.log.info("Trying transfer using replica manager as FTS failed") + if getattr(self, "DMMode", True): + self.log.info("Trying transfer using DataManager as FTS failed") + else: + self.log.info("DataManager replication disabled, skipping") + return S_OK() + else: self.log.info("Transferring files using Data manager...") errors = defaultdict(int) @@ -506,6 +532,13 @@ def dmTransfer(self, fromFTS=False): if self.rmsMonitoring: self.rmsMonitoringReporter.addRecord(self.createRMSRecord("Attempted", len(waitingFiles))) + allLFNs = [opFile.LFN for opFile in waitingFiles] + res = self.dm.getActiveReplicas(allLFNs, getUrl=False, preferDisk=True) + if not res["OK"]: + self.log.error("Failed to get active replicas", res["Message"]) + return res + allActiveReplicas = res["Value"] + for opFile in waitingFiles: if opFile.Error in ( "Couldn't get metadata", @@ -520,7 +553,7 @@ def dmTransfer(self, fromFTS=False): lfn = opFile.LFN # Check if replica is at the specified source - replicas = self._filterReplicas(opFile) + replicas = self._filterReplicas(opFile, allActiveReplicas) if not replicas["OK"]: self.log.error("Failed to check replicas", replicas["Message"]) continue @@ -547,7 +580,7 @@ def dmTransfer(self, fromFTS=False): err = "File doesn't exist" errors[err] += 1 self.log.verbose( - "Unable to replicate", "File {} doesn't exist at {}".format(opFile.LFN, ",".join(noReplicas)) + "Unable to replicate", f"File {opFile.LFN} doesn't exist at {','.join(noReplicas)}" ) opFile.Error = err opFile.Status = "Failed" @@ -556,7 +589,7 @@ def dmTransfer(self, fromFTS=False): errors[err] += 1 self.log.error( "Unable to replicate", - "{}, all replicas have a bad checksum at {}".format(opFile.LFN, ",".join(badReplicas)), + f"{opFile.LFN}, all replicas have a bad checksum at {','.join(badReplicas)}", ) opFile.Error = err opFile.Status = "Failed" @@ -565,19 +598,12 @@ def dmTransfer(self, fromFTS=False): errors[err] += 1 self.log.verbose( "Unable to schedule transfer", - "{}, {} at {}".format(opFile.LFN, err, ",".join(noActiveReplicas)), + f"{opFile.LFN}, {err} at {','.join(noActiveReplicas)}", ) opFile.Error = err # All source SEs are banned, delay execution by 1 hour delayExecution = 60 continue - # # get the first one in the list - if sourceSE not in validReplicas: - if sourceSE: - err = "File not at specified source" - errors[err] += 1 - self.log.warn(f"{lfn} is not at specified sourceSE {sourceSE}, changed to {validReplicas[0]}") - sourceSE = validReplicas[0] # # loop over targetSE catalogs = self.operation.Catalog @@ -585,28 +611,23 @@ def dmTransfer(self, fromFTS=False): catalogs = [cat.strip() for cat in catalogs.split(",")] for targetSE in self.operation.targetSEList: - # # call DataManager if targetSE in validReplicas: self.log.warn(f"Request to replicate {lfn} to an existing location: {targetSE}") continue + sourceSE = self.operation.SourceSE if self.operation.SourceSE else None res = self.dm.replicateAndRegister(lfn, targetSE, sourceSE=sourceSE, catalog=catalogs) if res["OK"]: - if lfn in res["Value"]["Successful"]: - if "replicate" in res["Value"]["Successful"][lfn]: - repTime = res["Value"]["Successful"][lfn]["replicate"] prString = f"file {lfn} replicated at {targetSE} in {repTime} s." if "register" in res["Value"]["Successful"][lfn]: - regTime = res["Value"]["Successful"][lfn]["register"] - prString += " and registered in %s s." % regTime + prString += f" and registered in {regTime} s." self.log.info(prString) else: - prString += " but failed to register" self.log.warn(prString) @@ -616,19 +637,16 @@ def dmTransfer(self, fromFTS=False): self.request.insertAfter(registerOperation, self.operation) else: - self.log.error("Failed to replicate", f"{lfn} to {targetSE}") opFile.Error = "Failed to replicate" else: - reason = res["Value"]["Failed"][lfn] self.log.error("Failed to replicate and register", f"File {lfn} at {targetSE}:", reason) opFile.Error = reason else: - - opFile.Error = "DataManager error: %s" % res["Message"] + opFile.Error = f"DataManager error: {res['Message']}" self.log.error("DataManager error", res["Message"]) if not opFile.Error: @@ -636,7 +654,7 @@ def dmTransfer(self, fromFTS=False): self.rmsMonitoringReporter.addRecord(self.createRMSRecord("Successful", 1)) if len(self.operation.targetSEList) > 1: - self.log.info("file %s has been replicated to all targetSEs" % lfn) + self.log.info(f"file {lfn} has been replicated to all targetSEs") opFile.Status = "Done" elif self.rmsMonitoring: self.rmsMonitoringReporter.addRecord(self.createRMSRecord("Failed", 1)) diff --git a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/StagingCallback.py b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/StagingCallback.py index fddab690376..c49bdc840db 100644 --- a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/StagingCallback.py +++ b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/StagingCallback.py @@ -28,7 +28,7 @@ def __call__(self): """update the job status""" # # decode arguments jobID = self.operation.Arguments - self.log.info("Performing callback to job %s" % jobID) + self.log.info(f"Performing callback to job {jobID}") res = JobStateUpdateClient().updateJobFromStager(jobID, "Done") diff --git a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/test/Test_ArchiveFiles.py b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/test/Test_ArchiveFiles.py index 7a5208d9258..8c10094625a 100644 --- a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/test/Test_ArchiveFiles.py +++ b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/test/Test_ArchiveFiles.py @@ -163,10 +163,7 @@ def test_run_IgnoreMissingFiles(archiveFiles, _myMocker, listOfLFNs): ) for index, opFile in enumerate(archiveFiles.operation): LOG.debug("%s", opFile) # lazy evaluation of the argument! - if index == 5: - assert opFile.Status == "Done" - else: - assert opFile.Status == "Done" + assert opFile.Status == "Done" def test_checkFilePermissions(archiveFiles, _myMocker): diff --git a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/test/Test_ReplicateAndRegister.py b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/test/Test_ReplicateAndRegister.py new file mode 100644 index 00000000000..51a9da6548c --- /dev/null +++ b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/test/Test_ReplicateAndRegister.py @@ -0,0 +1,692 @@ +""" That is probably one of the most convoluted unit test file we have... +It could have been done in a simpler way, but I want to use this test as a "doc" +as I am sure this trick will show useful one day... + +The function to be tested is filterReplicas. The feature I am adding when writing these +tests is a cache: instead of calling getReplicas everytime in the filterReplicas function, +I call it once before for all the files, and pass the result to filterReplicas. + +In order to test this new feature, I want to be able to repeat each test, once with the cache, +and once without. +This could easily be done with a fixture, where I would call getReplicas if needed. + +But what if I want to call getReplicas with ALL the lfns at once, and use that result +(just like I would do in real life) ? If I want to avoid keeping a static list of LFNs +up to date, I have to collect it dynamically. And this is what I do + +Another interesting feature of this test is that I am basing the behavior of the mocks +(getReplicas, getFileMetadata, etc) on the LFN itself. +""" +import errno +import json +import os +import re +import pytest +import DIRAC + +from abc import ABC, abstractmethod + +from zlib import adler32 + +from DIRAC import S_OK, S_ERROR +from DIRAC.DataManagementSystem.Agent.RequestOperations.ReplicateAndRegister import filterReplicas + +from DIRAC.RequestManagementSystem.Client.File import File + + +# Although we will in general use regular expression to drive the behavior +# of the mocks, there are a few very specific files. +# So we may as well just use them as constant. +# They are mostly cases that return S_ERROR straight away.... + +# LFN for a non existing file +NON_EXISTING_LFN = "/lhcb/i/dont/exist.txt" + +# Complete failure of getting replicas +ERROR_GETTING_REPLICAS = "/lhcb/S_ERROR/replicas" + +# The file simply has no replicas available +NO_AVAILABLE_REPLICAS = "/lhcb/no/available/replicas" + + +# Complete failure of getting FC metadata +ERROR_GETTING_FC_METADATA = "/lhcb/S_ERROR/fc_metadata" + +# Complete failure of getting SE metadata +ERROR_GETTING_SE_METADATA = "/lhcb/S_ERROR/se_metadata" + +# SE at which we will find the files by default +DEFAULT_SE = "DiskSE" + + +# Now starts the regular expression magic. +# The idea is that the LFN will be structured like +# /MARKER_XXX/value1/value2/END/MARKER_YYY/value3/END... +# We split the LFN based on the marker and end mark, +# and each mock looks for the marker it cares about +# (for example MARKER_BAD_SE_REPLICAS for the mock_SE_getReplicas) + +# The file replicas at given SEs +# The "Disk" and "Tape" part are interpreted as such +MARKER_WITH_REPLICAS = "WITH_REPLICAS" + +# To declare that the specific replica has wrong cks +MARKER_BAD_SE_REPLICAS = "BAD_SE_REPLICAS" + +# The cks is wrong in the FC +MARKER_BAD_FC_REPLICAS = "BAD_FC_REPLICAS" + +# The file physicaly does not exist but is registered +MARKER_BAD_SE_NOT_EXIST = "BAD_SE_NOT_EXIST" + +# The end marker, same for all +END_MARKER = "END" + +# Collect all the markers to build the regular expression +ALL_MARKERS = "|".join(v for k, v in globals().items() if k.startswith("MARKER_")) +# That is the regular expression to split the LFN +# /MARKER1/val1/val2/END/MARKER2/val3/END... +MARKER_PATTERN = rf"/*(?P({ALL_MARKERS})+)/(?P.*)/" + + +def _splitLFN(lfn): + """Split the LFN based on the pattern""" + instructions = {} + try: + # Split at each end marker + for group in re.split(rf"{END_MARKER}", lfn): + if not group: + continue + + # interpret the specific pattern + matchgroup = re.match(MARKER_PATTERN, group).groupdict() + instructions[matchgroup["pat"]] = matchgroup["values"] + except Exception: + pass + return instructions + + +def mock_DM_getReplicas(self, lfns, **kwargs): + """Mock the DataManager.getReplicas method. + + It returns based on the LFN. + See the code for details. + + :param preferDisk: only return disk replicas if any + + """ + if isinstance(lfns, str): + lfns = [lfns] + + successful = {} + failed = {} + + for lfn in lfns: + instructions = _splitLFN(lfn) + # If the lfn does not exist, put it in Failed dict + if lfn == NON_EXISTING_LFN: + failed[lfn] = os.strerror(errno.ENOENT) + # Return a complete failure if it is expected (ERROR_GETTING_REPLICAS) + elif lfn == ERROR_GETTING_REPLICAS: + return S_ERROR(f"Complete failure {lfn}") + elif lfn == NO_AVAILABLE_REPLICAS: + continue + # If the LFN specifies the replicas + elif MARKER_WITH_REPLICAS in instructions: + ses = instructions[MARKER_WITH_REPLICAS].split("/") + # If we have the preferDisk flag, and we have disk replicas, + # only return disk storage, otherwise return them all + if kwargs.get("preferDisk"): + if any("Disk" in se for se in ses): + ses = [se for se in ses if "Disk" in se] + + successful[lfn] = dict.fromkeys(ses) + + else: + successful[lfn] = {DEFAULT_SE: None} + return S_OK({"Successful": successful, "Failed": failed}) + + +def mock_FC_getFileMetadata(self, lfns, **kwargs): + """ + Return the FC metadata of the files. + The returned checksum just corresponds to the adler32 of the LFN + See code for details + """ + + if isinstance(lfns, str): + lfns = [lfns] + + successful = {} + failed = {} + for lfn in lfns: + if lfn == ERROR_GETTING_FC_METADATA: + S_ERROR(f"Complete failure {lfn}") + else: + successful[lfn] = {"Size": len(lfn), "Checksum": adler32(lfn.encode())} + + return S_OK({"Successful": successful, "Failed": failed}) + + +def mock_SE_getFileMetadata(self, lfns, **kwargs): + """ + Return the SE metadata of the files. + See code for details + """ + if isinstance(lfns, str): + lfns = [lfns] + + successful = {} + failed = {} + for lfn in lfns: + instructions = _splitLFN(lfn) + # If a complete failure, return + if lfn == ERROR_GETTING_SE_METADATA: + return S_ERROR(f"Complete failure {lfn}") + # if a replicas is marked as not existing, return + # the specific No such file or directory error + elif MARKER_BAD_SE_NOT_EXIST in instructions: + badSEs = instructions[MARKER_BAD_SE_NOT_EXIST].split("/") + if self.name in badSEs: + failed[lfn] = os.strerror(errno.ENOENT) + continue + # If a replicas is marked as bad, return a wrong checksum + elif MARKER_BAD_SE_REPLICAS in instructions: + badSEs = instructions[MARKER_BAD_SE_REPLICAS].split("/") + if self.name in badSEs: + successful[lfn] = {"Size": len(lfn), "Checksum": adler32(b"bad!")} + continue + + successful[lfn] = {"Size": len(lfn), "Checksum": adler32(lfn.encode())} + + return S_OK({"Successful": successful, "Failed": failed}) + + +@pytest.fixture(scope="function", autouse=True) +def monkeypatchForAllTest(monkeypatch): + """This fixture will run for all test methods and will mock + a few DMS methods + """ + monkeypatch.setattr( + DIRAC.DataManagementSystem.Client.DataManager.DataManager, + "getReplicas", + mock_DM_getReplicas, + ) + monkeypatch.setattr( + DIRAC.Resources.Catalog.FileCatalog.FileCatalog, "getFileMetadata", mock_FC_getFileMetadata, raising=False + ) + monkeypatch.setattr( + DIRAC.Resources.Storage.StorageElement.StorageElementItem, + "getFileMetadata", + mock_SE_getFileMetadata, + raising=False, + ) + + +def _compareFileAttr(stateBefore, stateAfter, attrExpectedToDiffer): + """Given a json dump of the state before and after, + make sure that the attributes that changed are those expected + """ + setBefore = set(json.loads(stateBefore["Value"]).items()) + setAfter = set(json.loads(stateAfter["Value"]).items()) + assert {t[0] for t in setBefore ^ setAfter} == attrExpectedToDiffer + + +def runAsPytest(cls): + """ + Wizardry + This is a class decorator. It has several purposes: + + * allows pytest to discover the test classes (see below) by turning a class + to a function (a callable class does not cut it for pytest) + * parametrize all the tests (with or without the replicas cache) + * implements the test logic (prepare, execute filterReplicas, assert results) + * collects all the LFNs of all the tests + """ + + # instantiate the test class + clsInst = cls() + + # Parametrize the function + @pytest.mark.parametrize("withReplicasCache", [True, False]) + def func(withReplicasCache): + # Get the rmsFile to test + rmsFile = clsInst.initialize_test() + # Get the activeReplicas cache (or None...) + activeReplicas = getattr(runAsPytest, "allValidReplicas") if withReplicasCache else None + + # Call the filterReplicas function on the rmsFile, and specify the kwargs + # specific to the test case (like opSources) + res = filterReplicas(rmsFile, activeReplicas=activeReplicas, **clsInst.filterReplicasKwargs) + + # Check the test results + clsInst.check_result(res) + + # We keep the instantiated class as an attribute + # of the decorator. + # This is used in test_all + func.inner_cls = clsInst + + # Here we collect all the LFNs in each test and add them to the set + # "allLFNs", that we keep as a runAsPytest function attribute. + # This part is executed upon import, so by the time the tests actually + # execute, the list of LFN is complete + # The replicas cache is built from this list of LFN at the end of the file + # + # Extra note about: + # {clsInst.lfn} if isinstance(clsInst.lfn, str) else set(clsInst.lfn) + # this is not needed here because each test case is for a single LFN, + # but there could be test cases with multiple LFNs, in which case this is needed + # (see test_case_all below ) + setattr( + runAsPytest, + "allLFNs", + getattr(runAsPytest, "allLFNs", set()) | {clsInst.lfn} if isinstance(clsInst.lfn, str) else set(clsInst.lfn), + ) + + return func + + +class BaseTestCase(ABC): + """ + This is the base class for all our tests. These classes are meant to be + instantiated, and the logic executed by the runAsPytest decorator + """ + + # kwargs to pass to filterReplicas + filterReplicasKwargs = {} + # The rmsFile generated by the test + rmsFile = None + + # The lfn that will be used for the rmsFile + # It must be a class attribute to be availabe + # for runAsPytest decorator to collect it + lfn = None + + # A json dump of the state of the rmsFile + # before going through the filterReplicas function + stateBefore = None + + def initialize_test(self): + """ + Create the rmsFile and store the following attribute + + :returns: the rmsFile + """ + self.rmsFile = File() + self.rmsFile.LFN = self.lfn + self.stateBefore = self.rmsFile.toJSON() + return self.rmsFile + + @abstractmethod + def check_result(self, res): + """This takes the output of filterReplicas + and perform whichever assert it expects + """ + ... + + +@runAsPytest +class test_case_nonExistingFile(BaseTestCase): + """Test case of a file that does not exist""" + + lfn = NON_EXISTING_LFN + + def check_result(self, res): + assert not res["OK"] + assert self.rmsFile.Status == "Failed" + assert self.rmsFile.Error == os.strerror(errno.ENOENT) + + +# We don't run that one as a parametric case, +# as the failure of getting replicas would show up +# before calling filterReplicas in case we use the +# replicas cache +def test_errorGettingReplicas(): + """When we have a complete failure of getting the replicas""" + rmsFile = File() + rmsFile.LFN = ERROR_GETTING_REPLICAS + stateBefore = rmsFile.toJSON() + res = filterReplicas(rmsFile) + stateAfter = rmsFile.toJSON() + + # The failure should be transmitted + assert not res["OK"] + assert ERROR_GETTING_REPLICAS in res["Message"] + # Make sure the File status did not change + assert stateBefore == stateAfter + + +@runAsPytest +class test_case_errorGettingFCMetadata(BaseTestCase): + """When we have a complete failure of getting the file metadata from FC, + we just keep on going + """ + + lfn = ERROR_GETTING_FC_METADATA + + def check_result(self, res): + stateAfter = self.rmsFile.toJSON() + + assert res["OK"] + + # We should have a Valid replicas + assert DEFAULT_SE in res["Value"]["Valid"] + + # Since the rest of the function goes fine + # and we can get the checksum through the SE, + # the state does change: checksum and checksum types are set + _compareFileAttr(self.stateBefore, stateAfter, {"ChecksumType", "Checksum"}) + + +@runAsPytest +class test_case_fileHasTwoOKReplicas(BaseTestCase): + """Here we test that if a file has two replicas that + are fine, we get them both as option""" + + lfn = os.path.join("/", MARKER_WITH_REPLICAS, "Disk1", "Disk2", END_MARKER) + + def check_result(self, res): + stateAfter = self.rmsFile.toJSON() + assert res["OK"] + assert sorted(res["Value"]["Valid"]) == sorted(["Disk1", "Disk2"]) + + _compareFileAttr(self.stateBefore, stateAfter, {"ChecksumType", "Checksum"}) + + +@runAsPytest +class test_case_filterOutReplicas(BaseTestCase): + """We test that the filtering works""" + + # Take a file with two replicas, and restrict the source to a single one + lfn = os.path.join("/", MARKER_WITH_REPLICAS, "Disk1", "Disk2", END_MARKER) + + def initialize_test(self): + rmsFile = super().initialize_test() + self.filterReplicasKwargs = {"opSources": ["Disk1"]} + return rmsFile + + def check_result(self, res): + stateAfter = self.rmsFile.toJSON() + assert res["OK"] + assert ["Disk1"] == res["Value"]["Valid"] + _compareFileAttr(self.stateBefore, stateAfter, {"ChecksumType", "Checksum"}) + + +@runAsPytest +class test_case_filterOutImpossibleReplicas(BaseTestCase): + """Restrict the replicas to none of the possibility + The Valid replicas will be empty, + while all the others will be considered as NoActiveReplicas + """ + + # Take a file with two replicas, and restrict the source to a third one + lfn = os.path.join("/", MARKER_WITH_REPLICAS, "Disk1", "Disk2", END_MARKER) + + def initialize_test(self): + rmsFile = super().initialize_test() + self.filterReplicasKwargs = {"opSources": ["Disk3"]} + return rmsFile + + def check_result(self, res): + stateAfter = self.rmsFile.toJSON() + + assert res["OK"] + assert res["Value"]["Valid"] == [] + assert set(res["Value"]["NoActiveReplicas"]) == {"Disk1", "Disk2"} + + # The attributes should still be updated + _compareFileAttr(self.stateBefore, stateAfter, {"ChecksumType", "Checksum"}) + + +@runAsPytest +class test_case_rmsFileChecksumDifferentFromSE(BaseTestCase): + """If the RMS File has a checksum different from the one in the SE, + the replicas at that SE should be marked as bad + """ + + lfn = os.path.join("/", MARKER_WITH_REPLICAS, "Disk1", END_MARKER) + + def initialize_test(self): + rmsFile = super().initialize_test() + # We specify a checksum different from the SE + rmsFile.Checksum = adler32(b"notthegoodone") + self.stateBefore = rmsFile.toJSON() + return rmsFile + + def check_result(self, res): + stateAfter = self.rmsFile.toJSON() + + assert res["OK"] + # The replicas should be considered bad + assert res["Value"]["Bad"] == ["Disk1"] + # No changes should have happened to the file + + assert self.stateBefore == stateAfter + + +@runAsPytest +class test_case_fileHasMultipleBadReplicas(BaseTestCase): + """One file has multiple replicas, and all of them are bad""" + + # We want two replicas at disk1 and disk2 + lfnParts = ["/", MARKER_WITH_REPLICAS, "Disk1", "Disk2", END_MARKER] + + # all of which are bad + lfnParts += [MARKER_BAD_SE_REPLICAS, "Disk1", "Disk2", END_MARKER] + + lfn = os.path.join(*lfnParts) + + def check_result(self, res): + stateAfter = self.rmsFile.toJSON() + + assert res["OK"] + # The replicas should be considered bad + assert res["Value"]["Bad"] == ["Disk1", "Disk2"] + + # We should have added the catalog checksum + _compareFileAttr(self.stateBefore, stateAfter, {"ChecksumType", "Checksum"}) + + assert self.rmsFile.Checksum == adler32(self.rmsFile.LFN.encode()) + + +@runAsPytest +class test_case_fileHasOneGoodAndOneBadReplicas(BaseTestCase): + """The file has one good and one bad replicas""" + + # We want two replicas at disk1 and disk2 + lfnParts = ["/", MARKER_WITH_REPLICAS, "Disk1", "Disk2", END_MARKER] + + # one of which are bad + lfnParts += [MARKER_BAD_SE_REPLICAS, "Disk1", END_MARKER] + + lfn = os.path.join(*lfnParts) + + def check_result(self, res): + stateAfter = self.rmsFile.toJSON() + + assert res["OK"] + # The replicas at Disk1 should be considered bad + assert res["Value"]["Bad"] == ["Disk1"] + # The replicas at Disk2 should be considered valid + assert res["Value"]["Valid"] == ["Disk2"] + + # We should have added the catalog checksum + _compareFileAttr(self.stateBefore, stateAfter, {"ChecksumType", "Checksum"}) + assert self.rmsFile.Checksum == adler32(self.rmsFile.LFN.encode()) + + +@runAsPytest +class test_case_fileHasDiskAndTapeReplicas(BaseTestCase): + """The file has two disks and one tape replicas, we should favor the disk""" + + lfnParts = ["/", MARKER_WITH_REPLICAS, "Disk1", "Disk2", "Disk3", "Tape1", END_MARKER] + + # Assume disk2 is bad + lfnParts += [MARKER_BAD_SE_REPLICAS, "Disk3", END_MARKER] + + lfn = os.path.join(*lfnParts) + + def check_result(self, res): + stateAfter = self.rmsFile.toJSON() + + assert res["OK"] + + # We only want the disk replicas + assert res["Value"]["Valid"] == ["Disk1", "Disk2"] + assert res["Value"]["Bad"] == ["Disk3"] + + # We should have added the catalog checksum + _compareFileAttr(self.stateBefore, stateAfter, {"ChecksumType", "Checksum"}) + assert self.rmsFile.Checksum == adler32(self.rmsFile.LFN.encode()) + + +@runAsPytest +class test_case_fileHasOnlyTapeReplicas(BaseTestCase): + """The file has only tape replicas, we should ues them""" + + lfnParts = ["/", MARKER_WITH_REPLICAS, "Tape1", "Tape2", END_MARKER] + + lfn = os.path.join(*lfnParts) + + def check_result(self, res): + stateAfter = self.rmsFile.toJSON() + + assert res["OK"] + + assert res["Value"]["Valid"] == ["Tape1", "Tape2"] + + # We should have added the catalog checksum + _compareFileAttr(self.stateBefore, stateAfter, {"ChecksumType", "Checksum"}) + + assert self.rmsFile.Checksum == adler32(self.rmsFile.LFN.encode()) + + +@pytest.mark.xfail(reason="https://github.com/DIRACGrid/DIRAC/issues/6689") +@runAsPytest +class test_case_fileHasDiskReplicasButWeFilterOnTape(BaseTestCase): + """If we have a disk and tape replicas, but specify the tape replicas as source, + we want the tape one + """ + + lfn = os.path.join("/", MARKER_WITH_REPLICAS, "Disk1", "Tape1", END_MARKER) + + def initialize_test(self): + rmsFile = super().initialize_test() + self.filterReplicasKwargs = {"opSources": ["Tape1"]} + return rmsFile + + def check_result(self, res): + stateAfter = self.rmsFile.toJSON() + + assert res["OK"] + assert res["Value"]["Valid"] == ["Tape1"] + + _compareFileAttr(self.stateBefore, stateAfter, {"ChecksumType", "Checksum"}) + + +@runAsPytest +class test_case_noActiveReplicas(BaseTestCase): + """The file has no replicas available""" + + lfn = NO_AVAILABLE_REPLICAS + + def check_result(self, res): + stateAfter = self.rmsFile.toJSON() + + assert res["OK"] + # When no replicas are available, we just get "None" + assert res["Value"]["NoReplicas"] == [None] + + # We should have added the catalog checksum + _compareFileAttr(self.stateBefore, stateAfter, {"ChecksumType", "Checksum"}) + + assert self.rmsFile.Checksum == adler32(self.rmsFile.LFN.encode()) + + +@runAsPytest +class test_case_errorGettingSEMetadata(BaseTestCase): + """When we have a complete failure of getting the file metadata from SE, + the files goes to "NoMetadata" + """ + + lfn = ERROR_GETTING_SE_METADATA + + def check_result(self, res): + stateAfter = self.rmsFile.toJSON() + + assert res["OK"] + + # We should have a Valid replicas + assert DEFAULT_SE in res["Value"]["NoMetadata"] + + # We got the checksum from the FC + + _compareFileAttr(self.stateBefore, stateAfter, {"ChecksumType", "Checksum"}) + + +@runAsPytest +class test_case_fileRegisteredButDoesNotPhysicalyExist(BaseTestCase): + """File has two replicas registered, but one of them does not exist""" + + # Two replicas + lfnParts = ["/", MARKER_WITH_REPLICAS, "Disk1", "Disk2", END_MARKER] + # but it doesn't exist + lfnParts += [MARKER_BAD_SE_NOT_EXIST, "Disk2", END_MARKER] + + lfn = os.path.join(*lfnParts) + + def check_result(self, res): + stateAfter = self.rmsFile.toJSON() + + assert res["OK"] + + assert res["Value"]["Valid"] == ["Disk1"] + assert res["Value"]["NoReplicas"] == ["Disk2"] + # We should have added the catalog checksum + _compareFileAttr(self.stateBefore, stateAfter, {"ChecksumType", "Checksum"}) + + +@pytest.mark.xfail(reason="filterReplicas does not support bulk") +@runAsPytest +class test_all(BaseTestCase): + """ + This test re-run all the other tests, but by giving all the LFNs at once. + I wanted to make filterReplicas a bulk method (not a single rmsFile, but multiple), + so this would have been a neat way of testing it. + I've decided not to do it in the end, but I keep the test here, again for doc purpose + (It could run as a standard pytest, but it is so elegant to re-use the + BaseTestClass and runAsPytest decorator...) + """ + + # We collect all the test cases that were decorated with @runAsPytest + # and store the underlying class instance + allTestCases = [getattr(v, "inner_cls") for k, v in globals().items() if k.startswith("test_case")] + + # Use all the LFNs as input for filterReplicas + lfn = getattr(runAsPytest, "allLFNs") + + def initialize_test(self): + # Initialize all the rmsFiles + self.allRmsFiles = [x.initialize_test() for x in self.allTestCases] + # Store all the json states + self.allStateBefore = [rf.toJSON() for rf in self.allRmsFiles] + return self.allRmsFiles + + def check_result(self, res): + # Here res would be a bulk output like Successful/Failed + # So you would need to map the entry in it to one of the + # instance in allTestCases (you could do a dict indexed by the LFN) + # And you would call check_result on each of them + raise NotImplementedError + + +# Here we store the output of getReplicas as the allValidReplicas attribute +# of the runAsPytest decorator. +# By the time we reach here, all the tests with decorator have been parsed, +# so the list of LFNs is complete +setattr( + runAsPytest, + "allValidReplicas", + mock_DM_getReplicas(None, getattr(runAsPytest, "allLFNs"), preferDisk=True)["Value"], +) diff --git a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/test/Test_RequestOperations.py b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/test/Test_RequestOperations.py index 8452a3c2608..6acada64316 100644 --- a/src/DIRAC/DataManagementSystem/Agent/RequestOperations/test/Test_RequestOperations.py +++ b/src/DIRAC/DataManagementSystem/Agent/RequestOperations/test/Test_RequestOperations.py @@ -31,8 +31,8 @@ class MoveReplicaSuccess(ReqOpsTestCase): def setUp(self): self.op = Operation() self.op.Type = "MoveFile" - self.op.SourceSE = "{},{}".format("sourceSE1", "sourceSE2") - self.op.TargetSE = "{},{}".format("targetSE1", "targetSE2") + self.op.SourceSE = "sourceSE1,sourceSE2" + self.op.TargetSE = "targetSE1,targetSE2" self.File = File() self.File.LFN = "/cta/file1" @@ -69,7 +69,6 @@ def setUp(self): self.assertEqual( self.mr.request.Status, 'Waiting' )""" def test__dmRemoval(self): - res = {"OK": True, "Value": {"Successful": {self.File.LFN: {"DIRACFileCatalog": True}}, "Failed": {}}} self.mr.dm.removeReplica.return_value = res @@ -91,8 +90,8 @@ class MoveReplicaFailure(ReqOpsTestCase): def setUp(self): self.op = Operation() self.op.Type = "MoveReplica" - self.op.SourceSE = "{},{}".format("sourceSE1", "sourceSE2") - self.op.TargetSE = "{},{}".format("targetSE1", "targetSE2") + self.op.SourceSE = "sourceSE1,sourceSE2" + self.op.TargetSE = "targetSE1,targetSE2" self.File = File() self.File.LFN = "/cta/file1" @@ -110,7 +109,6 @@ def setUp(self): self.mr.ci = MagicMock() def test__dmTransfer(self): - successful = {} for sourceSE in self.op.sourceSEList: successful[sourceSE] = "dips://" + sourceSE.lower() + ":9148/DataManagement/StorageElement" + self.File.LFN @@ -141,7 +139,6 @@ def test__dmTransfer(self): self.assertEqual(self.mr.request.Status, "Waiting") def test__dmRemoval(self): - res = { "OK": True, "Value": {"Successful": {}, "Failed": {self.File.LFN: "Write access not permitted for this credential"}}, diff --git a/src/DIRAC/DataManagementSystem/Client/CmdDirCompletion/DirectoryCompletion.py b/src/DIRAC/DataManagementSystem/Client/CmdDirCompletion/DirectoryCompletion.py index 6ba294f9d5c..5a3c6978856 100644 --- a/src/DIRAC/DataManagementSystem/Client/CmdDirCompletion/DirectoryCompletion.py +++ b/src/DIRAC/DataManagementSystem/Client/CmdDirCompletion/DirectoryCompletion.py @@ -44,18 +44,13 @@ def parse_text_line(self, text, line, cwd): # check absolute path def check_absolute(self, path): - if path.startswith(self.fs.seq): - return True - else: - return False + return path.startswith(self.fs.seq) # generate absolute path def generate_absolute(self, path, cwd): if self.check_absolute(path): - pass - else: - path = os.path.join(cwd, path) - return path + return path + return os.path.join(cwd, path) # get the parent directory or the current directory # Using the last char "/" to determine diff --git a/src/DIRAC/DataManagementSystem/Client/ConsistencyInspector.py b/src/DIRAC/DataManagementSystem/Client/ConsistencyInspector.py index 85ec237a701..1865c737dd3 100644 --- a/src/DIRAC/DataManagementSystem/Client/ConsistencyInspector.py +++ b/src/DIRAC/DataManagementSystem/Client/ConsistencyInspector.py @@ -79,7 +79,7 @@ def __init__(self, interactive=True, transClient=None, dm=None, fc=None, dic=Non def __logVerbose(self, msg, msg1=""): """logger helper for verbose information""" if self._verbose: - newMsg = "[ConsistencyChecks] " + ("[%s] " % str(self.prod)) if self.prod else "" + newMsg = "[ConsistencyChecks] " + f"[{str(self.prod)}] " if self.prod else "" # Add that prefix to all lines of the message newMsg1 = msg1.replace("\n", "\n" + newMsg) newMsg += msg.replace("\n", "\n" + newMsg) @@ -121,7 +121,7 @@ def getReplicasPresence(self, lfns): break else: time.sleep(0.1) - self.__write(" (%.1f seconds)\n" % (time.time() - startTime)) + self.__write(f" ({time.time() - startTime:.1f} seconds)\n") if notPresent: self.__logVerbose("Files without replicas:", "\n".join([""] + sorted(notPresent))) @@ -144,16 +144,16 @@ def getReplicasPresenceFromDirectoryScan(self, lfns): dirs.setdefault(dirN, []).append(lfn) if compare: - self.__write("Checking File Catalog for %d files from %d directories " % (len(lfns), len(dirs))) + self.__write(f"Checking File Catalog for {len(lfns)} files from {len(dirs)} directories ") else: - self.__write("Getting files from %d directories " % len(dirs)) + self.__write(f"Getting files from {len(dirs)} directories ") startTime = time.time() for dirN in sorted(dirs): startTime1 = time.time() self.__write(".") lfnsFound = self._getFilesFromDirectoryScan(dirN) - gLogger.verbose("Obtained %d files in %.1f seconds" % (len(lfnsFound), time.time() - startTime1)) + gLogger.verbose(f"Obtained {len(lfnsFound)} files in {time.time() - startTime1:.1f} seconds") if compare: pr, notPr = self.__compareLFNLists(dirs[dirN], lfnsFound) notPresent += notPr @@ -161,8 +161,8 @@ def getReplicasPresenceFromDirectoryScan(self, lfns): else: present += lfnsFound - self.__write(" (%.1f seconds)\n" % (time.time() - startTime)) - gLogger.info("Found %d files with replicas and %d without" % (len(present), len(notPresent))) + self.__write(f" ({time.time() - startTime:.1f} seconds)\n") + gLogger.info(f"Found {len(present)} files with replicas and {len(notPresent)} without") return present, notPresent ########################################################################## @@ -172,13 +172,13 @@ def __compareLFNLists(self, lfns, lfnsFound): present = [] notPresent = lfns startTime = time.time() - self.__logVerbose("Comparing list of %d LFNs with second list of %d" % (len(lfns), len(lfnsFound))) + self.__logVerbose(f"Comparing list of {len(lfns)} LFNs with second list of {len(lfnsFound)}") if lfnsFound: setLfns = set(lfns) setLfnsFound = set(lfnsFound) present = list(setLfns & setLfnsFound) notPresent = list(setLfns - setLfnsFound) - self.__logVerbose("End of comparison: %.1f seconds" % (time.time() - startTime)) + self.__logVerbose(f"End of comparison: {time.time() - startTime:.1f} seconds") return present, notPresent def _getFilesFromDirectoryScan(self, dirs): @@ -190,7 +190,7 @@ def _getFilesFromDirectoryScan(self, dirs): gLogger.setLevel(level) if not res["OK"]: if "No such file or directory" not in res["Message"]: - gLogger.error("Error getting files from directories %s:" % dirs, res["Message"]) + gLogger.error(f"Error getting files from directories {dirs}:", res["Message"]) return [] if res["Value"]: lfnsFound = res["Value"] @@ -216,7 +216,7 @@ def _getTSFiles(self): self.runsList.extend( [run["RunNumber"] for run in res["Value"] if run["RunNumber"] not in self.runsList] ) - gLogger.notice("%d runs selected" % len(res["Value"])) + gLogger.notice(f"{len(res['Value'])} runs selected") elif not self.runsList: gLogger.notice("No runs selected, check completed") DIRAC.exit(0) @@ -225,7 +225,7 @@ def _getTSFiles(self): res = self.transClient.getTransformation(self.prod) if not res["OK"]: - gLogger.error("Failed to find transformation %s" % self.prod) + gLogger.error(f"Failed to find transformation {self.prod}") return [], [], [] status = res["Value"]["Status"] if status not in ("Active", "Stopped", "Completed", "Idle"): @@ -362,20 +362,20 @@ def compareChecksum(self, lfns): self.__write(".") replicasRes = self.fileCatalog.getReplicas(lfnChunk) if not replicasRes["OK"]: - gLogger.error("error: %s" % replicasRes["Message"]) - return S_ERROR(errno.ENOENT, "error: %s" % replicasRes["Message"]) + gLogger.error(f"error: {replicasRes['Message']}") + return S_ERROR(errno.ENOENT, f"error: {replicasRes['Message']}") replicasRes = replicasRes["Value"] if replicasRes["Failed"]: retDict["NoReplicas"].update(replicasRes["Failed"]) replicas.update(replicasRes["Successful"]) - self.__write("Get FC metadata for %d files to be checked: " % len(lfns)) + self.__write(f"Get FC metadata for {len(lfns)} files to be checked: ") metadata = {} for lfnChunk in breakListIntoChunks(replicas, chunkSize): self.__write(".") res = self.fileCatalog.getFileMetadata(lfnChunk) if not res["OK"]: - return S_ERROR(errno.ENOENT, "error %s" % res["Message"]) + return S_ERROR(errno.ENOENT, f"error {res['Message']}") metadata.update(res["Value"]["Successful"]) gLogger.notice("Check existence and compare checksum file by file...") @@ -403,7 +403,7 @@ def compareChecksum(self, lfns): self.__write(".") metadata = oSe.getFileMetadata(surlChunk) if not metadata["OK"]: - gLogger.error("Error: getFileMetadata returns %s. Ignore those replicas" % (metadata["Message"])) + gLogger.error(f"Error: getFileMetadata returns {metadata['Message']}. Ignore those replicas") # Remove from list of replicas as we don't know whether it is OK or # not for lfn in seFiles[se]: @@ -420,7 +420,7 @@ def compareChecksum(self, lfns): gLogger.setLevel(logLevel) - gLogger.notice("Verifying checksum of %d files" % len(replicas)) + gLogger.notice(f"Verifying checksum of {len(replicas)} files") for lfn in replicas: # get the lfn checksum from the FC replicaDict = replicas[lfn] @@ -588,7 +588,7 @@ def checkPhysicalFiles(self, replicas, catalogMetadata, ses=None): if (ses) and (se not in ses): continue seLfns.setdefault(se, []).append(lfn) - gLogger.info("{} {}".format("Storage Element".ljust(20), "Replicas".rjust(20))) + gLogger.info(f"{'Storage Element'.ljust(20)} {'Replicas'.rjust(20)}") for se in sorted(seLfns): files = len(seLfns[se]) @@ -643,7 +643,7 @@ def __checkPhysicalFileMetadata(self, lfns, se): self.dic.reportProblematicReplicas(unavailableReplicas, se, "PFNUnavailable") if zeroSizeReplicas: self.dic.reportProblematicReplicas(zeroSizeReplicas, se, "PFNZeroSize") - gLogger.info("Checking the integrity of physical files at %s complete" % se) + gLogger.info(f"Checking the integrity of physical files at {se} complete") return S_OK(pfnMetadata) ########################################################################## @@ -658,22 +658,20 @@ def _getDirectoryContent(directory): """Inner function: recursively scan a directory, returns list of LFNs""" filesInDirectory = {} - gLogger.debug("Examining %s" % directory) + gLogger.debug(f"Examining {directory}") res = self.fileCatalog.listDirectory(directory) if not res["OK"]: gLogger.error("Failed to get directory contents", res["Message"]) return res if directory in res["Value"]["Failed"]: - gLogger.error( - "Failed to get directory content", "{} {}".format(directory, res["Value"]["Failed"][directory]) - ) + gLogger.error("Failed to get directory content", f"{directory} {res['Value']['Failed'][directory]}") return S_ERROR("Failed to get directory content") if directory not in res["Value"]["Successful"]: return S_ERROR("Directory not existing?") # first, adding the files found in the current directory - gLogger.debug("Files in %s: %d" % (directory, len(res["Value"]["Successful"][directory]["Files"]))) + gLogger.debug(f"Files in {directory}: {len(res['Value']['Successful'][directory]['Files'])}") filesInDirectory.update(res["Value"]["Successful"][directory]["Files"]) # then, looking for subDirectories content @@ -688,7 +686,7 @@ def _getDirectoryContent(directory): return S_OK(filesInDirectory) - gLogger.info("Obtaining the catalog contents for %d directories" % len(lfnDirs)) + gLogger.info(f"Obtaining the catalog contents for {len(lfnDirs)} directories") allFiles = {} for lfnDir in lfnDirs: @@ -696,10 +694,10 @@ def _getDirectoryContent(directory): if not dirContent["OK"]: return dirContent else: - gLogger.debug("Content of directory %s: %d files" % (lfnDir, len(dirContent["Value"]))) + gLogger.debug(f"Content of directory {lfnDir}: {len(dirContent['Value'])} files") allFiles.update(dirContent["Value"]) - gLogger.debug("Content of directories examined: %d files" % len(allFiles)) + gLogger.debug(f"Content of directories examined: {len(allFiles)} files") replicas = self.fileCatalog.getReplicas(list(allFiles)) if not replicas["OK"]: @@ -712,9 +710,9 @@ def _getDirectoryContent(directory): def _getCatalogReplicas(self, lfns): """Obtain the file replicas from the catalog while checking that there are replicas""" if not lfns: - return S_OK(([], [])) + return S_OK(({}, [])) - gLogger.info("Obtaining the replicas for %s files" % len(lfns)) + gLogger.info(f"Obtaining the replicas for {len(lfns)} files") zeroReplicaFiles = [] res = self.fileCatalog.getReplicas(lfns, allStatus=True) if not res["OK"]: @@ -735,7 +733,7 @@ def _getCatalogMetadata(self, lfns): if not lfns: return S_OK((allMetadata, missingCatalogFiles, zeroSizeFiles)) - gLogger.info("Obtaining the catalog metadata for %s files" % len(lfns)) + gLogger.info(f"Obtaining the catalog metadata for {len(lfns)} files") res = self.fileCatalog.getFileMetadata(lfns) if not res["OK"]: diff --git a/src/DIRAC/DataManagementSystem/Client/DataIntegrityClient.py b/src/DIRAC/DataManagementSystem/Client/DataIntegrityClient.py index 9e7e5b8d264..0258371a7d2 100755 --- a/src/DIRAC/DataManagementSystem/Client/DataIntegrityClient.py +++ b/src/DIRAC/DataManagementSystem/Client/DataIntegrityClient.py @@ -16,7 +16,6 @@ class DataIntegrityClient(Client): """Client exposing the DataIntegrity Service.""" def __init__(self, **kwargs): - super().__init__(**kwargs) self.setServer("DataManagement/DataIntegrity") self.dm = DataManager() @@ -38,7 +37,7 @@ def setFileProblematic(self, lfn, reason, sourceComponent=""): errStr = "DataIntegrityClient.setFileProblematic: Supplied file info must be list or a single LFN." gLogger.error(errStr) return S_ERROR(errStr) - gLogger.info("DataIntegrityClient.setFileProblematic: Attempting to update %s files." % len(lfns)) + gLogger.info(f"DataIntegrityClient.setFileProblematic: Attempting to update {len(lfns)} files.") fileMetadata = {} for lfn in lfns: fileMetadata[lfn] = {"Prognosis": reason, "LFN": lfn, "PFN": "", "SE": ""} @@ -80,7 +79,7 @@ def setReplicaProblematic(self, replicaTuple, sourceComponent=""): ) gLogger.error(errStr) return S_ERROR(errStr) - gLogger.info("DataIntegrityClient.setReplicaProblematic: Attempting to update %s replicas." % len(replicaTuple)) + gLogger.info(f"DataIntegrityClient.setReplicaProblematic: Attempting to update {len(replicaTuple)} replicas.") replicaDict = {} for lfn, pfn, se, reason in replicaTuple: replicaDict[lfn] = {"Prognosis": reason, "LFN": lfn, "PFN": pfn, "SE": se} diff --git a/src/DIRAC/DataManagementSystem/Client/DataManager.py b/src/DIRAC/DataManagementSystem/Client/DataManager.py index 77d8080653b..e2013471c89 100644 --- a/src/DIRAC/DataManagementSystem/Client/DataManager.py +++ b/src/DIRAC/DataManagementSystem/Client/DataManager.py @@ -8,6 +8,7 @@ This module consists of DataManager and related classes. """ + # # imports from datetime import datetime, timedelta import fnmatch @@ -25,12 +26,14 @@ from DIRAC.Core.Utilities.ReturnValues import returnSingleResult from DIRAC.Core.Security.ProxyInfo import getProxyInfo from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations +from DIRAC.DataManagementSystem.Client import MAX_FILENAME_LENGTH from DIRAC.MonitoringSystem.Client.DataOperationSender import DataOperationSender from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers from DIRAC.Resources.Catalog.FileCatalog import FileCatalog from DIRAC.Resources.Storage.StorageElement import StorageElement from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus + # # RSCID def _isOlderThan(stringTime, days): """Check if a time stamp is older than a given number of days""" @@ -139,7 +142,7 @@ def cleanLogicalDirectory(self, lfnDir): for folder in lfnDir: res = self.__cleanDirectory(folder) if not res["OK"]: - log.debug("Failed to clean directory.", "{} {}".format(folder, res["Message"])) + log.debug("Failed to clean directory.", f"{folder} {res['Message']}") retDict["Failed"][folder] = res["Message"] else: log.debug("Successfully removed directory.", folder) @@ -160,29 +163,27 @@ def __cleanDirectory(self, folder): errStr = "Write access not permitted for this credential." log.debug(errStr, folder) return S_ERROR(errStr) - res = self.__getCatalogDirectoryContents([folder], includeDirectories=True) + + res = returnSingleResult(self.fileCatalog.getDirectoryDump([folder])) + if not res["OK"]: return res - if not res["Value"]: + if not (res["Value"]["Files"] or res["Value"]["SubDirs"]): # folder is empty, just remove it and return res = returnSingleResult(self.fileCatalog.removeDirectory(folder, recursive=True)) return res # create a list of folders so that empty folders are also deleted - areDirs = self.fileCatalog.isDirectory(res["Value"]) - if not areDirs["OK"]: - return areDirs - listOfFolders = [aDir for aDir in areDirs["Value"]["Successful"] if areDirs["Value"]["Successful"][aDir]] - for lfn in listOfFolders: - res["Value"].pop(lfn) - - res = self.removeFile(res["Value"]) + listOfFolders = res["Value"]["SubDirs"] + listOfFiles = res["Value"]["Files"] + + res = self.removeFile(listOfFiles) if not res["OK"]: return res for lfn, reason in res["Value"]["Failed"].items(): # can be an iterator log.error("Failed to remove file found in the catalog", f"{lfn} {reason}") - res = returnSingleResult(self.removeFile(["%s/dirac_directory" % folder])) + res = returnSingleResult(self.removeFile([f"{folder}/dirac_directory"])) if not res["OK"]: if not DErrno.cmpError(res, errno.ENOENT): log.warn("Failed to delete dirac_directory placeholder file") @@ -238,34 +239,6 @@ def __removeStorageDirectory(self, directory, storageElement): ) return S_OK() - def __getCatalogDirectoryContents(self, directories, includeDirectories=False): - """ls recursively all files in directories - - :param self: self reference - :param list directories: folder names - :param bool includeDirectories: if True includes directories in the return dictionary - :return: S_OK with dict of LFNs and their attribute dictionary - """ - log = self.log.getSubLogger("__getCatalogDirectoryContents") - log.debug("Obtaining the catalog contents for %d directories:" % len(directories)) - activeDirs = directories - allFiles = {} - while len(activeDirs) > 0: - currentDir = activeDirs[0] - res = returnSingleResult(self.fileCatalog.listDirectory(currentDir, verbose=True)) - activeDirs.remove(currentDir) - - if not res["OK"]: - log.debug("Problem getting the %s directory content" % currentDir, res["Message"]) - else: - dirContents = res["Value"] - activeDirs.extend(dirContents["SubDirs"]) - allFiles.update(dirContents["Files"]) - if includeDirectories: - allFiles.update(dirContents["SubDirs"]) - log.debug("Found %d files" % len(allFiles)) - return S_OK(allFiles) - def getReplicasFromDirectory(self, directory): """get all replicas from a given directory @@ -276,11 +249,16 @@ def getReplicasFromDirectory(self, directory): directories = [directory] else: directories = directory - res = self.__getCatalogDirectoryContents(directories) + res = returnSingleResult(self.fileCatalog.getDirectoryDump(directories)) if not res["OK"]: return res - allReplicas = {lfn: metadata["Replicas"] for lfn, metadata in res["Value"].items()} # can be an iterator - return S_OK(allReplicas) + + lfns = res["Value"]["Files"] + res = self.fileCatalog.getReplicas(lfns, allStatus=True) + if not res["OK"]: + return res + res["Value"] = res["Value"]["Successful"] + return res def getFilesFromDirectory(self, directory, days=0, wildcard="*"): """get all files from :directory: older than :days: days matching to :wildcard: @@ -306,12 +284,12 @@ def getFilesFromDirectory(self, directory, days=0, wildcard="*"): res = returnSingleResult(self.fileCatalog.listDirectory(currentDir, verbose=(days != 0))) activeDirs.remove(currentDir) if not res["OK"]: - log.debug("Error retrieving directory contents", "{} {}".format(currentDir, res["Message"])) + log.debug("Error retrieving directory contents", f"{currentDir} {res['Message']}") else: dirContents = res["Value"] subdirs = dirContents["SubDirs"] files = dirContents["Files"] - log.debug("%s: %d files, %d sub-directories" % (currentDir, len(files), len(subdirs))) + log.debug(f"{currentDir}: {len(files)} files, {len(subdirs)} sub-directories") for subdir in subdirs: if (not days) or _isOlderThan(subdirs[subdir]["CreationDate"], days): if subdir[0] != "/": @@ -331,10 +309,14 @@ def getFilesFromDirectory(self, directory, days=0, wildcard="*"): # These are the data transfer methods # - def getFile(self, lfn, destinationDir="", sourceSE=None): - """Get a local copy of a LFN from Storage Elements. + def getFile(self, lfn, destinationDir="", sourceSE=None, diskOnly=False): + """Get local copy of LFN(s) from Storage Elements. - 'lfn' is the logical file name for the desired file + :param mixed lfn: a single LFN or list of LFNs. + :param str destinationDir: directory to which the file(s) will be downnloaded. (Default: current working directory). + :param str sourceSE: source SE from which to download (Default: all replicas will be attempted). + :param bool diskOnly: chooses the disk ONLY replica(s). (Default: False) + :return: S_OK({"Successful": {}, "Failed": {}})/S_ERROR(errMessage). """ log = self.log.getSubLogger("getFile") fileMetadata = {} @@ -346,8 +328,8 @@ def getFile(self, lfn, destinationDir="", sourceSE=None): errStr = "Supplied lfn must be string or list of strings." log.debug(errStr) return S_ERROR(errStr) - log.debug("Attempting to get %s files." % len(lfns)) - res = self.getActiveReplicas(lfns, getUrl=False) + log.debug(f"Attempting to get {len(lfns)} files.") + res = self.getActiveReplicas(lfns, getUrl=False, diskOnly=diskOnly) if not res["OK"]: return res failed = res["Value"]["Failed"] @@ -385,7 +367,7 @@ def __getFile(self, lfn, replicas, metadata, destinationDir, sourceSE=None): sortedSEs = self._getSEProximity(replicas) else: if sourceSE not in replicas: - return S_ERROR("No replica at %s" % sourceSE) + return S_ERROR(f"No replica at {sourceSE}") else: sortedSEs = [sourceSE] @@ -397,7 +379,7 @@ def __getFile(self, lfn, replicas, metadata, destinationDir, sourceSE=None): if not res["OK"]: errTuple = ( "Error getting file from storage:", - "{} from {}, {}".format(lfn, storageElementName, res["Message"]), + f"{lfn} from {storageElementName}, {res['Message']}", ) errToReturn = res else: @@ -414,7 +396,7 @@ def __getFile(self, lfn, replicas, metadata, destinationDir, sourceSE=None): elif (metadata["Checksum"]) and (not compareAdler(metadata["Checksum"], localAdler)): errTuple = ( "Mismatch of checksums:", - "downloaded = {}, catalog = {}".format(localAdler, metadata["Checksum"]), + f"downloaded = {localAdler}, catalog = {metadata['Checksum']}", ) errToReturn = S_ERROR(DErrno.EBADCKS, errTuple[1]) else: @@ -452,6 +434,9 @@ def putAndRegister(self, lfn, fileName, diracSE, guid=None, path=None, checksum= 'overwrite' removes file from the file catalogue and SE before attempting upload """ + if len(os.path.basename(lfn)) > MAX_FILENAME_LENGTH: + return S_ERROR(errno.ENAMETOOLONG, f"maximum {MAX_FILENAME_LENGTH} characters allowed") + res = self.__hasAccess("addFile", lfn) if not res["OK"]: return res @@ -486,7 +471,7 @@ def putAndRegister(self, lfn, fileName, diracSE, guid=None, path=None, checksum= log.debug("Checksum calculation failed, try again") checksum = fileAdler(fileName) if checksum: - log.debug("Checksum calculated to be %s." % checksum) + log.debug(f"Checksum calculated to be {checksum}.") else: return S_ERROR(DErrno.EBADCKS, "Unable to calculate checksum") @@ -514,13 +499,13 @@ def putAndRegister(self, lfn, fileName, diracSE, guid=None, path=None, checksum= else: errStr = "The supplied LFN already exists in the File Catalog." log.debug(errStr, lfn) - return S_ERROR("{} {}".format(errStr, res["Value"]["Successful"][lfn])) + return S_ERROR(f"{errStr} {res['Value']['Successful'][lfn]}") else: # If the returned LFN is different, this is the name of a file # with the same GUID errStr = "This file GUID already exists for another file" log.debug(errStr, res["Value"]["Successful"][lfn]) - return S_ERROR("{} {}".format(errStr, res["Value"]["Successful"][lfn])) + return S_ERROR(f"{errStr} {res['Value']['Successful'][lfn]}") ########################################################## # Instantiate the destination storage element here. @@ -528,8 +513,8 @@ def putAndRegister(self, lfn, fileName, diracSE, guid=None, path=None, checksum= res = storageElement.isValid() if not res["OK"]: errStr = "The storage element is not currently valid." - log.verbose(errStr, "{} {}".format(diracSE, res["Message"])) - return S_ERROR("{} {}".format(errStr, res["Message"])) + log.verbose(errStr, f"{diracSE} {res['Message']}") + return S_ERROR(f"{errStr} {res['Message']}") fileDict = {lfn: fileName} @@ -549,7 +534,6 @@ def putAndRegister(self, lfn, fileName, diracSE, guid=None, path=None, checksum= if not res["OK"]: # We don't consider it a failure if the SE is not valid if not DErrno.cmpError(res, errno.EACCES): - accountingDict["TransferOK"] = 0 accountingDict["FinalStatus"] = "Failed" sendingResult = self.dataOpSender.sendData( @@ -561,11 +545,11 @@ def putAndRegister(self, lfn, fileName, diracSE, guid=None, path=None, checksum= log.error("Couldn't commit data operation", sendingResult["Message"]) return sendingResult log.verbose("Done committing") - log.debug("putAndRegister: Sending took %.1f seconds" % (time.time() - transferStartTime)) + log.debug(f"putAndRegister: Sending took {time.time() - transferStartTime:.1f} seconds") errStr = "Failed to put file to Storage Element." - log.debug(errStr, "{}: {}".format(fileName, res["Message"])) - return S_ERROR("{} {}".format(errStr, res["Message"])) + log.debug(errStr, f"{fileName}: {res['Message']}") + return S_ERROR(f"{errStr} {res['Message']}") successful[lfn] = {"put": putTime} ########################################################### @@ -575,7 +559,7 @@ def putAndRegister(self, lfn, fileName, diracSE, guid=None, path=None, checksum= if not res["OK"]: errStr = "Failed to generate destination PFN." log.debug(errStr, res["Message"]) - return S_ERROR("{} {}".format(errStr, res["Message"])) + return S_ERROR(f"{errStr} {res['Message']}") destUrl = res["Value"] fileTuple = (lfn, destUrl, size, destinationSE, guid, checksum) @@ -585,7 +569,7 @@ def putAndRegister(self, lfn, fileName, diracSE, guid=None, path=None, checksum= "Size": size, "TargetSE": destinationSE, "GUID": guid, - "Addler": checksum, + "Adler": checksum, } startTime = time.time() res = self.registerFile(fileTuple) @@ -602,7 +586,7 @@ def putAndRegister(self, lfn, fileName, diracSE, guid=None, path=None, checksum= elif lfn in res["Value"]["Failed"]: errStr = "Failed to register file." - log.debug(errStr, "{} {}".format(lfn, res["Value"]["Failed"][lfn])) + log.debug(errStr, f"{lfn} {res['Value']['Failed'][lfn]}") accountingDict["FinalStatus"] = "Failed" failed[lfn] = {"register": registerDict} else: @@ -617,7 +601,7 @@ def putAndRegister(self, lfn, fileName, diracSE, guid=None, path=None, checksum= log.error("Couldn't commit data operation", sendingResult["Message"]) return sendingResult log.verbose("Done committing") - log.debug("putAndRegister: Sending took %.1f seconds" % (time.time() - startTime)) + log.debug(f"putAndRegister: Sending took {time.time() - startTime:.1f} seconds") return S_OK({"Successful": successful, "Failed": failed}) @@ -640,7 +624,7 @@ def replicateAndRegister(self, lfn, destSE, sourceSE="", destPath="", localCache if not res["OK"]: errStr = "Completely failed to replicate file." log.debug(errStr, res["Message"]) - return S_ERROR("{} {}".format(errStr, res["Message"])) + return S_ERROR(f"{errStr} {res['Message']}") if not res["Value"]: # The file was already present at the destination SE log.debug(f"{lfn} already present at {destSE}.") @@ -748,21 +732,21 @@ def __replicate(self, lfn, destSEName, sourceSEName="", destPath="", localCache= return S_ERROR(errStr) # Check that the destination storage element is sane and resolve its name - log.debug("Verifying destination StorageElement validity (%s)." % (destSEName)) + log.debug(f"Verifying destination StorageElement validity ({destSEName}).") destStorageElement = StorageElement(destSEName, vo=self.voName) res = destStorageElement.isValid() if not res["OK"]: errStr = "The storage element is not currently valid." - log.verbose(errStr, "{} {}".format(destSEName, res["Message"])) - return S_ERROR("{} {}".format(errStr, res["Message"])) + log.verbose(errStr, f"{destSEName} {res['Message']}") + return S_ERROR(f"{errStr} {res['Message']}") # Get the real name of the SE destSEName = destStorageElement.storageElementName() ########################################################### # Check whether the destination storage element is banned - log.verbose("Determining whether %s ( destination ) is Write-banned." % destSEName) + log.verbose(f"Determining whether {destSEName} ( destination ) is Write-banned.") if not destStorageElement.status()["Write"]: infoStr = "Supplied destination Storage Element is not currently allowed for Write." @@ -770,24 +754,24 @@ def __replicate(self, lfn, destSEName, sourceSEName="", destPath="", localCache= return S_ERROR(infoStr) # Get the LFN replicas from the file catalog - log.debug("Attempting to obtain replicas for %s." % (lfn)) + log.debug(f"Attempting to obtain replicas for {lfn}.") res = returnSingleResult(self.getReplicas(lfn, getUrl=False)) if not res["OK"]: errStr = "Failed to get replicas for LFN." - log.debug(errStr, "{} {}".format(lfn, res["Message"])) - return S_ERROR("{} {}".format(errStr, res["Message"])) + log.debug(errStr, f"{lfn} {res['Message']}") + return S_ERROR(f"{errStr} {res['Message']}") log.debug("Successfully obtained replicas for LFN.") lfnReplicas = res["Value"] ########################################################### # If the file catalog size is zero fail the transfer - log.debug("Attempting to obtain size for %s." % lfn) + log.debug(f"Attempting to obtain size for {lfn}.") res = returnSingleResult(self.fileCatalog.getFileSize(lfn)) if not res["OK"]: errStr = "Failed to get size for LFN." - log.debug(errStr, "{} {}".format(lfn, res["Message"])) - return S_ERROR("{} {}".format(errStr, res["Message"])) + log.debug(errStr, f"{lfn} {res['Message']}") + return S_ERROR(f"{errStr} {res['Message']}") catalogSize = res["Value"] @@ -796,12 +780,12 @@ def __replicate(self, lfn, destSEName, sourceSEName="", destPath="", localCache= log.debug(errStr, lfn) return S_ERROR(errStr) - log.debug("File size determined to be %s." % catalogSize) + log.debug(f"File size determined to be {catalogSize}.") ########################################################### # If the LFN already exists at the destination we have nothing to do if self.__isSEInList(destSEName, lfnReplicas): - log.debug("__replicate: LFN is already registered at %s." % destSEName) + log.debug(f"__replicate: LFN is already registered at {destSEName}.") return S_OK() ########################################################### @@ -839,24 +823,21 @@ def __replicate(self, lfn, destSEName, sourceSEName="", destPath="", localCache= destPath = lfn for candidateSEName in possibleSourceSEs: - - log.debug("Consider %s as a source" % candidateSEName) + log.debug(f"Consider {candidateSEName} as a source") # Check that the candidate is active if not self.__checkSEStatus(candidateSEName, status="Read"): - log.debug("%s is currently not allowed as a source." % candidateSEName) + log.debug(f"{candidateSEName} is currently not allowed as a source.") continue else: - log.debug("%s is available for use." % candidateSEName) + log.debug(f"{candidateSEName} is available for use.") candidateSE = StorageElement(candidateSEName, vo=self.voName) # Check that the SE is valid res = candidateSE.isValid() if not res["OK"]: - log.verbose( - "The storage element is not currently valid.", "{} {}".format(candidateSEName, res["Message"]) - ) + log.verbose("The storage element is not currently valid.", f"{candidateSEName} {res['Message']}") continue else: log.debug("The storage is currently valid", candidateSEName) @@ -864,7 +845,7 @@ def __replicate(self, lfn, destSEName, sourceSEName="", destPath="", localCache= # Check that the file size corresponds to the one in the FC res = returnSingleResult(candidateSE.getFileSize(lfn)) if not res["OK"]: - log.debug("could not get fileSize on %s" % candidateSEName, res["Message"]) + log.debug(f"could not get fileSize on {candidateSEName}", res["Message"]) continue seFileSize = res["Value"] @@ -874,6 +855,10 @@ def __replicate(self, lfn, destSEName, sourceSEName="", destPath="", localCache= else: log.debug("Catalog size and physical size match") + # The file at the SE seems healthy, so we could potentially use this SE + # for non TPC transfer in case everything else fails. + possibleIntermediateSEs.append(candidateSE) + res = destStorageElement.negociateProtocolWithOtherSE(candidateSE, protocols=self.thirdPartyProtocols) if not res["OK"]: @@ -883,7 +868,6 @@ def __replicate(self, lfn, destSEName, sourceSEName="", destPath="", localCache= replicationProtocols = res["Value"] if not replicationProtocols: - possibleIntermediateSEs.append(candidateSE) log.debug("No protocol suitable for replication found") continue @@ -899,7 +883,6 @@ def __replicate(self, lfn, destSEName, sourceSEName="", destPath="", localCache= # But that is the only way to make sure we are not replicating # over ourselves. for compatibleProtocol in replicationProtocols: - # Compare the urls to make sure we are not overwriting res = returnSingleResult(candidateSE.getURL(lfn, protocol=compatibleProtocol)) if not res["OK"]: @@ -911,7 +894,6 @@ def __replicate(self, lfn, destSEName, sourceSEName="", destPath="", localCache= destURL = "" res = returnSingleResult(destStorageElement.getURL(destPath, protocol=compatibleProtocol)) if not res["OK"]: - # for some protocols, in particular srm # you might get an error because the file does not exist # which is exactly what we want @@ -957,13 +939,12 @@ def __replicate(self, lfn, destSEName, sourceSEName="", destPath="", localCache= localDir = os.path.realpath(localCache if localCache else ".") localFile = os.path.join(localDir, os.path.basename(lfn)) - log.debug("Will try intermediate transfer from %s sources" % len(possibleIntermediateSEs)) + log.debug(f"Will try intermediate transfer from {len(possibleIntermediateSEs)} sources") for candidateSE in possibleIntermediateSEs: - res = returnSingleResult(candidateSE.getFile(lfn, localPath=localDir)) if not res["OK"]: - log.debug("Error getting the file from %s" % candidateSE.name, res["Message"]) + log.debug(f"Error getting the file from {candidateSE.name}", res["Message"]) continue res = returnSingleResult(destStorageElement.putFile({destPath: localFile})) @@ -975,7 +956,7 @@ def __replicate(self, lfn, destSEName, sourceSEName="", destPath="", localCache= log.error("Error removing local file", f"{localFile} {e}") if not res["OK"]: - log.debug("Error putting file coming from %s" % candidateSE.name, res["Message"]) + log.debug(f"Error putting file coming from {candidateSE.name}", res["Message"]) # if the put is the problem, it's maybe pointless to try the other # candidateSEs... continue @@ -1012,6 +993,8 @@ def registerFile(self, fileTuple, catalog=""): fileTuples = fileTuple elif isinstance(fileTuple, tuple): fileTuples = [fileTuple] + else: + return S_ERROR(f"fileTuple is none of list,set,tuple: {type(fileTuple)}") for fileTuple in fileTuples: if not isinstance(fileTuple, tuple): errStr = "Supplied file info must be tuple or list of tuples." @@ -1019,7 +1002,7 @@ def registerFile(self, fileTuple, catalog=""): return S_ERROR(errStr) if not fileTuples: return S_OK({"Successful": [], "Failed": {}}) - log.debug("Attempting to register %s files." % len(fileTuples)) + log.debug(f"Attempting to register {len(fileTuples)} files.") res = self.__registerFile(fileTuples, catalog) if not res["OK"]: errStr = "Completely failed to register files." @@ -1044,7 +1027,7 @@ def __registerFile(self, fileTuples, catalog): if catalog: fileCatalog = FileCatalog(catalog, vo=self.voName) if not fileCatalog.isOK(): - return S_ERROR("Can't get FileCatalog %s" % catalog) + return S_ERROR(f"Can't get FileCatalog {catalog}") else: fileCatalog = self.fileCatalog @@ -1065,6 +1048,8 @@ def registerReplica(self, replicaTuple, catalog=""): replicaTuples = replicaTuple elif isinstance(replicaTuple, tuple): replicaTuples = [replicaTuple] + else: + return S_ERROR(f"replicaTuple is not of type list,set or tuple: {type(replicaTuple)}") for replicaTuple in replicaTuples: if not isinstance(replicaTuple, tuple): errStr = "Supplied file info must be tuple or list of tuples." @@ -1072,7 +1057,7 @@ def registerReplica(self, replicaTuple, catalog=""): return S_ERROR(errStr) if not replicaTuples: return S_OK({"Successful": [], "Failed": {}}) - log.debug("Attempting to register %s replicas." % len(replicaTuples)) + log.debug(f"Attempting to register {len(replicaTuples)} replicas.") res = self.__registerReplica(replicaTuples, catalog) if not res["OK"]: errStr = "Completely failed to register replicas." @@ -1093,7 +1078,7 @@ def __registerReplica(self, replicaTuples, catalog): res = destStorageElement.isValid() if not res["OK"]: errStr = "The storage element is not currently valid." - log.verbose(errStr, "{} {}".format(storageElementName, res["Message"])) + log.verbose(errStr, f"{storageElementName} {res['Message']}") for lfn, url in replicaTuple: failed[lfn] = errStr else: @@ -1105,7 +1090,7 @@ def __registerReplica(self, replicaTuples, catalog): else: replicaTuple = (lfn, res["Value"], storageElementName, False) replicaTuples.append(replicaTuple) - log.debug("Successfully resolved %s replicas for registration." % len(replicaTuples)) + log.debug(f"Successfully resolved {len(replicaTuples)} replicas for registration.") # HACK! replicaDict = {} for lfn, url, se, _master in replicaTuples: @@ -1119,7 +1104,7 @@ def __registerReplica(self, replicaTuples, catalog): if not res["OK"]: errStr = "Completely failed to register replicas." log.debug(errStr, res["Message"]) - return S_ERROR("{} {}".format(errStr, res["Message"])) + return S_ERROR(f"{errStr} {res['Message']}") failed.update(res["Value"]["Failed"]) successful = res["Value"]["Successful"] resDict = {"Successful": successful, "Failed": failed} @@ -1172,13 +1157,13 @@ def removeFile(self, lfn, force=None): return res if res["Value"]["Failed"]: errStr = "Write access not permitted for this credential." - log.debug(errStr, "for %d files" % len(res["Value"]["Failed"])) + log.debug(errStr, f"for {len(res['Value']['Failed'])} files") failed.update(dict.fromkeys(res["Value"]["Failed"], errStr)) lfns = res["Value"]["Successful"] if lfns: - log.debug("Attempting to remove %d files from Storage and Catalogue. Get replicas first" % len(lfns)) + log.debug(f"Attempting to remove {len(lfns)} files from Storage and Catalogue. Get replicas first") res = self.fileCatalog.getReplicas(lfns, allStatus=True) if not res["OK"]: errStr = "DataManager.removeFile: Completely failed to get replicas for lfns." @@ -1220,17 +1205,17 @@ def __removeFile(self, lfnDict): if not res["OK"]: errStr = res["Message"] for lfn in lfns: - failed[lfn] = failed.setdefault(lfn, "") + " %s" % errStr + failed[lfn] = failed.setdefault(lfn, "") + f" {errStr}" else: for lfn, errStr in res["Value"]["Failed"].items(): # can be an iterator - failed[lfn] = failed.setdefault(lfn, "") + " %s" % errStr + failed[lfn] = failed.setdefault(lfn, "") + f" {errStr}" completelyRemovedFiles = set(lfnDict) - set(failed) if completelyRemovedFiles: res = self.fileCatalog.removeFile(list(completelyRemovedFiles)) if not res["OK"]: failed.update( - dict.fromkeys(completelyRemovedFiles, "Failed to remove file from the catalog: %s" % res["Message"]) + dict.fromkeys(completelyRemovedFiles, f"Failed to remove file from the catalog: {res['Message']}") ) else: failed.update(res["Value"]["Failed"]) @@ -1266,7 +1251,7 @@ def removeReplica(self, storageElementName, lfn): return res if res["Value"]["Failed"]: errStr = "Write access not permitted for this credential." - log.debug(errStr, "for %d files" % len(res["Value"]["Failed"])) + log.debug(errStr, f"for {len(res['Value']['Failed'])} files") failed.update(dict.fromkeys(res["Value"]["Failed"], errStr)) lfns -= set(res["Value"]["Failed"]) @@ -1432,7 +1417,7 @@ def __removeCatalogReplica(self, replicaTuples): errStr = "Completely failed to remove replica: " log.debug(errStr, res["Message"]) - return S_ERROR("{} {}".format(errStr, res["Message"])) + return S_ERROR(f"{errStr} {res['Message']}") success = res["Value"]["Successful"] failed = res["Value"]["Failed"] @@ -1450,7 +1435,7 @@ def __removeCatalogReplica(self, replicaTuples): # Only for logging information if success: - log.debug("Removed %d replicas" % len(success)) + log.debug(f"Removed {len(success)} replicas") for lfn in success: log.debug("Successfully removed replica.", lfn) @@ -1472,8 +1457,8 @@ def __removePhysicalReplica(self, storageElementName, lfnsToRemove, replicaDict= res = storageElement.isValid() if not res["OK"]: errStr = "The storage element is not currently valid." - log.verbose(errStr, "{} {}".format(storageElementName, res["Message"])) - return S_ERROR("{} {}".format(errStr, res["Message"])) + log.verbose(errStr, f"{storageElementName} {res['Message']}") + return S_ERROR(f"{errStr} {res['Message']}") startTime = datetime.utcnow() transferStartTime = time.time() @@ -1547,8 +1532,8 @@ def put(self, lfn, fileName, diracSE, path=None): res = storageElement.isValid() if not res["OK"]: errStr = "The storage element is not currently valid." - log.verbose(errStr, "{} {}".format(diracSE, res["Message"])) - return S_ERROR("{} {}".format(errStr, res["Message"])) + log.verbose(errStr, f"{diracSE} {res['Message']}") + return S_ERROR(f"{errStr} {res['Message']}") fileDict = {lfn: fileName} successful = {} @@ -1561,9 +1546,9 @@ def put(self, lfn, fileName, diracSE, path=None): if not res["OK"]: errStr = "Failed to put file to Storage Element." failed[lfn] = res["Message"] - log.debug(errStr, "{}: {}".format(fileName, res["Message"])) + log.debug(errStr, f"{fileName}: {res['Message']}") else: - log.debug("Put file to storage in %s seconds." % putTime) + log.debug(f"Put file to storage in {putTime} seconds.") successful[lfn] = res["Value"] resDict = {"Successful": successful, "Failed": failed} return S_OK(resDict) @@ -1573,10 +1558,16 @@ def put(self, lfn, fileName, diracSE, path=None): # File catalog methods # - def getActiveReplicas(self, lfns, getUrl=True, diskOnly=False, preferDisk=False): + def getActiveReplicas(self, lfns, getUrl=True, diskOnly=False, preferDisk=False, protocol=None): """Get all the replicas for the SEs which are in Active status for reading.""" return self.getReplicas( - lfns, allStatus=False, getUrl=getUrl, diskOnly=diskOnly, preferDisk=preferDisk, active=True + lfns, + allStatus=False, + getUrl=getUrl, + diskOnly=diskOnly, + preferDisk=preferDisk, + active=True, + protocol=protocol, ) def __filterTapeReplicas(self, replicaDict, diskOnly=False): @@ -1646,13 +1637,13 @@ def checkActiveReplicas(self, replicaDict): Check a replica dictionary for active replicas, and verify input structure first """ if not isinstance(replicaDict, dict): - return S_ERROR("Wrong argument type %s, expected a dictionary" % type(replicaDict)) + return S_ERROR(f"Wrong argument type {type(replicaDict)}, expected a dictionary") for key in ["Successful", "Failed"]: if key not in replicaDict: - return S_ERROR('Missing key "%s" in replica dictionary' % key) + return S_ERROR(f'Missing key "{key}" in replica dictionary') if not isinstance(replicaDict[key], dict): - return S_ERROR("Wrong argument type %s, expected a dictionary" % type(replicaDict[key])) + return S_ERROR(f"Wrong argument type {type(replicaDict[key])}, expected a dictionary") activeDict = {"Successful": {}, "Failed": replicaDict["Failed"].copy()} for lfn, replicas in replicaDict["Successful"].items(): # can be an iterator @@ -1681,12 +1672,17 @@ def __checkSEStatus(self, se, status="Read"): """returns the value of a certain SE status flag (access or other)""" return StorageElement(se, vo=self.voName).status().get(status, False) - def getReplicas(self, lfns, allStatus=True, getUrl=True, diskOnly=False, preferDisk=False, active=False): + def getReplicas( + self, lfns, allStatus=True, getUrl=True, diskOnly=False, preferDisk=False, active=False, protocol=None + ): """get replicas from catalogue and filter if requested Warning: all filters are independent, hence active and preferDisk should be set if using forJobs """ catalogReplicas = {} failed = {} + if not protocol: + protocol = self.registrationProtocol + for lfnChunk in breakListIntoChunks(lfns, 1000): res = self.fileCatalog.getReplicas(lfnChunk, allStatus=allStatus) if res["OK"]: @@ -1707,9 +1703,7 @@ def getReplicas(self, lfns, allStatus=True, getUrl=True, diskOnly=False, preferD for se in se_lfn: seObj = StorageElement(se, vo=self.voName) - succPfn = ( - seObj.getURL(se_lfn[se], protocol=self.registrationProtocol).get("Value", {}).get("Successful", {}) - ) + succPfn = seObj.getURL(se_lfn[se], protocol=protocol).get("Value", {}).get("Successful", {}) for lfn in succPfn: catalogReplicas[lfn][se] = succPfn[lfn] @@ -1720,10 +1714,10 @@ def getReplicas(self, lfns, allStatus=True, getUrl=True, diskOnly=False, preferD self.__filterTapeReplicas(result, diskOnly=diskOnly) return S_OK(result) - def getReplicasForJobs(self, lfns, allStatus=False, getUrl=True, diskOnly=False): + def getReplicasForJobs(self, lfns, allStatus=False, getUrl=True, diskOnly=False, protocol=None): """get replicas useful for jobs""" # Call getReplicas with no filter and enforce filters in this method - result = self.getReplicas(lfns, allStatus=allStatus, getUrl=getUrl) + result = self.getReplicas(lfns, allStatus=allStatus, getUrl=getUrl, protocol=protocol) if not result["OK"]: return result replicaDict = result["Value"] @@ -1781,7 +1775,7 @@ def __executeIfReplicaExists(self, storageElementName, lfn, method, **kwargs): res = fcn(lfnList, **kwargs) # # check result if not res["OK"]: - errStr = "Failed to execute %s StorageElement method." % method + errStr = f"Failed to execute {method} StorageElement method." log.error(errStr, res["Message"]) return res diff --git a/src/DIRAC/DataManagementSystem/Client/DirectoryListing.py b/src/DIRAC/DataManagementSystem/Client/DirectoryListing.py index 495410fb3f4..b61dc802f68 100644 --- a/src/DIRAC/DataManagementSystem/Client/DirectoryListing.py +++ b/src/DIRAC/DataManagementSystem/Client/DirectoryListing.py @@ -12,7 +12,6 @@ class DirectoryListing: def __init__(self): - self.entries = [] def addFile(self, name, fileDict, repDict, numericid): @@ -61,6 +60,7 @@ def addDirectory(self, name, dirDict, numericid): date = dirDict["ModificationDate"] nlinks = 0 size = 0 + gname = None if "Owner" in dirDict: uname = dirDict["Owner"] elif "OwnerDN" in dirDict: @@ -173,7 +173,7 @@ def humanReadableSize(self, num, suffix="B"): if abs(num) < 1024.0: return f"{num:3.1f}{unit}{suffix}" num /= 1024.0 - return "{:.1f}{}{}".format(num, "Yi", suffix) + return f"{num:.1f}Yi{suffix}" def printListing(self, reverse, timeorder, sizeorder, humanread): """ """ diff --git a/src/DIRAC/DataManagementSystem/Client/FTS3Client.py b/src/DIRAC/DataManagementSystem/Client/FTS3Client.py index 5be554595b2..4322cd7e0c5 100644 --- a/src/DIRAC/DataManagementSystem/Client/FTS3Client.py +++ b/src/DIRAC/DataManagementSystem/Client/FTS3Client.py @@ -43,7 +43,7 @@ def getOperation(self, operationID, **kwargs): opObj, _size = decode(opJSON) return S_OK(opObj) except Exception as e: - return S_ERROR("Exception when decoding the FTS3Operation object %s" % e) + return S_ERROR(f"Exception when decoding the FTS3Operation object {e}") def getActiveJobs(self, limit=20, lastMonitor=None, jobAssignmentTag="Assigned", **kwargs): """Get all the FTSJobs that are not in a final state @@ -61,7 +61,7 @@ def getActiveJobs(self, limit=20, lastMonitor=None, jobAssignmentTag="Assigned", activeJobs, _size = decode(activeJobsJSON) return S_OK(activeJobs) except Exception as e: - return S_ERROR("Exception when decoding the active jobs json %s" % e) + return S_ERROR(f"Exception when decoding the active jobs json {e}") def getNonFinishedOperations(self, limit=20, operationAssignmentTag="Assigned", **kwargs): """Get all the FTS3Operations that have files in New or Failed state @@ -82,7 +82,7 @@ def getNonFinishedOperations(self, limit=20, operationAssignmentTag="Assigned", operations, _size = decode(operationsJSON) return S_OK(operations) except Exception as e: - return S_ERROR(0, "Exception when decoding the non finished operations json %s" % e) + return S_ERROR(0, f"Exception when decoding the non finished operations json {e}") def getOperationsFromRMSOpID(self, rmsOpID, **kwargs): """Get the FTS3Operations matching a given RMS Operation @@ -99,4 +99,4 @@ def getOperationsFromRMSOpID(self, rmsOpID, **kwargs): operations, _size = decode(operationsJSON) return S_OK(operations) except Exception as e: - return S_ERROR(0, "Exception when decoding the operations json %s" % e) + return S_ERROR(0, f"Exception when decoding the operations json {e}") diff --git a/src/DIRAC/DataManagementSystem/Client/FTS3File.py b/src/DIRAC/DataManagementSystem/Client/FTS3File.py index 79a2e304758..bcd4e34a55f 100644 --- a/src/DIRAC/DataManagementSystem/Client/FTS3File.py +++ b/src/DIRAC/DataManagementSystem/Client/FTS3File.py @@ -21,6 +21,7 @@ class FTS3File(JSerializable): "Started", # From FTS: File transfer has started "Not_used", # From FTS: Transfer not being considered yet, waiting for another one (multihop) "Archiving", # From FTS: file not yet migrated to tape + "Token_prep", # From FTS: When using token, used before Submitted until FTS fetched a refresh token ] # These are the states that we consider final. @@ -59,7 +60,6 @@ class FTS3File(JSerializable): ] def __init__(self): - self.status = FTS3File.INIT_STATE self.attempt = 0 diff --git a/src/DIRAC/DataManagementSystem/Client/FTS3Job.py b/src/DIRAC/DataManagementSystem/Client/FTS3Job.py index 69b8886b2b5..4d17ca1b430 100644 --- a/src/DIRAC/DataManagementSystem/Client/FTS3Job.py +++ b/src/DIRAC/DataManagementSystem/Client/FTS3Job.py @@ -1,11 +1,23 @@ """ FTS3Job module containing only the FTS3Job class """ + import datetime import errno +from packaging.version import Version + # Requires at least version 3.3.3 +from fts3 import __version__ as fts3_version import fts3.rest.client.easy as fts3 from fts3.rest.client.exceptions import FTS3ClientException, NotFound +# There is a breaking change in the API in 3.13 +# https://gitlab.cern.ch/fts/fts-rest-flask/-/commit/5faa283e0cd4b80a0139a547c4a6356522c8449d +FTS3_SPACETOKEN_API_CHANGE = Version("3.13") +if Version(fts3_version) >= FTS3_SPACETOKEN_API_CHANGE: + DESTINATION_SPACETOKEN_ATTR = "destination_spacetoken" +else: + DESTINATION_SPACETOKEN_ATTR = "spacetoken" + # We specifically use Request in the FTS client because of a leak in the # default pycurl. See https://its.cern.ch/jira/browse/FTS-261 from fts3.rest.client.request import Request as ftsSSLRequest @@ -13,6 +25,7 @@ from DIRAC.Resources.Storage.StorageElement import StorageElement from DIRAC.FrameworkSystem.Client.Logger import gLogger +from DIRAC.FrameworkSystem.Client.TokenManagerClient import gTokenManager from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR from DIRAC.Core.Utilities.DErrno import cmpError @@ -45,6 +58,9 @@ class FTS3Job(JSerializable): ] FINAL_STATES = ["Canceled", "Failed", "Finished", "Finisheddirty"] + + # This field is only used for optimizing sql queries (`in`` instead of `not in`) + NON_FINAL_STATES = list(set(ALL_STATES) - set(FINAL_STATES)) INIT_STATE = "Submitted" # END states @@ -63,7 +79,6 @@ class FTS3Job(JSerializable): ] def __init__(self): - self.submitTime = None self.lastUpdate = None self.lastMonitor = None @@ -143,7 +158,7 @@ def monitor(self, context=None, ftsServer=None, ucert=None): self.status = "Failed" return S_ERROR(errno.ESRCH, f"FTSGUID {self.ftsGUID} not found on {self.ftsServer}") except FTS3ClientException as e: - return S_ERROR("Error getting the job status %s" % e) + return S_ERROR(f"Error getting the job status {e}") now = datetime.datetime.utcnow().replace(microsecond=0) self.lastMonitor = now @@ -191,6 +206,12 @@ def monitor(self, context=None, ftsServer=None, ucert=None): # monitoring calls if file_state in FTS3File.FTS_FINAL_STATES: filesStatus[file_id]["ftsGUID"] = None + # TODO: update status to defunct if not recoverable here ? + + # If the file is failed, check if it is recoverable + if file_state in FTS3File.FTS_FAILED_STATES: + if not fileDict.get("Recoverable", True): + filesStatus[file_id]["status"] = "Defunct" # If the file is not in a final state, but the job is, we return an error # FTS can have inconsistencies where the FTS Job is in a final state @@ -218,6 +239,28 @@ def monitor(self, context=None, ftsServer=None, ucert=None): return S_OK(filesStatus) + def cancel(self, context): + """Cancel the job on the FTS server. Note that it will cancel all the files. + See https://fts3-docs.web.cern.ch/fts3-docs/fts-rest/docs/api.html#delete-jobsjobidlist + for behavior details + """ + + try: + cancelDict = fts3.cancel(context, self.ftsGUID) + newStatus = cancelDict["job_state"].capitalize() + # If the status is already canceled + # (for a reason or another, don't change the error message) + # If the new status is Canceled, set it, and update the reason + if newStatus == "Canceled" and self.status != "Canceled": + self.status = "Canceled" + self.error = "Matching RMS Request was canceled" + return S_OK() + # The job is not found + except NotFound: + return S_ERROR(errno.ESRCH, f"FTSGUID {self.ftsGUID} not found on {self.ftsServer}") + except FTS3ClientException as e: + return S_ERROR(f"Error canceling the job {e}") + @staticmethod def __fetchSpaceToken(seName, vo): """Fetch the space token of storage element @@ -259,7 +302,18 @@ def __isTapeSE(seName, vo): return isTape - def _constructTransferJob(self, pinTime, allLFNs, target_spacetoken, protocols=None): + @staticmethod + def __seTokenSupport(seObj): + """Check whether a given SE supports token + + :param seObj: StorageElement object + + :returns: True/False + In case of error, returns False + """ + return seObj.options.get("TokenSupport", "").lower() in ("true", "yes") + + def _constructTransferJob(self, pinTime, allLFNs, target_spacetoken, protocols=None, tokensEnabled=False): """Build a job for transfer Some attributes of the job are expected to be set @@ -287,11 +341,12 @@ def _constructTransferJob(self, pinTime, allLFNs, target_spacetoken, protocols=N log = gLogger.getSubLogger(f"constructTransferJob/{self.operationID}/{self.sourceSE}_{self.targetSE}") isMultiHop = False + useTokens = False # Check if it is a multiHop transfer if self.multiHopSE: if len(allLFNs) != 1: - log.debug("Multihop job has %s files while only 1 allowed" % len(allLFNs)) + log.debug(f"Multihop job has {len(allLFNs)} files while only 1 allowed") return S_ERROR(errno.E2BIG, "Trying multihop job with more than one file !") allHops = [(self.sourceSE, self.multiHopSE), (self.multiHopSE, self.targetSE)] isMultiHop = True @@ -316,7 +371,6 @@ def _constructTransferJob(self, pinTime, allLFNs, target_spacetoken, protocols=N fileIDsInTheJob = set() for hopId, (hopSrcSEName, hopDstSEName) in enumerate(allHops, start=1): - # Again, this is relevant only for the very initial source # but code factorization is more important hopSrcIsTape = self.__isTapeSE(hopSrcSEName, self.vo) @@ -368,7 +422,7 @@ def _constructTransferJob(self, pinTime, allLFNs, target_spacetoken, protocols=N # This test is important, because multiple files would result in the source # being deleted ! if len(allLFNs) != 1: - log.debug("Multihop job has %s files while only 1 allowed" % len(allLFNs)) + log.debug(f"Multihop job has {len(allLFNs)} files while only 1 allowed") return S_ERROR(errno.E2BIG, "Trying multihop job with more than one file !") res = srcSE.getURL(allSrcDstSURLs, protocol=srcSE.localStageProtocolList) @@ -384,16 +438,18 @@ def _constructTransferJob(self, pinTime, allLFNs, target_spacetoken, protocols=N allStageURLs = res["Value"]["Successful"] for ftsFile in self.filesToSubmit: - if ftsFile.lfn in failedLFNs: - log.debug("Not preparing transfer for file %s" % ftsFile.lfn) + log.debug(f"Not preparing transfer for file {ftsFile.lfn}") continue + srcToken = None + dstToken = None + sourceSURL, targetSURL = allSrcDstSURLs[ftsFile.lfn] stageURL = allStageURLs.get(ftsFile.lfn) if sourceSURL == targetSURL: - log.error("sourceSURL equals to targetSURL", "%s" % ftsFile.lfn) + log.error("sourceSURL equals to targetSURL", f"{ftsFile.lfn}") ftsFile.error = "sourceSURL equals to targetSURL" ftsFile.status = "Defunct" continue @@ -412,10 +468,9 @@ def _constructTransferJob(self, pinTime, allLFNs, target_spacetoken, protocols=N # * srcProto://myFile -> destProto://myFile if stageURL: - # We do not set a fileID in the metadata # such that we do not update the DB when monitoring - stageTrans_metadata = {"desc": "PreStage %s" % ftsFileID} + stageTrans_metadata = {"desc": f"PreStage {ftsFileID}"} # If we use an activity, also set it as file metadata # for WLCG monitoring purposes @@ -426,7 +481,7 @@ def _constructTransferJob(self, pinTime, allLFNs, target_spacetoken, protocols=N stageTrans = fts3.new_transfer( stageURL, stageURL, - checksum="ADLER32:%s" % ftsFile.checksum, + checksum=f"ADLER32:{ftsFile.checksum}", filesize=ftsFile.size, metadata=stageTrans_metadata, activity=self.activity, @@ -436,9 +491,9 @@ def _constructTransferJob(self, pinTime, allLFNs, target_spacetoken, protocols=N # If it is the last hop only, we set the fileID metadata # for monitoring if hopId == nbOfHops: - trans_metadata = {"desc": "Transfer %s" % ftsFileID, "fileID": ftsFileID} + trans_metadata = {"desc": f"Transfer {ftsFileID}", "fileID": ftsFileID} else: - trans_metadata = {"desc": "MultiHop %s" % ftsFileID} + trans_metadata = {"desc": f"MultiHop {ftsFileID}"} # If we use an activity, also set it as file metadata # for WLCG monitoring purposes @@ -446,6 +501,44 @@ def _constructTransferJob(self, pinTime, allLFNs, target_spacetoken, protocols=N if self.activity: trans_metadata["activity"] = self.activity + # Add tokens if both storages support it and if the requested + if tokensEnabled and self.__seTokenSupport(srcSE) and self.__seTokenSupport(dstSE): + # We get a read token for the source + # offline_access is to allow FTS to refresh it + res = srcSE.getWLCGTokenPath(ftsFile.lfn) + if not res["OK"]: + return res + srcTokenPath = res["Value"] + res = gTokenManager.getToken( + userGroup=self.userGroup, + requiredTimeLeft=3600, + scope=[f"storage.read:/{srcTokenPath}", "offline_access"], + useCache=False, + ) + if not res["OK"]: + return res + srcToken = res["Value"]["access_token"] + + # We get a token with modify and read for the destination + # We need the read to be able to stat + # CAUTION: only works with dcache for now, other storages + # interpret permissions differently + # offline_access is to allow FTS to refresh it + res = dstSE.getWLCGTokenPath(ftsFile.lfn) + if not res["OK"]: + return res + dstTokenPath = res["Value"] + res = gTokenManager.getToken( + userGroup=self.userGroup, + requiredTimeLeft=3600, + scope=[f"storage.modify:/{dstTokenPath}", f"storage.read:/{dstTokenPath}", "offline_access"], + useCache=False, + ) + if not res["OK"]: + return res + dstToken = res["Value"]["access_token"] + useTokens = True + # because of an xroot bug (https://github.com/xrootd/xrootd/issues/1433) # the checksum needs to be lowercase. It does not impact the other # protocol, so it's fine to put it here. @@ -454,10 +547,12 @@ def _constructTransferJob(self, pinTime, allLFNs, target_spacetoken, protocols=N trans = fts3.new_transfer( sourceSURL, targetSURL, - checksum="ADLER32:%s" % ftsFile.checksum.lower(), + checksum=f"ADLER32:{ftsFile.checksum.lower()}", filesize=ftsFile.size, metadata=trans_metadata, activity=self.activity, + source_token=srcToken, + destination_token=dstToken, ) transfers.append(trans) @@ -475,16 +570,17 @@ def _constructTransferJob(self, pinTime, allLFNs, target_spacetoken, protocols=N "rmsReqID": self.rmsReqID, "sourceSE": self.sourceSE, "targetSE": self.targetSE, + "useTokens": useTokens, # Store the information here to propagate it to submission } if self.activity: job_metadata["activity"] = self.activity + dest_spacetoken = {DESTINATION_SPACETOKEN_ATTR: target_spacetoken} job = fts3.new_job( transfers=transfers, overwrite=True, source_spacetoken=source_spacetoken, - spacetoken=target_spacetoken, bring_online=bring_online, copy_pin_lifetime=copy_pin_lifetime, retry=3, @@ -493,6 +589,7 @@ def _constructTransferJob(self, pinTime, allLFNs, target_spacetoken, protocols=N metadata=job_metadata, priority=self.priority, archive_timeout=archive_timeout, + **dest_spacetoken, ) return S_OK((job, fileIDsInTheJob)) @@ -586,18 +683,17 @@ def _constructStagingJob(self, pinTime, allLFNs, target_spacetoken): allTargetSURLs = res["Value"]["Successful"] for ftsFile in self.filesToSubmit: - if ftsFile.lfn in failedLFNs: - log.debug("Not preparing transfer for file %s" % ftsFile.lfn) + log.debug(f"Not preparing transfer for file {ftsFile.lfn}") continue sourceSURL = targetSURL = allTargetSURLs[ftsFile.lfn] ftsFileID = getattr(ftsFile, "fileID") - trans_metadata = {"desc": "Stage %s" % ftsFileID, "fileID": ftsFileID} + trans_metadata = {"desc": f"Stage {ftsFileID}", "fileID": ftsFileID} trans = fts3.new_transfer( sourceSURL, targetSURL, - checksum="ADLER32:%s" % ftsFile.checksum, + checksum=f"ADLER32:{ftsFile.checksum}", filesize=ftsFile.size, metadata=trans_metadata, activity=self.activity, @@ -621,21 +717,23 @@ def _constructStagingJob(self, pinTime, allLFNs, target_spacetoken): if self.activity: job_metadata["activity"] = self.activity + dest_spacetoken = {DESTINATION_SPACETOKEN_ATTR: target_spacetoken} + job = fts3.new_job( transfers=transfers, overwrite=True, source_spacetoken=target_spacetoken, - spacetoken=target_spacetoken, bring_online=bring_online, copy_pin_lifetime=copy_pin_lifetime, retry=3, metadata=job_metadata, priority=self.priority, + **dest_spacetoken, ) return S_OK((job, fileIDsInTheJob)) - def submit(self, context=None, ftsServer=None, ucert=None, pinTime=36000, protocols=None): + def submit(self, context=None, ftsServer=None, ucert=None, pinTime=36000, protocols=None, fts_access_token=None): """submit the job to the FTS server Some attributes are expected to be defined for the submission to work: @@ -659,17 +757,13 @@ def submit(self, context=None, ftsServer=None, ucert=None, pinTime=36000, protoc :param ucert: path to the user certificate/proxy. Might be inferred by the fts cli (see its doc) :param protocols: list of protocols from which we should choose the protocol to use + :param fts_access_token: token to be used to talk to FTS and to be passed when creating a context :returns: S_OK([FTSFiles ids of files submitted]) """ log = gLogger.getLocalSubLogger(f"submit/{self.operationID}/{self.sourceSE}_{self.targetSE}") - if not context: - if not ftsServer: - ftsServer = self.ftsServer - context = fts3.Context(endpoint=ftsServer, ucert=ucert, request_class=ftsSSLRequest, verify=False) - # Construct the target SURL res = self.__fetchSpaceToken(self.targetSE, self.vo) if not res["OK"]: @@ -679,7 +773,10 @@ def submit(self, context=None, ftsServer=None, ucert=None, pinTime=36000, protoc allLFNs = [ftsFile.lfn for ftsFile in self.filesToSubmit] if self.type == "Transfer": - res = self._constructTransferJob(pinTime, allLFNs, target_spacetoken, protocols=protocols) + res = self._constructTransferJob( + pinTime, allLFNs, target_spacetoken, protocols=protocols, tokensEnabled=bool(fts_access_token) + ) + elif self.type == "Staging": res = self._constructStagingJob(pinTime, allLFNs, target_spacetoken) # elif self.type == 'Removal': @@ -690,9 +787,24 @@ def submit(self, context=None, ftsServer=None, ucert=None, pinTime=36000, protoc job, fileIDsInTheJob = res["Value"] + # If we need a token, don't use the context given in parameter + # because the one given in parameter is only with X509 creds + if job["params"].get("job_metadata", {}).get("useTokens"): + if not fts_access_token: + return S_ERROR("Job needs token support but no FTS token was supplied") + context = None + + if not context: + if not ftsServer: + ftsServer = self.ftsServer + res = self.generateContext(ftsServer, ucert, fts_access_token) + if not res["OK"]: + return res + context = res["Value"] + try: self.ftsGUID = fts3.submit(context, job) - log.info("Got GUID %s" % self.ftsGUID) + log.info(f"Got GUID {self.ftsGUID}") # Only increase the amount of attempt # if we succeeded in submitting -> no ! Why did I do that ?? @@ -720,38 +832,50 @@ def submit(self, context=None, ftsServer=None, ucert=None, pinTime=36000, protoc except FTS3ClientException as e: log.exception("Error at submission", repr(e)) - return S_ERROR("Error at submission: %s" % e) + return S_ERROR(f"Error at submission: {e}") return S_OK(fileIDsInTheJob) @staticmethod - def generateContext(ftsServer, ucert, lifetime=25200): + def generateContext(ftsServer, ucert, fts_access_token=None, lifetime=25200): """This method generates an fts3 context + Only a certificate or an fts token can be given + :param ftsServer: address of the fts3 server :param ucert: the path to the certificate to be used + :param fts_access_token: token to access FTS :param lifetime: duration (in sec) of the delegation to the FTS3 server (default is 7h, like FTS3 default) :returns: an fts3 context """ + if fts_access_token and ucert: + return S_ERROR("fts_access_token and ucert cannot be both set") + try: - context = fts3.Context(endpoint=ftsServer, ucert=ucert, request_class=ftsSSLRequest, verify=False) + context = fts3.Context( + endpoint=ftsServer, + ucert=ucert, + request_class=ftsSSLRequest, + verify=False, + fts_access_token=fts_access_token, + ) - # Explicitely delegate to be sure we have the lifetime we want - # Note: the delegation will re-happen only when the FTS server - # decides that there is not enough timeleft. - # At the moment, this is 1 hour, which effectively means that if you do - # not submit a job for more than 1h, you have no valid proxy in FTS servers - # anymore. In future release of FTS3, the delegation will be triggered when - # one third of the lifetime will be left. - # Also, the proxy given as parameter might have less than "lifetime" left - # since it is cached, but it does not matter, because in the FTS3Agent - # we make sure that we renew it often enough - # Finally, FTS3 has an issue with handling the lifetime of the proxy, - # because it does not check all the chain. This is under discussion - # https://its.cern.ch/jira/browse/FTS-1575 - fts3.delegate(context, lifetime=datetime.timedelta(seconds=lifetime)) + # The delegation only makes sense for X509 auth + if ucert: + # Explicitely delegate to be sure we have the lifetime we want + # Note: the delegation will re-happen only when the FTS server + # decides that there is not enough timeleft. + # At the moment, this is 1 hour, which effectively means that if you do + # not submit a job for more than 1h, you have no valid proxy in FTS servers + # anymore, and all the jobs failed. So we force it when + # one third of the lifetime will be left. + # Also, the proxy given as parameter might have less than "lifetime" left + # since it is cached, but it does not matter, because in the FTS3Agent + # we make sure that we renew it often enough + td_lifetime = datetime.timedelta(seconds=lifetime) + fts3.delegate(context, lifetime=td_lifetime, delegate_when_lifetime_lt=td_lifetime // 3) return S_OK(context) except FTS3ClientException as e: diff --git a/src/DIRAC/DataManagementSystem/Client/FTS3Operation.py b/src/DIRAC/DataManagementSystem/Client/FTS3Operation.py index ac7c592e642..67b7042bd09 100644 --- a/src/DIRAC/DataManagementSystem/Client/FTS3Operation.py +++ b/src/DIRAC/DataManagementSystem/Client/FTS3Operation.py @@ -150,7 +150,7 @@ def init_on_load(self): self.fts3Plugin = FTS3Utilities.getFTS3Plugin(vo=self.vo) opID = getattr(self, "operationID", None) - loggerName = "%s/" % opID if opID else "" + loggerName = f"{opID}/" if opID else "" loggerName += f"req_{self.rmsReqID}/op_{self.rmsOpID}" self._log = gLogger.getSubLogger(loggerName) @@ -298,9 +298,7 @@ def _updateRmsOperationStatus(self): * ftsFilesByTarget: dict {SE: [ftsFiles that were successful]} """ - log = self._log.getLocalSubLogger( - "_updateRmsOperationStatus/{}/{}".format(getattr(self, "operationID"), self.rmsReqID) - ) + log = self._log.getLocalSubLogger(f"_updateRmsOperationStatus/{getattr(self, 'operationID')}/{self.rmsReqID}") res = self.reqClient.getRequest(self.rmsReqID) if not res["OK"]: @@ -331,7 +329,6 @@ def _updateRmsOperationStatus(self): # { SE : [FTS3Files] } ftsFilesByTarget = {} for ftsFile in self.ftsFiles: - if ftsFile.status == "Defunct": log.info("File failed to transfer, setting it to failed in RMS", f"{ftsFile.lfn} {ftsFile.targetSE}") defunctRmsFileIDs.add(ftsFile.rmsFileID) @@ -365,20 +362,19 @@ def _updateRmsOperationStatus(self): return S_OK({"request": request, "operation": operation, "ftsFilesByTarget": ftsFilesByTarget}) @classmethod - def fromRMSObjects(cls, rmsReq, rmsOp, username): + def fromRMSObjects(cls, rmsReq, rmsOp): """Construct an FTS3Operation object from the RMS Request and Operation corresponding. The attributes taken are the OwnerGroup, Request and Operation IDS, sourceSE, and activity and priority if they are defined in the Argument field of the operation :param rmsReq: RMS Request object :param rmsOp: RMS Operation object - :param username: username to which associate the FTS3Operation (normally comes from the Req OwnerDN) :returns: FTS3Operation object """ ftsOp = cls() - ftsOp.username = username + ftsOp.username = rmsReq.Owner ftsOp.userGroup = rmsReq.OwnerGroup ftsOp.rmsReqID = rmsReq.RequestID @@ -401,11 +397,10 @@ class FTS3TransferOperation(FTS3Operation): """Class to be used for a Replication operation""" def prepareNewJobs(self, maxFilesPerJob=100, maxAttemptsPerFile=10): - log = self._log.getSubLogger("_prepareNewJobs") filesToSubmit = self._getFilesToSubmit(maxAttemptsPerFile=maxAttemptsPerFile) - log.debug("%s ftsFiles to submit" % len(filesToSubmit)) + log.debug(f"{len(filesToSubmit)} ftsFiles to submit") newJobs = [] @@ -416,13 +411,12 @@ def prepareNewJobs(self, maxFilesPerJob=100, maxAttemptsPerFile=10): filesGroupedByTarget = res["Value"] for targetSE, ftsFiles in filesGroupedByTarget.items(): - res = self._checkSEAccess(targetSE, "WriteAccess", vo=self.vo) if not res["OK"]: # If the SE is currently banned, we just skip it if cmpError(res, errno.EACCES): - log.info("Write access currently not permitted to %s, skipping." % targetSE) + log.info(f"Write access currently not permitted to {targetSE}, skipping.") else: log.error(res) for ftsFile in ftsFiles: @@ -444,22 +438,20 @@ def prepareNewJobs(self, maxFilesPerJob=100, maxAttemptsPerFile=10): # If the error is that the file does not exist in the catalog # fail it ! if cmpError(errMsg, errno.ENOENT): - log.error("The file does not exist, setting it Defunct", "%s" % ftsFile.lfn) + log.error("The file does not exist, setting it Defunct", f"{ftsFile.lfn}") ftsFile.status = "Defunct" # We don't need to check the source, since it is already filtered by the DataManager for sourceSE, ftsFiles in uniqueTransfersBySource.items(): - # Checking whether we will need multiHop transfer multiHopSE = self.fts3Plugin.findMultiHopSEToCoverUpForWLCGFailure(sourceSE, targetSE) if multiHopSE: - - log.verbose("WLCG failure manifestation, use %s for multihop, max files per job is 1" % multiHopSE) + log.verbose(f"WLCG failure manifestation, use {multiHopSE} for multihop, max files per job is 1") # Check that we can write and read from it try: for accessType in ("Read", "Write"): - res = self._checkSEAccess(multiHopSE, "%sAccess" % accessType, vo=self.vo) + res = self._checkSEAccess(multiHopSE, f"{accessType}Access", vo=self.vo) if not res["OK"]: # If the SE is currently banned, we just skip it @@ -485,7 +477,6 @@ def prepareNewJobs(self, maxFilesPerJob=100, maxAttemptsPerFile=10): maxFilesPerJob = 1 for ftsFilesChunk in breakListIntoChunks(ftsFiles, maxFilesPerJob): - newJob = self._createNewJob( "Transfer", ftsFilesChunk, targetSE, sourceSE=sourceSE, multiHopSE=multiHopSE ) @@ -517,7 +508,7 @@ def __needsMultiHopStaging(self, sourceSEName, destSEName): # is compatible with staging tpcProtocols = self.fts3Plugin.selectTPCProtocols(sourceSEName=sourceSEName, destSEName=destSEName) - res = dstSE.generateTransferURLsBetweenSEs("/%s/fakeLFN" % self.vo, srcSE, protocols=tpcProtocols) + res = dstSE.generateTransferURLsBetweenSEs(f"/{self.vo}/fakeLFN", srcSE, protocols=tpcProtocols) # There is an error, but let's ignore it, # it will be dealt with in the FTS3Job logic @@ -581,7 +572,7 @@ def _callback(self): registrationProtocols = DMSHelpers(vo=self.vo).getRegistrationProtocols() - log.info("will create %s 'RegisterReplica' operations" % len(ftsFilesByTarget)) + log.info(f"will create {len(ftsFilesByTarget)} 'RegisterReplica' operations") for target, ftsFileList in ftsFilesByTarget.items(): log.info(f"creating 'RegisterReplica' operation for targetSE {target} with {len(ftsFileList)} files...") @@ -619,11 +610,10 @@ class FTS3StagingOperation(FTS3Operation): """Class to be used for a Staging operation""" def prepareNewJobs(self, maxFilesPerJob=100, maxAttemptsPerFile=10): - log = gLogger.getSubLogger("_prepareNewJobs") filesToSubmit = self._getFilesToSubmit(maxAttemptsPerFile=maxAttemptsPerFile) - log.debug("%s ftsFiles to submit" % len(filesToSubmit)) + log.debug(f"{len(filesToSubmit)} ftsFiles to submit") newJobs = [] @@ -631,14 +621,12 @@ def prepareNewJobs(self, maxFilesPerJob=100, maxAttemptsPerFile=10): filesGroupedByTarget = FTS3Utilities.groupFilesByTarget(filesToSubmit) for targetSE, ftsFiles in filesGroupedByTarget.items(): - res = self._checkSEAccess(targetSE, "ReadAccess", vo=self.vo) if not res["OK"]: log.error(res) continue for ftsFilesChunk in breakListIntoChunks(ftsFiles, maxFilesPerJob): - newJob = self._createNewJob("Staging", ftsFilesChunk, targetSE, sourceSE=targetSE) newJobs.append(newJob) diff --git a/src/DIRAC/DataManagementSystem/Client/FailoverTransfer.py b/src/DIRAC/DataManagementSystem/Client/FailoverTransfer.py index 39f542f0c9e..7f65c1c433d 100644 --- a/src/DIRAC/DataManagementSystem/Client/FailoverTransfer.py +++ b/src/DIRAC/DataManagementSystem/Client/FailoverTransfer.py @@ -16,6 +16,8 @@ temporary replica. """ + +import errno import time from DIRAC import S_OK, S_ERROR, gLogger @@ -84,7 +86,6 @@ def transferAndRegisterFile( fileChecksum = fileMetaDict.get("Checksum", None) for se in destinationSEList: - # We put here some retry in case the problem comes from the FileCatalog # being unavailable. If it is, then the `hasAccess` call would fail, # and we would not make any failover request. So the only way is to wait a bit @@ -108,6 +109,9 @@ def transferAndRegisterFile( break elif cmpError(result, EFCERR): self.log.debug("transferAndRegisterFile: FC unavailable, retry") + elif cmpError(result, errno.ENAMETOOLONG): + self.log.debug(f"transferAndRegisterFile: this file won't be uploaded: {result}") + return result elif retryUpload and len(destinationSEList) == 1: self.log.debug("transferAndRegisterFile: Failed uploading to the only SE, retry") else: @@ -130,7 +134,7 @@ def transferAndRegisterFile( errorDict = result["Value"]["Failed"][lfn] if "register" not in errorDict: self.log.error("dm.putAndRegister failed with unknown error", str(errorDict)) - errorList.append("Unknown error while attempting upload to %s" % se) + errorList.append(f"Unknown error while attempting upload to {se}") continue # fileDict = errorDict['register'] @@ -155,7 +159,7 @@ def transferAndRegisterFile( metadata["registration"] = "request" return S_OK(metadata) - self.log.error("Failed to upload output data file", "Encountered %s errors" % len(errorList)) + self.log.error("Failed to upload output data file", f"Encountered {len(errorList)} errors") return S_ERROR("Failed to upload output data file") ############################################################################# @@ -207,7 +211,7 @@ def commitRequest(self): isValid = RequestValidator().validate(self.request) if not isValid["OK"]: - return S_ERROR("Failover request is not valid: %s" % isValid["Message"]) + return S_ERROR(f"Failover request is not valid: {isValid['Message']}") else: requestClient = ReqClient() result = requestClient.putRequest(self.request) @@ -260,7 +264,6 @@ def _setRegistrationRequest(self, lfn, targetSE, fileDict, catalog): catalog = [catalog] for cat in catalog: - register = Operation() register.Type = "RegisterFile" register.Catalog = cat diff --git a/src/DIRAC/DataManagementSystem/Client/FileCatalogClientCLI.py b/src/DIRAC/DataManagementSystem/Client/FileCatalogClientCLI.py index 629ec79ad37..a8619e16c18 100644 --- a/src/DIRAC/DataManagementSystem/Client/FileCatalogClientCLI.py +++ b/src/DIRAC/DataManagementSystem/Client/FileCatalogClientCLI.py @@ -1,17 +1,12 @@ #!/usr/bin/env python """ File Catalog Client Command Line Interface. """ -# TODO: This should be modernised to use subprocess(32) -try: - import commands -except ImportError: - # Python 3's subprocess module contains a compatibility layer - import subprocess as commands import os.path import time import sys import getopt +import uuid -from DIRAC import S_ERROR, S_OK +from DIRAC import S_ERROR from DIRAC.Core.Utilities.ReturnValues import returnSingleResult from DIRAC.Core.Base.CLI import CLI from DIRAC.Core.Security.ProxyInfo import getProxyInfo @@ -58,7 +53,6 @@ def __init__(self, client): self.ul_dc = DirectoryCompletion(self.ul_fs) def getPath(self, apath): - if apath.find("/") == 0: path = apath else: @@ -141,7 +135,7 @@ def do_add(self, args): dirac = Dirac() result = dirac.addFile(lfn, pfn, se, guid, printOutput=False) if not result["OK"]: - print("Error: %s" % (result["Message"])) + print(f"Error: {result['Message']}") else: print(f"File {lfn} successfully uploaded to the {se} SE") @@ -189,9 +183,9 @@ def do_get(self, args): os.chdir(localCWD) if not result["OK"]: - print("Error: %s" % (result["Message"])) + print(f"Error: {result['Message']}") else: - print("File %s successfully downloaded" % lfn) + print(f"File {lfn} successfully downloaded") def complete_get(self, text, line, begidx, endidx): result = [] @@ -239,7 +233,7 @@ def do_unregister(self, args): return return self.removeDirectory(argss) else: - print("Error: illegal option %s" % option) + print(f"Error: illegal option {option}") # An Auto Completion For ``register`` _available_unregister_cmd = ["replica", "file", "dir", "directory"] @@ -388,11 +382,11 @@ def removeReplica(self, args): if result["OK"]: if "Failed" in result["Value"]: if lfn in result["Value"]["Failed"]: - print("ERROR: %s" % (result["Value"]["Failed"][lfn])) + print(f"ERROR: {result['Value']['Failed'][lfn]}") elif lfn in result["Value"]["Successful"]: print(f"File {lfn} at {rmse} removed from the catalog") else: - print("ERROR: Unexpected returned value %s" % result["Value"]) + print(f"ERROR: Unexpected returned value {result['Value']}") else: print(f"File {lfn} at {rmse} removed from the catalog") else: @@ -412,11 +406,11 @@ def removeFile(self, args): if result["OK"]: if "Failed" in result["Value"]: if lfn in result["Value"]["Failed"]: - print("ERROR: %s" % (result["Value"]["Failed"][lfn])) + print(f"ERROR: {result['Value']['Failed'][lfn]}") elif lfn in result["Value"]["Successful"]: print("File", lfn, "removed from the catalog") else: - print("ERROR: Unexpected result %s" % result["Value"]) + print(f"ERROR: Unexpected result {result['Value']}") else: print("File", lfn, "removed from the catalog") else: @@ -461,7 +455,7 @@ def removeDirectory(self, lfn, recursive=False, forceNonEmpty=False): print("Error:", result["Message"]) return result if result["Value"]["Failed"]: - print("Error: failed to remove %d files" % len(result["Value"]["Failed"])) + print(f"Error: failed to remove {len(result['Value']['Failed'])} files") return S_ERROR("Failed to remove files") else: print("Error: failed to remove non empty directory") @@ -477,7 +471,7 @@ def removeDirectory(self, lfn, recursive=False, forceNonEmpty=False): return result except Exception as x: print("Error: rmdir failed with exception: ", x) - return S_ERROR("Exception: %s" % str(x)) + return S_ERROR(f"Exception: {str(x)}") def do_replicate(self, args): """Replicate a given file to a given SE @@ -499,7 +493,7 @@ def do_replicate(self, args): dirac = Dirac() result = dirac.replicateFile(lfn, se, sourceSE, printOutput=True) if not result["OK"]: - print("Error: %s" % (result["Message"])) + print(f"Error: {result['Message']}") elif not result["Value"]: print("Replica is already present at the target SE") else: @@ -573,7 +567,7 @@ def registerFile(self, args): """ if len(args) != 6: - print("Command takes 6 arguments, %d given" % len(args)) + print(f"Command takes 6 arguments, {len(args)} given") print(self.do_register.__doc__) return @@ -584,7 +578,7 @@ def registerFile(self, args): infoDict["Size"] = int(args[2]) infoDict["SE"] = args[3] if args[4] == "None": - _status, guid = commands.getstatusoutput("uuidgen") + guid = str(uuid.uuid4()) else: guid = args[4] infoDict["GUID"] = guid @@ -665,17 +659,15 @@ def do_ancestorset(self, args): print("Failed to add file ancestors to the catalog: ", end=" ") print(result["Value"]["Failed"][lfn]) else: - print("Added %d ancestors to file %s" % (len(ancestors), lfn)) + print(f"Added {len(ancestors)} ancestors to file {lfn}") except Exception as x: print("Exception while adding ancestors: ", str(x)) def complete_ancestorset(self, text, line, begidx, endidx): - args = line.split() - if len(args) == 1: - cur_path = "" - elif len(args) > 1: + cur_path = "" + if len(args) > 1: # If the line ends with ' ' # this means a new parameter begin. if line.endswith(" "): @@ -833,7 +825,7 @@ def do_user(self, args): elif option == "show": result = self.fc.getUsers() if not result["OK"]: - print("Error: %s" % result["Message"]) + print(f"Error: {result['Message']}") else: if not result["Value"]: print("No entries found") @@ -884,7 +876,7 @@ def do_group(self, args): elif option == "show": result = self.fc.getGroups() if not result["OK"]: - print("Error: %s" % result["Message"]) + print(f"Error: {result['Message']}") else: if not result["Value"]: print("No entries found") @@ -918,7 +910,7 @@ def registerUser(self, argss): result = self.fc.addUser(username) if not result["OK"]: - print("Error: %s" % result["Message"]) + print(f"Error: {result['Message']}") else: print("User ID:", result["Value"]) @@ -932,7 +924,7 @@ def deleteUser(self, args): result = self.fc.deleteUser(username) if not result["OK"]: - print("Error: %s" % result["Message"]) + print(f"Error: {result['Message']}") def registerGroup(self, argss): """Add new group to the File Catalog @@ -944,7 +936,7 @@ def registerGroup(self, argss): result = self.fc.addGroup(gname) if not result["OK"]: - print("Error: %s" % result["Message"]) + print(f"Error: {result['Message']}") else: print("Group ID:", result["Value"]) @@ -958,7 +950,7 @@ def deleteGroup(self, args): result = self.fc.deleteGroup(gname) if not result["OK"]: - print("Error: %s" % result["Message"]) + print(f"Error: {result['Message']}") def do_mkdir(self, args): """Make directory @@ -1060,18 +1052,18 @@ def do_id(self, args): """Get user identity""" result = getProxyInfo() if not result["OK"]: - print("Error: %s" % result["Message"]) + print(f"Error: {result['Message']}") return user = result["Value"]["username"] group = result["Value"]["group"] result = self.fc.getUsers() if not result["OK"]: - print("Error: %s" % result["Message"]) + print(f"Error: {result['Message']}") return userDict = result["Value"] result = self.fc.getGroups() if not result["OK"]: - print("Error: %s" % result["Message"]) + print(f"Error: {result['Message']}") return groupDict = result["Value"] idUser = userDict.get(user, 0) @@ -1092,9 +1084,9 @@ def do_lcd(self, args): try: os.chdir(localDir) newDir = os.getcwd() - print("Local directory: %s" % newDir) + print(f"Local directory: {newDir}") except Exception: - print("%s seems not a directory" % localDir) + print(f"{localDir} seems not a directory") def complete_lcd(self, text, line, begidx, endidx): # TODO @@ -1515,7 +1507,7 @@ def do_size(self, args): values.append(("Total", int_with_commas(totalSize), str(totalFiles))) printTable(fields, values) if "QueryTime" in result["Value"]: - print("Query time %.2f sec" % result["Value"]["QueryTime"]) + print(f"Query time {result['Value']['QueryTime']:.2f} sec") else: print("Directory size failed:", result["Value"]["Failed"][path]) else: @@ -1696,9 +1688,7 @@ def removeMeta(self, argss): def setMeta(self, argss): """Set metadata value for a directory""" if len(argss) < 3 or len(argss) % 2 != 1: - print( - "Error: command requires at least 3 arguments (or odd number of arguments > 3), %d given" % len(argss) - ) + print(f"Error: command requires at least 3 arguments (or odd number of arguments > 3), {len(argss)} given") return path = argss[0] if path == ".": @@ -1713,7 +1703,7 @@ def setMeta(self, argss): print(path, metadict) result = self.fc.setMetadata(path, metadict) if not result["OK"]: - print("Error: %s" % result["Message"]) + print(f"Error: {result['Message']}") def getMeta(self, argss): """Get metadata for the given directory""" @@ -1749,7 +1739,7 @@ def getMeta(self, argss): if dirFlag: result = self.fc.getDirectoryUserMetadata(path) if not result["OK"]: - print("Error: %s" % result["Message"]) + print(f"Error: {result['Message']}") return if result["Value"]: metaDict = result["MetadataOwner"] @@ -1770,7 +1760,7 @@ def getMeta(self, argss): if setFlag and expandFlag: result = self.fc.getMetadataSet(value, expandFlag) if not result["OK"]: - print("Error: %s" % result["Message"]) + print(f"Error: {result['Message']}") return for m, v in result["Value"].items(): print(" " * 10, m.rjust(20), ":", v) @@ -1779,7 +1769,7 @@ def getMeta(self, argss): else: result = self.fc.getFileUserMetadata(path) if not result["OK"]: - print("Error: %s" % result["Message"]) + print(f"Error: {result['Message']}") return if result["Value"]: for meta, value in result["Value"].items(): @@ -1801,7 +1791,7 @@ def metaTag(self, argss): if argss[0].lower() == "where": result = self.fc.getMetadataFields() if not result["OK"]: - print("Error: %s" % result["Message"]) + print(f"Error: {result['Message']}") return if not result["Value"]: print("Error: no metadata fields defined") @@ -1815,9 +1805,9 @@ def metaTag(self, argss): name, value = arg.split("=") if name not in typeDict: if name not in typeDictfm: - print("Error: metadata field %s not defined" % name) + print(f"Error: metadata field {name} not defined") else: - print("No support for meta data at File level yet: %s" % name) + print(f"No support for meta data at File level yet: {name}") return mtype = typeDict[name] mvalue = value @@ -1835,22 +1825,22 @@ def metaTag(self, argss): result = self.fc.getCompatibleMetadata(metaDict, path) if not result["OK"]: - print("Error: %s" % result["Message"]) + print(f"Error: {result['Message']}") return tagDict = result["Value"] if tag in tagDict: if tagDict[tag]: - print("Possible values for %s:" % tag) + print(f"Possible values for {tag}:") for v in tagDict[tag]: print(v) else: - print("No compatible values found for %s" % tag) + print(f"No compatible values found for {tag}") def showMeta(self): """Show defined metadata indices""" result = self.fc.getMetadataFields() if not result["OK"]: - print("Error: %s" % result["Message"]) + print(f"Error: {result['Message']}") else: if not result["Value"]: print("No entries found") @@ -1900,12 +1890,12 @@ def registerMeta(self, argss): elif mtype.lower() == "metaset": rtype = "MetaSet" else: - print("Error: illegal metadata type %s" % mtype) + print(f"Error: illegal metadata type {mtype}") return result = self.fc.addMetadataField(mname, rtype, fdType) if not result["OK"]: - print("Error: %s" % result["Message"]) + print(f"Error: {result['Message']}") else: print(f"Added metadata field {mname} of type {mtype}") @@ -1921,9 +1911,9 @@ def registerMetaset(self, argss): result = self.fc.addMetadataSet(setName, setDict) if not result["OK"]: - print("Error: %s" % result["Message"]) + print(f"Error: {result['Message']}") else: - print("Added metadata set %s" % setName) + print(f"Added metadata set {setName}") def do_find(self, args): """Find all files satisfying the given metadata information @@ -1966,11 +1956,10 @@ def do_find(self, args): result = self.fc.findFilesByMetadata(metaDict, path) if not result["OK"]: - print("Error: %s" % result["Message"]) + print(f"Error: {result['Message']}") return if result["Value"]: - lfnList = result["Value"] if dirsOnly: listToPrint = {os.path.dirname(fullpath) for fullpath in lfnList} @@ -1985,7 +1974,7 @@ def do_find(self, args): print("No matching data found") if verbose and "QueryTime" in result: - print("QueryTime %.2f sec" % result["QueryTime"]) + print(f"QueryTime {result['QueryTime']:.2f} sec") def complete_find(self, text, line, begidx, endidx): result = [] @@ -2014,7 +2003,7 @@ def __createQuery(self, args): result = self.fc.getMetadataFields() if not result["OK"]: - print("Error: %s" % result["Message"]) + print(f"Error: {result['Message']}") return None if not result["Value"]: print("Error: no metadata fields defined") @@ -2223,7 +2212,7 @@ def do_stats(self, args): return if not result["OK"]: - print("Error: %s" % result["Message"]) + print(f"Error: {result['Message']}") return fields = ["Counter", "Number"] records = [] @@ -2264,11 +2253,10 @@ def do_repair(self, args): print(f"{repType} repair: {repResult}") total = time.time() - start - print("Catalog repair operation done in %.2f sec" % total) + print(f"Catalog repair operation done in {total:.2f} sec") if __name__ == "__main__": - if len(sys.argv) > 2: print(FileCatalogClientCLI.__doc__) sys.exit(2) diff --git a/src/DIRAC/DataManagementSystem/Client/MetaQuery.py b/src/DIRAC/DataManagementSystem/Client/MetaQuery.py index 2302dab096e..0cafe0ee3b5 100644 --- a/src/DIRAC/DataManagementSystem/Client/MetaQuery.py +++ b/src/DIRAC/DataManagementSystem/Client/MetaQuery.py @@ -52,7 +52,6 @@ class MetaQuery: def __init__(self, queryDict=None, typeDict=None): - self.__metaQueryDict = {} if queryDict is not None: self.__metaQueryDict = queryDict @@ -75,11 +74,11 @@ def setMetaQuery(self, queryList, metaTypeDict=None): operation = op break if not operation: - return S_ERROR("Illegal query element %s" % arg) + return S_ERROR(f"Illegal query element {arg}") name, value = arg.split(operation) if name not in self.__metaTypeDict: - return S_ERROR("Metadata field %s not defined" % name) + return S_ERROR(f"Metadata field {name} not defined") mtype = self.__metaTypeDict[name] else: @@ -160,11 +159,9 @@ def setMetaQuery(self, queryList, metaTypeDict=None): return S_OK(metaDict) def getMetaQuery(self): - return self.__metaQueryDict def getMetaQueryAsJson(self): - return json.dumps(self.__metaQueryDict) def applyQuery(self, userMetaDict): @@ -192,7 +189,6 @@ def getTypedValue(value, mtype): return value for meta, value in self.__metaQueryDict.items(): - # Check if user dict contains all the requested meta data userValue = userMetaDict.get(meta, None) if userValue is None: diff --git a/src/DIRAC/DataManagementSystem/Client/__init__.py b/src/DIRAC/DataManagementSystem/Client/__init__.py index 6f5707ba432..f06ddfbf90b 100755 --- a/src/DIRAC/DataManagementSystem/Client/__init__.py +++ b/src/DIRAC/DataManagementSystem/Client/__init__.py @@ -1,3 +1,6 @@ """ DIRAC.DataManagementSystem.Client package """ + +#: Maximum number of characters for a filename, this should be the same as the FileName column of the DFC +MAX_FILENAME_LENGTH = 128 diff --git a/src/DIRAC/DataManagementSystem/Client/test/Test_Client_DataManagementSystem.py b/src/DIRAC/DataManagementSystem/Client/test/Test_Client_DataManagementSystem.py index 16964b90983..b1c3872e173 100644 --- a/src/DIRAC/DataManagementSystem/Client/test/Test_Client_DataManagementSystem.py +++ b/src/DIRAC/DataManagementSystem/Client/test/Test_Client_DataManagementSystem.py @@ -13,7 +13,6 @@ class UtilitiesTestCase(unittest.TestCase): def setUp(self): - gLogger.setLevel("DEBUG") self.lfnDict = { @@ -100,83 +99,6 @@ def test__getFileTypesCount(self): # res = self.ci.catalogDirectoryToSE(lfnDir) # self.assertTrue(res['OK']) - def test__getCatalogDirectoryContents(self): - lfnDirs = ["/this/is/dir1/", "/this/is/dir2/"] - - res = self.ci._getCatalogDirectoryContents(lfnDirs) - self.assertTrue(res["OK"]) - - resExpected = { - "Metadata": { - "/this/is/dir1/file1.txt": { - "MetaData": { - "Checksum": "7149ed85", - "ChecksumType": "Adler32", - "CreationDate": datetime.datetime(2014, 12, 4, 12, 16, 56), - "FileID": 156301805, - "GID": 2695, - "GUID": "6A5C6C86-AD7B-E411-9EDB", - "Mode": 436, - "ModificationDate": datetime.datetime(2014, 12, 4, 12, 16, 56), - "Owner": "phicharp", - "OwnerGroup": "lhcb_prod", - "Size": 206380531, - "Status": "AprioriGood", - "Type": "File", - "UID": 19503, - } - }, - "/this/is/dir1/file2.foo.bar": { - "MetaData": { - "Checksum": "7149ed86", - "ChecksumType": "Adler32", - "CreationDate": datetime.datetime(2014, 12, 4, 12, 16, 56), - "FileID": 156301805, - "GID": 2695, - "GUID": "6A5C6C86-AD7B-E411-9EDB", - "Mode": 436, - "ModificationDate": datetime.datetime(2014, 12, 4, 12, 16, 56), - "Owner": "phicharp", - "OwnerGroup": "lhcb_prod", - "Size": 206380532, - "Status": "AprioriGood", - "Type": "File", - "UID": 19503, - } - }, - "/this/is/dir2/subdir1/file3.pippo": { - "MetaData": { - "Checksum": "7149ed86", - "ChecksumType": "Adler32", - "CreationDate": datetime.datetime(2014, 12, 4, 12, 16, 56), - "FileID": 156301805, - "GID": 2695, - "GUID": "6A5C6C86-AD7B-E411-9EDB", - "Mode": 436, - "ModificationDate": datetime.datetime(2014, 12, 4, 12, 16, 56), - "Owner": "phicharp", - "OwnerGroup": "lhcb_prod", - "Size": 206380532, - "Status": "AprioriGood", - "Type": "File", - "UID": 19503, - } - }, - }, - "Replicas": { - "/this/is/dir1/file1.txt": { - "SE1": "smr://srm.SE1.ch:8443/srm/v2/server?SFN=/this/is/dir1/file1.txt", - "SE2": "smr://srm.SE2.fr:8443/srm/v2/server?SFN=/this/is/dir1/file1.txt", - }, - "/this/is/dir1/file2.foo.bar": { - "SE1": "smr://srm.SE1.ch:8443/srm/v2/server?SFN=/this/is/dir1/file2.foo.bar", - "SE3": "smr://srm.SE3.es:8443/srm/v2/server?SFN=/this/is/dir1/file2.foo.bar", - }, - }, - } - - self.assertEqual(res["Value"], resExpected) - if __name__ == "__main__": suite = unittest.defaultTestLoader.loadTestsFromTestCase(UtilitiesTestCase) diff --git a/src/DIRAC/DataManagementSystem/Client/test/Test_FTS3Objects.py b/src/DIRAC/DataManagementSystem/Client/test/Test_FTS3Objects.py index e28ee1a6959..6471ec65d0b 100644 --- a/src/DIRAC/DataManagementSystem/Client/test/Test_FTS3Objects.py +++ b/src/DIRAC/DataManagementSystem/Client/test/Test_FTS3Objects.py @@ -1,22 +1,15 @@ -import os -import pytest -import tempfile import errno -import DIRAC +from unittest import mock -from DIRAC.ConfigurationSystem.private.ConfigurationClient import ConfigurationClient -from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData -from diraccfg import CFG -from DIRAC.DataManagementSystem.private.FTS3Plugins.DefaultFTS3Plugin import DefaultFTS3Plugin +import pytest + +import DIRAC from DIRAC import S_OK from DIRAC.Core.Utilities.DErrno import cmpError - -from DIRAC.Resources.Storage.StorageBase import StorageBase - -from DIRAC.DataManagementSystem.Client.FTS3Job import FTS3Job from DIRAC.DataManagementSystem.Client.FTS3File import FTS3File -from DIRAC.DataManagementSystem.Client.FTS3Operation import FTS3Operation - +from DIRAC.DataManagementSystem.Client.FTS3Job import FTS3Job +from DIRAC.Resources.Storage.StorageBase import StorageBase +from DIRAC.tests.Utilities.utils import generateDIRACConfig DIRAC.gLogger.setLevel("DEBUG") # pylint: disable=redefined-outer-name @@ -38,14 +31,7 @@ def mock_StorageFactory_generateStorageObject(self, storageName, pluginName, par return S_OK(storageObj) -@pytest.fixture(scope="module", autouse=True) -def generateConfig(): - """ - Generate the configuration that will be used for all the test - """ - - testCfgFileName = os.path.join(tempfile.gettempdir(), "test_FTS3Plugin.cfg") - cfgContent = """ +CFG_CONTENT = """ DIRAC { VirtualOrganization = lhcb @@ -160,23 +146,14 @@ def generateConfig(): } } } - """ - with open(testCfgFileName, "w") as f: - f.write(cfgContent) - # Load the configuration - ConfigurationClient(fileToLoadList=[testCfgFileName]) # we replace the configuration by our own one. - yield - - try: - os.remove(testCfgFileName) - except OSError: - pass - # SUPER UGLY: one must recreate the CFG objects of gConfigurationData - # not to conflict with other tests that might be using a local dirac.cfg - gConfigurationData.localCFG = CFG() - gConfigurationData.remoteCFG = CFG() - gConfigurationData.mergedCFG = CFG() - gConfigurationData.generateNewVersion() +""" + + +@pytest.fixture(scope="module", autouse=True) +def loadCS(): + """Load the CFG_CONTENT as a DIRAC Configuration for this module""" + with generateDIRACConfig(CFG_CONTENT, "test_FTS3Objects.cfg"): + yield @pytest.fixture(scope="function", autouse=True) @@ -195,6 +172,20 @@ def monkeypatchForAllTest(monkeypatch): lambda _self, _seName, _vo: S_OK(), ) + def mock_init(self, useProxy=False, vo=None): + self.proxy = False + self.proxy = useProxy + self.resourceStatus = mock.MagicMock() + self.vo = vo + self.remoteProtocolSections = [] + self.localProtocolSections = [] + self.name = "" + self.options = {} + self.protocols = {} + self.storages = {} + + monkeypatch.setattr(DIRAC.Resources.Storage.StorageFactory.StorageFactory, "__init__", mock_init) + def generateFTS3Job(sourceSE, targetSE, lfns, multiHopSE=None): """Utility to create a new FTS3Job object with some FTS3Files diff --git a/src/DIRAC/DataManagementSystem/Client/test/new_dir_completion.py b/src/DIRAC/DataManagementSystem/Client/test/new_dir_completion.py index 44eedb91b98..7cf3ee9d741 100644 --- a/src/DIRAC/DataManagementSystem/Client/test/new_dir_completion.py +++ b/src/DIRAC/DataManagementSystem/Client/test/new_dir_completion.py @@ -9,7 +9,6 @@ class DirCompletion(cmd.Cmd): - ulfs = UnixLikeFileSystem() dc = DirectoryCompletion(ulfs) @@ -48,6 +47,5 @@ def complete_ls(self, text, line, begidx, endidx): if __name__ == "__main__": - cli = DirCompletion() cli.cmdloop() diff --git a/src/DIRAC/DataManagementSystem/ConfigTemplate.cfg b/src/DIRAC/DataManagementSystem/ConfigTemplate.cfg index 24c177d253d..16c103c67cd 100644 --- a/src/DIRAC/DataManagementSystem/ConfigTemplate.cfg +++ b/src/DIRAC/DataManagementSystem/ConfigTemplate.cfg @@ -38,14 +38,6 @@ Services } } ##END - FileCatalogProxy - { - Port = 9138 - Authorization - { - Default = authenticated - } - } FileCatalog { Port = 9197 @@ -108,23 +100,20 @@ Services } ##END - StorageElementProxy + ##BEGIN S3Gateway + S3Gateway { - BasePath = storageElement - Port = 9139 + Port = 9169 Authorization { Default = authenticated - FileTransfer - { - Default = authenticated - } } } - ##BEGIN S3Gateway - S3Gateway + ##END + ##BEGIN TornadoS3Gateway + TornadoS3Gateway { - Port = 9169 + Protocol = https Authorization { Default = authenticated @@ -143,6 +132,12 @@ Agents OperationBulkSize = 20 # How many Job we will monitor in one loop JobBulkSize = 20 + # split jobBulkSize in several chunks + # Bigger numbers (like 100) are efficient when there's a single agent + # When there are multiple agents, it may slow down the overall because + # of lock and race conditions + # (This number should of course be smaller or equal than JobBulkSize) + JobMonitoringBatchSize = 20 # Max number of files to go in a single job MaxFilesPerJob = 100 # Max number of attempt per file @@ -155,8 +150,11 @@ Agents KickAssignedHours = 1 # Max number of kicks per cycle KickLimitPerCycle = 100 - # Lifetime in sec of the Proxy we download to delegate to FTS3 (default 12h) - ProxyLifetime = 43200 + # Lifetime in sec of the Proxy we download to delegate to FTS3 (default 36h) + ProxyLifetime = 129600 + # Whether we use tokens to submit jobs to FTS3 + # VERY EXPERIMENTAL + UseTokens = False } ##END FTS3Agent } diff --git a/src/DIRAC/DataManagementSystem/DB/DataIntegrityDB.py b/src/DIRAC/DataManagementSystem/DB/DataIntegrityDB.py index a1bc368f3a4..402876dd657 100644 --- a/src/DIRAC/DataManagementSystem/DB/DataIntegrityDB.py +++ b/src/DIRAC/DataManagementSystem/DB/DataIntegrityDB.py @@ -39,7 +39,7 @@ def __init__(self, parentLogger=None): retVal = self.__initializeDB() if not retVal["OK"]: - raise Exception("Can't create tables: %s" % retVal["Message"]) + raise Exception(f"Can't create tables: {retVal['Message']}") def __initializeDB(self): """Make sure the table is created""" @@ -158,7 +158,7 @@ def getTransformationProblematics(self, transID): def incrementProblematicRetry(self, fileID): """Increment retry count""" - req = "UPDATE Problematics SET Retries=Retries+1, LastUpdate=UTC_TIMESTAMP() WHERE FileID = %s;" % (fileID) + req = f"UPDATE Problematics SET Retries=Retries+1, LastUpdate=UTC_TIMESTAMP() WHERE FileID = {fileID};" res = self._update(req) return res diff --git a/src/DIRAC/DataManagementSystem/DB/FTS3DB.py b/src/DIRAC/DataManagementSystem/DB/FTS3DB.py index 29c5dffe66c..ed45cf2097a 100644 --- a/src/DIRAC/DataManagementSystem/DB/FTS3DB.py +++ b/src/DIRAC/DataManagementSystem/DB/FTS3DB.py @@ -1,5 +1,5 @@ -""" Frontend to FTS3 MySQL DB. Written using sqlalchemy -""" +"""Frontend to FTS3 MySQL DB. Written using sqlalchemy""" + # We disable the no-member error because # they are constructed by SQLAlchemy for all # the objects mapped to a table. @@ -7,37 +7,39 @@ import datetime import errno +from urllib.parse import quote_plus -from sqlalchemy.orm.exc import NoResultFound -from sqlalchemy.exc import SQLAlchemyError -from sqlalchemy.sql.expression import and_ -from sqlalchemy.orm import relationship, sessionmaker, mapper -from sqlalchemy.sql import update, delete, select from sqlalchemy import ( - create_engine, - Table, + BigInteger, Column, - MetaData, - ForeignKey, - Integer, - String, DateTime, Enum, - BigInteger, - SmallInteger, Float, + ForeignKey, + Integer, + MetaData, + SmallInteger, + String, + Table, + create_engine, func, text, ) +from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.orm import registry, relationship, sessionmaker +from sqlalchemy.orm.exc import NoResultFound +from sqlalchemy.sql import delete, select, update +from sqlalchemy.sql.expression import and_ # # from DIRAC -from DIRAC import S_OK, S_ERROR, gLogger -from DIRAC.DataManagementSystem.Client.FTS3Operation import FTS3Operation, FTS3TransferOperation, FTS3StagingOperation +from DIRAC import S_ERROR, S_OK, gLogger +from DIRAC.ConfigurationSystem.Client.Utilities import getDBParameters from DIRAC.DataManagementSystem.Client.FTS3File import FTS3File from DIRAC.DataManagementSystem.Client.FTS3Job import FTS3Job -from DIRAC.ConfigurationSystem.Client.Utilities import getDBParameters +from DIRAC.DataManagementSystem.Client.FTS3Operation import FTS3Operation, FTS3StagingOperation, FTS3TransferOperation metadata = MetaData() +mapper_registry = registry() # Define the default utc_timestampfunction. # We overwrite it in the case of sqlite in the tests @@ -62,7 +64,7 @@ mysql_engine="InnoDB", ) -mapper(FTS3File, fts3FileTable) +mapper_registry.map_imperatively(FTS3File, fts3FileTable) fts3JobTable = Table( @@ -86,7 +88,7 @@ mysql_engine="InnoDB", ) -mapper(FTS3Job, fts3JobTable) +mapper_registry.map_imperatively(FTS3Job, fts3JobTable) fts3OperationTable = Table( @@ -112,7 +114,7 @@ ) -fts3Operation_mapper = mapper( +fts3Operation_mapper = mapper_registry.map_imperatively( FTS3Operation, fts3OperationTable, properties={ @@ -126,10 +128,11 @@ ), "ftsJobs": relationship( FTS3Job, - lazy="subquery", # Immediately load the entirety of the object, + lazy="selectin", # Immediately load the entirety of the object, # but use a subquery to do it # This is to avoid the cartesian product between the three tables. - # https://docs.sqlalchemy.org/en/latest/orm/loading_relationships.html#subquery-eager-loading + # https://docs.sqlalchemy.org/en/20/orm/queryguide/relationships.html#selectin-eager-loading + # We use selectin and not subquery because of https://github.com/DIRACGrid/DIRAC/issues/6814 cascade="all, delete-orphan", # if a File is removed from the list, # remove it from the DB passive_deletes=True, # used together with cascade='all, delete-orphan' @@ -139,9 +142,13 @@ polymorphic_identity="Abs", ) -mapper(FTS3TransferOperation, fts3OperationTable, inherits=fts3Operation_mapper, polymorphic_identity="Transfer") +mapper_registry.map_imperatively( + FTS3TransferOperation, fts3OperationTable, inherits=fts3Operation_mapper, polymorphic_identity="Transfer" +) -mapper(FTS3StagingOperation, fts3OperationTable, inherits=fts3Operation_mapper, polymorphic_identity="Staging") +mapper_registry.map_imperatively( + FTS3StagingOperation, fts3OperationTable, inherits=fts3Operation_mapper, polymorphic_identity="Staging" +) # About synchronize_session: @@ -156,6 +163,7 @@ # We set it to `False` simply because we do not rely on the session cache. # Please see https://github.com/sqlalchemy/sqlalchemy/discussions/6159 for detailed discussion + ######################################################################## class FTS3DB: """ @@ -171,13 +179,13 @@ def __getDBConnectionInfo(self, fullname): result = getDBParameters(fullname) if not result["OK"]: - raise Exception("Cannot get database parameters: %s" % result["Message"]) + raise Exception(f"Cannot get database parameters: {result['Message']}") dbParameters = result["Value"] self.dbHost = dbParameters["Host"] self.dbPort = dbParameters["Port"] self.dbUser = dbParameters["User"] - self.dbPass = dbParameters["Password"] + self.dbPass = quote_plus(dbParameters["Password"]) self.dbName = dbParameters["DBName"] def __init__(self, pool_size=15, url=None, parentLogger=None): @@ -234,7 +242,6 @@ def persistOperation(self, operation): # because of the merge we have to explicitely set lastUpdate operation.lastUpdate = utc_timestamp() try: - # Merge it in case it already is in the DB operation = session.merge(operation) session.add(operation) @@ -246,7 +253,7 @@ def persistOperation(self, operation): except SQLAlchemyError as e: session.rollback() self.log.exception("persistOperation: unexpected exception", lException=e) - return S_ERROR("persistOperation: unexpected exception %s" % e) + return S_ERROR(f"persistOperation: unexpected exception {e}") finally: session.close() @@ -264,7 +271,6 @@ def getOperation(self, operationID): session = self.dbSession(expire_on_commit=False) try: - operation = session.query(FTS3Operation).filter(getattr(FTS3Operation, "operationID") == operationID).one() session.commit() @@ -275,9 +281,9 @@ def getOperation(self, operationID): except NoResultFound as e: # We use the ENOENT error, even if not really a file error :) - return S_ERROR(errno.ENOENT, "No FTS3Operation with id %s" % operationID) + return S_ERROR(errno.ENOENT, f"No FTS3Operation with id {operationID}") except SQLAlchemyError as e: - return S_ERROR("getOperation: unexpected exception : %s" % e) + return S_ERROR(f"getOperation: unexpected exception : {e}") finally: session.close() @@ -305,7 +311,8 @@ def getActiveJobs(self, limit=20, lastMonitor=None, jobAssignmentTag="Assigned") ftsJobsQuery = ( session.query(FTS3Job) .join(FTS3Operation) - .filter(~FTS3Job.status.in_(FTS3Job.FINAL_STATES)) + .filter(FTS3Job.status.in_(FTS3Job.NON_FINAL_STATES)) + .filter(FTS3Operation.status == "Active") .filter(FTS3Job.assignment.is_(None)) .filter(FTS3Operation.assignment.is_(None)) ) @@ -322,7 +329,7 @@ def getActiveJobs(self, limit=20, lastMonitor=None, jobAssignmentTag="Assigned") ftsJobs = ftsJobsQuery.all() if jobAssignmentTag: - jobAssignmentTag += "_%s" % datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S") + jobAssignmentTag += f"_{datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')}" jobIds = [job.jobID for job in ftsJobs] if jobIds: @@ -341,7 +348,7 @@ def getActiveJobs(self, limit=20, lastMonitor=None, jobAssignmentTag="Assigned") except SQLAlchemyError as e: session.rollback() - return S_ERROR("getAllActiveJobs: unexpected exception : %s" % e) + return S_ERROR(f"getAllActiveJobs: unexpected exception : {e}") finally: session.close() @@ -364,13 +371,11 @@ def updateFileStatus(self, fileStatusDict, ftsGUID=None): # This here is inneficient as we update every files, even if it did not change, and we commit every time. # It would probably be best to update only the files that changed. # However, commiting every time is the recommendation of MySQL - # (https://dev.mysql.com/doc/refman/5.7/en/innodb-deadlocks-handling.html) + # (https://dev.mysql.com/doc/refman/8.0/en/innodb-deadlocks-handling.html) for fileID, valueDict in fileStatusDict.items(): - session = self.dbSession() try: - updateDict = {FTS3File.status: valueDict["status"]} # We only update error if it is specified @@ -413,7 +418,7 @@ def updateFileStatus(self, fileStatusDict, ftsGUID=None): except SQLAlchemyError as e: session.rollback() self.log.exception("updateFileFtsStatus: unexpected exception", lException=e) - return S_ERROR("updateFileFtsStatus: unexpected exception %s" % e) + return S_ERROR(f"updateFileFtsStatus: unexpected exception {e}") finally: session.close() @@ -428,9 +433,7 @@ def updateJobStatus(self, jobStatusDict): """ session = self.dbSession() try: - for jobID, valueDict in jobStatusDict.items(): - updateDict = {FTS3Job.status: valueDict["status"]} # We only update error if it is specified @@ -462,7 +465,7 @@ def updateJobStatus(self, jobStatusDict): except SQLAlchemyError as e: session.rollback() self.log.exception("updateJobStatus: unexpected exception", lException=e) - return S_ERROR("updateJobStatus: unexpected exception %s" % e) + return S_ERROR(f"updateJobStatus: unexpected exception {e}") finally: session.close() @@ -482,7 +485,6 @@ def cancelNonExistingJob(self, operationID, ftsGUID): """ session = self.dbSession() try: - # We update both the rows of the Jobs and the Files tables # having matching operationID and ftsGUID # https://docs.sqlalchemy.org/en/13/core/tutorial.html#multiple-table-updates @@ -495,9 +497,9 @@ def cancelNonExistingJob(self, operationID, ftsGUID): { FTS3File.status: "Canceled", FTS3File.ftsGUID: None, - FTS3File.error: "Job %s not found" % ftsGUID, + FTS3File.error: f"Job {ftsGUID} not found", FTS3Job.status: "Canceled", - FTS3Job.error: "Job %s not found" % ftsGUID, + FTS3Job.error: f"Job {ftsGUID} not found", } ) .where( @@ -519,7 +521,7 @@ def cancelNonExistingJob(self, operationID, ftsGUID): except SQLAlchemyError as e: session.rollback() self.log.exception("cancelNonExistingJob: unexpected exception", lException=e) - return S_ERROR("cancelNonExistingJob: unexpected exception %s" % e) + return S_ERROR(f"cancelNonExistingJob: unexpected exception {e}") finally: session.close() @@ -536,14 +538,13 @@ def getNonFinishedOperations(self, limit=20, operationAssignmentTag="Assigned"): session = self.dbSession(expire_on_commit=False) try: - ftsOperations = [] # We need to do the select in two times because the join clause that makes the limit difficult # We get the list of operations ID that have associated jobs assigned opIDsWithJobAssigned = select(FTS3Job.operationID).filter(~FTS3Job.assignment.is_(None)) operationIDsQuery = ( - session.query(FTS3Operation.operationID) + session.query(FTS3Operation.operationID, FTS3Operation.lastUpdate) .filter(FTS3Operation.status.in_(["Active", "Processed"])) .filter(FTS3Operation.assignment.is_(None)) .filter(~FTS3Operation.operationID.in_(opIDsWithJobAssigned)) @@ -565,7 +566,7 @@ def getNonFinishedOperations(self, limit=20, operationAssignmentTag="Assigned"): ftsOperations = session.query(FTS3Operation).filter(FTS3Operation.operationID.in_(operationIDs)).all() if operationAssignmentTag: - operationAssignmentTag += "_%s" % datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S") + operationAssignmentTag += f"_{datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')}" session.execute( update(FTS3Operation) @@ -581,7 +582,7 @@ def getNonFinishedOperations(self, limit=20, operationAssignmentTag="Assigned"): except SQLAlchemyError as e: session.rollback() - return S_ERROR("getAllProcessedOperations: unexpected exception : %s" % e) + return S_ERROR(f"getAllProcessedOperations: unexpected exception : {e}") finally: session.close() @@ -598,7 +599,6 @@ def kickStuckOperations(self, limit=20, kickDelay=2): session = self.dbSession(expire_on_commit=False) try: - ftsOps = ( session.query(FTS3Operation.operationID) .filter( @@ -631,7 +631,7 @@ def kickStuckOperations(self, limit=20, kickDelay=2): except SQLAlchemyError as e: session.rollback() - return S_ERROR("kickStuckOperations: unexpected exception : %s" % e) + return S_ERROR(f"kickStuckOperations: unexpected exception : {e}") finally: session.close() @@ -648,7 +648,6 @@ def kickStuckJobs(self, limit=20, kickDelay=2): session = self.dbSession(expire_on_commit=False) try: - ftsJobs = ( session.query(FTS3Job.jobID) .filter(FTS3Job.lastUpdate < (func.date_sub(utc_timestamp(), text("INTERVAL %d HOUR" % kickDelay)))) @@ -676,7 +675,7 @@ def kickStuckJobs(self, limit=20, kickDelay=2): except SQLAlchemyError as e: session.rollback() - return S_ERROR("kickStuckJobs: unexpected exception : %s" % e) + return S_ERROR(f"kickStuckJobs: unexpected exception : {e}") finally: session.close() @@ -690,13 +689,11 @@ def deleteFinalOperations(self, limit=20, deleteDelay=180): session = self.dbSession(expire_on_commit=False) + fromDate = datetime.datetime.utcnow() - datetime.timedelta(days=deleteDelay) try: - ftsOps = ( session.query(FTS3Operation.operationID) - .filter( - FTS3Operation.lastUpdate < (func.date_sub(utc_timestamp(), text("INTERVAL %d DAY" % deleteDelay))) - ) + .filter(FTS3Operation.lastUpdate < fromDate) .filter(FTS3Operation.status.in_(FTS3Operation.FINAL_STATES)) .limit(limit) ) @@ -718,7 +715,7 @@ def deleteFinalOperations(self, limit=20, deleteDelay=180): except SQLAlchemyError as e: session.rollback() - return S_ERROR("deleteFinalOperations: unexpected exception : %s" % e) + return S_ERROR(f"deleteFinalOperations: unexpected exception : {e}") finally: session.close() @@ -736,7 +733,6 @@ def getOperationsFromRMSOpID(self, rmsOpID): session = self.dbSession(expire_on_commit=False) try: - operations = session.query(FTS3Operation).filter(FTS3Operation.rmsOpID == rmsOpID).all() session.commit() @@ -749,6 +745,6 @@ def getOperationsFromRMSOpID(self, rmsOpID): # If there is no such operation, return an empty list return S_OK([]) except SQLAlchemyError as e: - return S_ERROR("getOperationsFromRMSOpID: unexpected exception : %s" % e) + return S_ERROR(f"getOperationsFromRMSOpID: unexpected exception : {e}") finally: session.close() diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DatasetManager/DatasetManager.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DatasetManager/DatasetManager.py index 77ede27369c..f4bb88da11a 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DatasetManager/DatasetManager.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DatasetManager/DatasetManager.py @@ -8,12 +8,11 @@ class DatasetManager: - _tables = dict() _tables["FC_MetaDatasets"] = { "Fields": { "DatasetID": "INT AUTO_INCREMENT", - "DatasetName": "VARCHAR(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL", + "DatasetName": "VARCHAR(128) CHARACTER SET utfmb4 COLLATE utf8mb4_bin NOT NULL", "MetaQuery": "VARCHAR(512)", "DirID": "INT NOT NULL DEFAULT 0", "TotalSize": "BIGINT UNSIGNED NOT NULL", @@ -64,7 +63,7 @@ def setDatabase(self, database): if not result["OK"]: gLogger.error("Failed to create tables", str(self._tables.keys())) elif result["Value"]: - gLogger.info("Tables created: %s" % ",".join(result["Value"])) + gLogger.info(f"Tables created: {','.join(result['Value'])}") return result def _getConnection(self, connection=False): @@ -100,7 +99,6 @@ def addDataset(self, datasets, credDict): return S_OK({"Successful": successful, "Failed": failed}) def __addDataset(self, datasetName, metaQuery, credDict, uid, gid): - result = self.__getMetaQueryParameters(metaQuery, credDict) if not result["OK"]: return result @@ -145,7 +143,7 @@ def __addDataset(self, datasetName, metaQuery, credDict, uid, gid): result = self.db.insertFields("FC_MetaDatasets", inDict=inDict) if not result["OK"]: if "Duplicate" in result["Message"]: - return S_ERROR("Dataset %s already exists" % datasetName) + return S_ERROR(f"Dataset {datasetName} already exists") else: return result datasetID = result["lastRowId"] @@ -161,7 +159,6 @@ def _getDatasetDirectories(self, datasets): return dirDict def _findDatasets(self, datasets, connection=False): - connection = self._getConnection(connection) fullNames = [name for name in datasets if name.startswith("/")] shortNames = [name for name in datasets if not name.startswith("/")] @@ -183,7 +180,6 @@ def _findDatasets(self, datasets, connection=False): return S_OK(resultDict) def __findFullPathDatasets(self, datasets, connection): - dirDict = self._getDatasetDirectories(datasets) failed = {} successful = {} @@ -208,7 +204,7 @@ def __findFullPathDatasets(self, datasets, connection): dirID = directoryIDs[dirPath] wheres.append("( DirID=%d AND DatasetName IN (%s) )" % (dirID, stringListToString(dsNames))) - req = "SELECT DatasetName,DirID,DatasetID FROM FC_MetaDatasets WHERE %s" % " OR ".join(wheres) + req = f"SELECT DatasetName,DirID,DatasetID FROM FC_MetaDatasets WHERE {' OR '.join(wheres)}" result = self.db._query(req, conn=connection) if not result["OK"]: return result @@ -224,12 +220,11 @@ def __findFullPathDatasets(self, datasets, connection): return S_OK({"Successful": successful, "Failed": failed}) def __findNoPathDatasets(self, nodirDatasets, connection): - failed = {} successful = {} dsIDs = {} req = "SELECT COUNT(DatasetName),DatasetName,DatasetID FROM FC_MetaDatasets WHERE DatasetName in " - req += "( %s ) GROUP BY DatasetName,DatasetID" % stringListToString(nodirDatasets) + req += f"( {stringListToString(nodirDatasets)} ) GROUP BY DatasetName,DatasetID" result = self.db._query(req, conn=connection) if not result["OK"]: return result @@ -293,7 +288,7 @@ def getDatasetAnnotation(self, datasets, credDict={}): failed[dataset] = "Dataset not found" idString = ",".join([str(x) for x in dsDict]) - req = "SELECT DatasetID, Annotation FROM FC_DatasetAnnotations WHERE DatasetID in (%s)" % idString + req = f"SELECT DatasetID, Annotation FROM FC_DatasetAnnotations WHERE DatasetID in ({idString})" else: req = "SELECT DatasetID, Annotation FROM FC_DatasetAnnotations" result = self.db._query(req) @@ -367,13 +362,13 @@ def removeDataset(self, datasets, credDict): def __removeDataset(self, datasetName, credDict): """Remove existing dataset""" - req = "SELECT DatasetID FROM FC_MetaDatasets WHERE DatasetName='%s'" % datasetName + req = f"SELECT DatasetID FROM FC_MetaDatasets WHERE DatasetName='{datasetName}'" result = self.db._query(req) if not result["OK"]: return result if not result["Value"]: # No requested dataset - return S_OK("Dataset %s does not exist" % datasetName) + return S_OK(f"Dataset {datasetName} does not exist") datasetID = result["Value"][0][0] for table in ["FC_MetaDatasetFiles", "FC_MetaDatasets", "FC_DatasetAnnotations"]: @@ -403,12 +398,12 @@ def checkDataset(self, datasets, credDict): def __checkDataset(self, datasetName, credDict): """Check that the dataset parameters correspond to the actual state""" req = "SELECT MetaQuery,DatasetHash,TotalSize,NumberOfFiles FROM FC_MetaDatasets" - req += " WHERE DatasetName='%s'" % datasetName + req += f" WHERE DatasetName='{datasetName}'" result = self.db._query(req) if not result["OK"]: return result if not result["Value"]: - return S_ERROR("Unknown MetaDataset %s" % datasetName) + return S_ERROR(f"Unknown MetaDataset {datasetName}") row = result["Value"][0] metaQuery = eval(row[0]) @@ -470,7 +465,7 @@ def __updateDataset(self, datasetName, credDict): for field in changeDict: req += f"{field}='{str(changeDict[field][1])}', " req += "ModificationDate=UTC_TIMESTAMP() " - req += "WHERE DatasetName='%s'" % datasetName + req += f"WHERE DatasetName='{datasetName}'" result = self.db._update(req) return result @@ -512,13 +507,13 @@ def __getDatasets(self, datasetName): ] parameterString = ",".join(parameterList) - req = "SELECT %s FROM FC_MetaDatasets" % parameterString + req = f"SELECT {parameterString} FROM FC_MetaDatasets" dsName = os.path.basename(datasetName) if "*" in dsName: dName = dsName.replace("*", "%") - req += " WHERE DatasetName LIKE '%s'" % dName + req += f" WHERE DatasetName LIKE '{dName}'" elif dsName: - req += " WHERE DatasetName='%s'" % dsName + req += f" WHERE DatasetName='{dsName}'" result = self.db._query(req) if not result["OK"]: @@ -598,7 +593,6 @@ def getDatasetsInDirectory(self, dirID, verbose=False, connection=False): return S_OK(datasets) def __getDatasetDict(self, row): - resultDict = {} resultDict["DatasetID"] = int(row[0]) resultDict["MetaQuery"] = eval(row[1]) @@ -691,7 +685,7 @@ def setDatasetStatus(self, datasetName, status): return result intStatus = result["Value"] req = "UPDATE FC_MetaDatasets SET Status=%d, ModificationDate=UTC_TIMESTAMP() " % intStatus - req += "WHERE DatasetName='%s'" % datasetName + req += f"WHERE DatasetName='{datasetName}'" result = self.db._update(req) return result @@ -812,7 +806,7 @@ def __freezeDataset(self, datasetName, credDict): for fileID in fileIDList: valueList.append("(%d,%d)" % (datasetID, fileID)) valueString = ",".join(valueList) - req = "INSERT INTO FC_MetaDatasetFiles (DatasetID,FileID) VALUES %s" % valueString + req = f"INSERT INTO FC_MetaDatasetFiles (DatasetID,FileID) VALUES {valueString}" result = self.db._update(req) if not result["OK"]: return result diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py index 4ad676ef8ef..e807feaa3a8 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryClosure.py @@ -7,6 +7,7 @@ you do it several times within 1 second, then there will be no changed, and affected = 0 """ +import errno import os from DIRAC import S_OK, S_ERROR @@ -170,7 +171,7 @@ def getPathIDs(self, path): if not result["OK"]: return result if not result["Value"]: - return S_ERROR("Directory %s not found" % path) + return S_ERROR(f"Directory {path} not found") dirID = result["Value"] @@ -200,7 +201,7 @@ def getChildren(self, path, connection=False): if not result["OK"]: return result if not result["Value"]: - return S_ERROR("Directory does not exist: %s" % path) + return S_ERROR(f"Directory does not exist: {path}") dirID = result["Value"] else: dirID = path @@ -226,7 +227,7 @@ def getSubdirectoriesByID(self, dirID, requestString=False, includeParent=False) """ if requestString: - reqStr = "SELECT ChildID FROM FC_DirectoryClosure WHERE ParentID = %s" % dirID + reqStr = f"SELECT ChildID FROM FC_DirectoryClosure WHERE ParentID = {dirID}" if not includeParent: reqStr += " AND Depth != 0" return S_OK(reqStr) @@ -393,7 +394,7 @@ def isEmpty(self, path): dirId = result["Value"] if not dirId: - return S_ERROR("Directory does not exist %s" % path) + return S_ERROR(f"Directory does not exist {path}") # Check if there are subdirectories result = self.countSubdirectories(dirId, includeParent=False) @@ -434,7 +435,7 @@ def getDirectoryParameters(self, pathOrDirId): elif isinstance(pathOrDirId, ((list,) + (int,))): psName = "ps_get_all_directory_info_from_id" else: - return S_ERROR("Unknown type of pathOrDirId %s" % type(pathOrDirId)) + return S_ERROR(f"Unknown type of pathOrDirId {type(pathOrDirId)}") result = self.db.executeStoredProcedureWithCursor(psName, (pathOrDirId,)) if not result["OK"]: @@ -454,7 +455,7 @@ def getDirectoryParameters(self, pathOrDirId): ] if not result["Value"]: - return S_ERROR("Directory does not exist %s" % pathOrDirId) + return S_ERROR(f"Directory does not exist {pathOrDirId}") row = result["Value"][0] @@ -507,7 +508,7 @@ def _setDirectoryParameter(self, path, pname, pvalue, recursive=False): # Either there were no changes, or the directory does not exist exists = self.existsDir(path).get("Value", {}).get("Exists") if not exists: - return S_ERROR(errno.ENOENT, "Directory does not exist: %s" % path) + return S_ERROR(errno.ENOENT, f"Directory does not exist: {path}") affected = 1 return S_OK(affected) @@ -657,3 +658,37 @@ def _changeDirectoryParameter(self, paths, directoryFunction, _fileFunction, rec successful[path] = True return S_OK({"Successful": successful, "Failed": failed}) + + def _getDirectoryDump(self, path): + """Recursively dump all the content of a directory + + :param str path: directory to dump + + :returns: dictionary with `Files` and `SubDirs` as keys + `Files` is a dict containing files metadata. + `SubDirs` is a list of directory + """ + + result = self.findDir(path) + if not result["OK"]: + return result + dirID = result["Value"] + if not dirID: + return S_ERROR(errno.ENOENT, f"{path} does not exist") + + result = self.db.executeStoredProcedureWithCursor("ps_get_directory_dump", (dirID,)) + + if not result["OK"]: + return result + + rows = result["Value"] + files = {} + subDirs = [] + + for lfn, size, creationDate in rows: + if size is None: + subDirs.append(lfn) + else: + files[lfn] = {"Size": int(size), "CreationDate": creationDate} + + return S_OK({"Files": files, "SubDirs": subDirs}) diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryFlatTree.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryFlatTree.py index 92cd78ac893..75f862e0599 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryFlatTree.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryFlatTree.py @@ -139,7 +139,7 @@ def makeDirectory(self, path, credDict, status=0): result = self.db._update(req) if not result["OK"]: self.removeDir(path) - return S_ERROR("Failed to create directory %s" % path) + return S_ERROR(f"Failed to create directory {path}") return S_OK(result["lastRowId"]) def makeDir(self, path): @@ -208,12 +208,12 @@ def getPathIDs(self, path): pelements.append(dPath) pathString = ["'" + p + "'" for p in pelements] - req = "SELECT DirID FROM DirectoryInfo WHERE DirName in (%s) ORDER BY DirID" % ",".join(pathString) + req = f"SELECT DirID FROM DirectoryInfo WHERE DirName in ({','.join(pathString)}) ORDER BY DirID" result = self.db._query(req) if not result["OK"]: return result if not result["Value"]: - return S_ERROR("Directory %s not found" % path) + return S_ERROR(f"Directory {path} not found") return S_OK([x[0] for x in result["Value"]]) def getChildren(self, path): diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryLevelTree.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryLevelTree.py index 3116126a951..6bdddfa71dd 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryLevelTree.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryLevelTree.py @@ -19,7 +19,6 @@ def __init__(self, database=None): self.treeTable = "FC_DirectoryLevelTree" def getTreeType(self): - return "Directory" def findDir(self, path, connection=False): @@ -29,7 +28,7 @@ def findDir(self, path, connection=False): if not dpath["OK"]: return dpath dpath = dpath["Value"] - req = "SELECT DirID,Level from FC_DirectoryLevelTree WHERE DirName=%s" % dpath + req = f"SELECT DirID,Level from FC_DirectoryLevelTree WHERE DirName={dpath}" result = self.db._query(req, conn=connection) if not result["OK"]: return result @@ -50,7 +49,7 @@ def findDirs(self, paths, connection=False): return dpath dpathList.append(dpath["Value"]) dpaths = ",".join(dpathList) - req = "SELECT DirName,DirID from FC_DirectoryLevelTree WHERE DirName in (%s)" % dpaths + req = f"SELECT DirName,DirID from FC_DirectoryLevelTree WHERE DirName in ({dpaths})" result = self.db._query(req, conn=connection) if not result["OK"]: return result @@ -141,10 +140,10 @@ def makeDir(self, path): result = self.db._getConnection() conn = result["Value"] - # result = self.db._query("LOCK TABLES FC_DirectoryLevelTree WRITE; ",conn) - result = self.db.insertFields("FC_DirectoryLevelTree", names, values, conn) + # result = self.db._query("LOCK TABLES FC_DirectoryLevelTree WRITE; ", conn=conn) + result = self.db.insertFields("FC_DirectoryLevelTree", names, values, conn=conn) if not result["OK"]: - # resUnlock = self.db._query("UNLOCK TABLES;",conn) + # resUnlock = self.db._query("UNLOCK TABLES;", conn=conn) if result["Message"].find("Duplicate") != -1: # The directory is already added resFind = self.findDir(path) @@ -162,25 +161,25 @@ def makeDir(self, path): if parentDirID: # lPath = "LPATH%d" % (level) # req = " SELECT @tmpvar:=max(%s)+1 FROM FC_DirectoryLevelTree WHERE Parent=%d; " % (lPath,parentDirID) - # resultLock = self.db._query("LOCK TABLES FC_DirectoryLevelTree WRITE; ",conn) - # result = self.db._query(req,conn) + # resultLock = self.db._query("LOCK TABLES FC_DirectoryLevelTree WRITE; ", conn=conn) + # result = self.db._query(req, conn=conn) # req = "UPDATE FC_DirectoryLevelTree SET %s=@tmpvar WHERE DirID=%d; " % (lPath,dirID) - # result = self.db._update(req,conn) - # result = self.db._query("UNLOCK TABLES;",conn) + # result = self.db._update(req, conn=conn) + # result = self.db._query("UNLOCK TABLES;", conn=conn) lPath = "LPATH%d" % (level) req = " SELECT @tmpvar:=max(%s)+1 FROM FC_DirectoryLevelTree WHERE Parent=%d FOR UPDATE; " % ( lPath, parentDirID, ) - resultLock = self.db._query("START TRANSACTION; ", conn) - result = self.db._query(req, conn) + resultLock = self.db._query("START TRANSACTION; ", conn=conn) + result = self.db._query(req, conn=conn) req = "UPDATE FC_DirectoryLevelTree SET %s=@tmpvar WHERE DirID=%d; " % (lPath, dirID) - result = self.db._update(req, conn) - result = self.db._query("COMMIT;", conn) + result = self.db._update(req, conn=conn) + result = self.db._query("COMMIT;", conn=conn) if not result["OK"]: return result else: - result = self.db._query("ROLLBACK;", conn) + result = self.db._query("ROLLBACK;", conn=conn) result = S_OK(dirID) result["NewDirectory"] = True @@ -244,12 +243,12 @@ def getDirectoryPaths(self, dirIDList): return S_OK({}) dirListString = ",".join([str(d) for d in dirs]) - req = "SELECT DirID,DirName FROM FC_DirectoryLevelTree WHERE DirID in ( %s )" % dirListString + req = f"SELECT DirID,DirName FROM FC_DirectoryLevelTree WHERE DirID in ( {dirListString} )" result = self.db._query(req) if not result["OK"]: return result if not result["Value"]: - return S_ERROR("Directories not found: %s" % dirListString) + return S_ERROR(f"Directories not found: {dirListString}") resultDict = {} for row in result["Value"]: @@ -275,17 +274,22 @@ def getPathIDs(self, path): pelements = [] dPath = "" for el in elements[1:]: + # skip entry path elements, e.g. path has trailing slash + if not el: + continue dPath += "/" + el pelements.append(dPath) pelements.append("/") pathString = ["'" + p + "'" for p in pelements] - req = "SELECT DirID FROM FC_DirectoryLevelTree WHERE DirName in (%s) ORDER BY DirID" % ",".join(pathString) + req = f"SELECT DirID FROM FC_DirectoryLevelTree WHERE DirName in ({','.join(pathString)}) ORDER BY DirID" result = self.db._query(req) if not result["OK"]: return result + if len(result["Value"]) != len(pelements): + return S_ERROR(f"Directory {path} not found") if not result["Value"]: - return S_ERROR("Directory %s not found" % path) + return S_ERROR(f"Directory {path} not found") return S_OK([x[0] for x in result["Value"]]) @@ -319,7 +323,7 @@ def getPathIDsByID(self, dirID): sel = " AND ".join(["Level=%d" % lev] + ["LPATH%d=%d" % (ll + 1, lpaths[ll]) for ll in range(lev)]) lpathSelects.append(sel) selection = "(" + ") OR (".join(lpathSelects) + ")" - req = "SELECT Level,DirID from FC_DirectoryLevelTree WHERE %s ORDER BY Level" % selection + req = f"SELECT Level,DirID from FC_DirectoryLevelTree WHERE {selection} ORDER BY Level" result = self.db._query(req) if not result["OK"]: return result @@ -335,7 +339,7 @@ def getChildren(self, path, connection=False): if not result["OK"]: return result if not result["Value"]: - return S_ERROR("Directory does not exist: %s" % path) + return S_ERROR(f"Directory does not exist: {path}") dirID = result["Value"] else: dirID = path @@ -415,7 +419,7 @@ def getAllSubdirectoriesByID(self, dirList): while parentList: subResult = [] dirListString = ",".join([str(d) for d in parentList]) - req = "SELECT DirID from FC_DirectoryLevelTree WHERE Parent in ( %s )" % dirListString + req = f"SELECT DirID from FC_DirectoryLevelTree WHERE Parent in ( {dirListString} )" result = self.db._query(req) if not result["OK"]: return result @@ -451,7 +455,6 @@ def recoverOrphanDirectories(self, credDict): parentDict = {} for dirID, parentID, _level in result["Value"]: - result = self.getDirectoryPath(dirID) if not result["OK"]: continue diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectorySimpleTree.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectorySimpleTree.py index 489d49f24b8..83725c6887d 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectorySimpleTree.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectorySimpleTree.py @@ -16,8 +16,7 @@ def __init__(self, database=None): self.treeTable = "FC_DirectoryTree" def findDir(self, path): - - req = "SELECT DirID from FC_DirectoryTree WHERE DirName='%s'" % path + req = f"SELECT DirID from FC_DirectoryTree WHERE DirName='{path}'" result = self.db._query(req) if not result["OK"]: return result @@ -42,7 +41,6 @@ def removeDir(self, path): return result def makeDir(self, path): - result = self.findDir(path) if not result["OK"]: return result @@ -118,12 +116,12 @@ def getPathIDs(self, path): pelements.append(dPath) pathString = ["'" + p + "'" for p in pelements] - req = "SELECT DirID FROM FC_DirectoryTree WHERE DirName in (%s) ORDER BY DirID" % pathString + req = f"SELECT DirID FROM FC_DirectoryTree WHERE DirName in ({pathString}) ORDER BY DirID" result = self.db._query(req) if not result["OK"]: return result if not result["Value"]: - return S_ERROR("Directory %s not found" % path) + return S_ERROR(f"Directory {path} not found") return S_OK([x[0] for x in result["Value"]]) diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryTreeBase.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryTreeBase.py index 09de98028a6..09cb78e2f14 100755 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryTreeBase.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryManager/DirectoryTreeBase.py @@ -1,4 +1,5 @@ """ DIRAC DirectoryTree base class """ +import errno import time import threading import os @@ -33,11 +34,9 @@ def findDirs(self, paths, connection=False): return S_ERROR("To be implemented on derived class") def makeDir(self, path): - return S_ERROR("To be implemented on derived class") def removeDir(self, path): - return S_ERROR("To be implemented on derived class") def getChildren(self, path, connection=False): @@ -116,7 +115,7 @@ def makeDirectory(self, path, credDict, status=0): if not dirDict: self.removeDir(path) - return S_ERROR("Failed to create directory %s" % path) + return S_ERROR(f"Failed to create directory {path}") return S_OK(dirID) ##################################################################### @@ -278,7 +277,7 @@ def __getDirID(self, path): return result dirID = result["Value"] if not dirID: - return S_ERROR("%s: not found" % str(path)) + return S_ERROR(f"{str(path)}: not found") return S_OK(dirID) else: return S_OK(path) @@ -557,7 +556,7 @@ def getFileIDsInDirectoryWithLimits(self, dirID, credDict, startItem=1, maxItems dirListString = ",".join([str(dir) for dir in dirs]) - req = "SELECT COUNT( DirID ) FROM FC_Files USE INDEX (DirID) WHERE DirID IN ( %s )" % dirListString + req = f"SELECT COUNT( DirID ) FROM FC_Files USE INDEX (DirID) WHERE DirID IN ( {dirListString} )" result = self.db._query(req) if not result["OK"]: return result @@ -676,6 +675,89 @@ def listDirectory(self, lfns, verbose=False): return S_OK({"Successful": successful, "Failed": failed}) + def getDirectoryDump(self, lfns): + """Get the dump of the directories in lfns""" + successful = {} + failed = {} + for path in lfns: + result = self._getDirectoryDump(path) + if not result["OK"]: + failed[path] = result["Message"] + else: + successful[path] = result["Value"] + + return S_OK({"Successful": successful, "Failed": failed}) + + def _getDirectoryDump(self, path): + """ + Recursively dump all the content of a directory + + :param str path: directory to dump + + :returns: dictionary with `Files` and `SubDirs` as keys + `Files` is a dict containing files metadata. + `SubDirs` is a list of directory + """ + result = self.findDir(path) + if not result["OK"]: + return result + directoryID = result["Value"] + if not directoryID: + return S_ERROR(errno.ENOENT, f"{path} does not exist") + directories = [] + + result = self.db.fileManager.getFilesInDirectory(directoryID) + if not result["OK"]: + return result + + filesInDir = result["Value"] + files = { + os.path.join(path, fileName): { + "Size": fileMetadata["MetaData"]["Size"], + "CreationDate": fileMetadata["MetaData"]["CreationDate"], + } + for fileName, fileMetadata in filesInDir.items() + } + + dirIDList = [directoryID] + + while dirIDList: + curDirID = dirIDList.pop() + result = self.getChildren(curDirID) + if not result["OK"]: + return result + newDirIDList = result["Value"] + for dirID in newDirIDList: + result = self.getDirectoryPath(dirID) + if not result["OK"]: + return result + dirName = result["Value"] + + directories.append(dirName) + + result = self.db.fileManager.getFilesInDirectory(dirID) + if not result["OK"]: + return result + + filesInDir = result["Value"] + + files.update( + { + os.path.join(dirName, fileName): { + "Size": fileMetadata["MetaData"]["Size"], + "CreationDate": fileMetadata["MetaData"]["CreationDate"], + } + for fileName, fileMetadata in filesInDir.items() + } + ) + + # Add to this list to get subdirectories of these directories + dirIDList.extend(newDirIDList) + + pathDict = {"Files": files, "SubDirs": directories} + + return S_OK(pathDict) + def getDirectoryReplicas(self, lfns, allStatus=False): """Get replicas for files in the given directories""" successful = {} @@ -809,10 +891,9 @@ def _getDirectoryLogicalSize(self, lfns, recursiveSum=True, connection=None): failed = {} treeTable = self.getTreeTable() for path in lfns: - if path == "/": req = "SELECT SUM(Size),COUNT(*) FROM FC_Files" - reqDir = "SELECT count(*) FROM %s" % treeTable + reqDir = f"SELECT count(*) FROM {treeTable}" else: result = self.findDir(path) if not result["OK"]: @@ -910,7 +991,6 @@ def _getDirectoryPhysicalSizeFromUsage_old(self, lfns, connection): successful = {} failed = {} for path in lfns: - if path == "/": req = "SELECT S.SEName, D.SESize, D.SEFiles FROM FC_DirectoryUsage as D, FC_StorageElements as S" req += " WHERE S.SEID=D.SEID" @@ -928,7 +1008,7 @@ def _getDirectoryPhysicalSizeFromUsage_old(self, lfns, connection): return result subDirString = result["Value"] req = "SELECT S.SEName, D.SESize, D.SEFiles FROM FC_DirectoryUsage as D, FC_StorageElements as S" - req += " JOIN (%s) AS F" % subDirString + req += f" JOIN ({subDirString}) AS F" req += " WHERE S.SEID=D.SEID AND D.DirID=F.DirID" result = self.db._query(req, conn=connection) @@ -1047,7 +1127,7 @@ def __rebuildDirectoryUsageLeaves(self): return result dirIDs = [x[0] for x in result["Value"]] - gLogger.verbose("Starting rebuilding Directory Usage, number of visible directories %d" % len(dirIDs)) + gLogger.verbose(f"Starting rebuilding Directory Usage, number of visible directories {len(dirIDs)}") insertFields = ["DirID", "SEID", "SESize", "SEFiles", "LastUpdate"] insertCount = 0 @@ -1057,7 +1137,6 @@ def __rebuildDirectoryUsageLeaves(self): empty = 0 for dirID in dirIDs: - count += 1 # Get the physical size @@ -1100,7 +1179,7 @@ def __rebuildDirectoryUsageLeaves(self): seSize, seFiles, ) - req += " WHERE DirID=%s AND SEID=0" % dirID + req += f" WHERE DirID={dirID} AND SEID=0" result = self.db._update(req) if not result["OK"]: return result @@ -1181,13 +1260,13 @@ def getDirectoryCounters(self, connection=False): return res resultDict["Empty Directories"] = res["Value"][0][0] - req = "SELECT COUNT(DirID) FROM %s WHERE DirID NOT IN ( SELECT DirID FROM FC_DirectoryInfo )" % treeTable + req = f"SELECT COUNT(DirID) FROM {treeTable} WHERE DirID NOT IN ( SELECT DirID FROM FC_DirectoryInfo )" res = self.db._query(req, conn=connection) if not res["OK"]: return res resultDict["DirTree w/o DirInfo"] = res["Value"][0][0] - req = "SELECT COUNT(DirID) FROM FC_DirectoryInfo WHERE DirID NOT IN ( SELECT DirID FROM %s )" % treeTable + req = f"SELECT COUNT(DirID) FROM FC_DirectoryInfo WHERE DirID NOT IN ( SELECT DirID FROM {treeTable} )" res = self.db._query(req, conn=connection) if not res["OK"]: return res diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryMetadata/DirectoryMetadata.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryMetadata/DirectoryMetadata.py index 0f8abc8775d..d6a511a31f9 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryMetadata/DirectoryMetadata.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryMetadata/DirectoryMetadata.py @@ -10,7 +10,6 @@ class DirectoryMetadata: def __init__(self, database=None): - self.db = database def setDatabase(self, database): @@ -34,7 +33,7 @@ def addMetadataField(self, pName, pType, credDict): if not result["OK"]: return result if pName in result["Value"]: - return S_ERROR("The metadata %s is already defined for Files" % pName) + return S_ERROR(f"The metadata {pName} is already defined for Files") result = self._getMetadataFields(credDict) if not result["OK"]: @@ -42,9 +41,7 @@ def addMetadataField(self, pName, pType, credDict): if pName in result["Value"]: if pType.lower() == result["Value"][pName].lower(): return S_OK("Already exists") - return S_ERROR( - "Attempt to add an existing metadata with different type: {}/{}".format(pType, result["Value"][pName]) - ) + return S_ERROR(f"Attempt to add an existing metadata with different type: {pType}/{result['Value'][pName]}") valueType = pType if pType.lower()[:3] == "int": @@ -86,12 +83,12 @@ def deleteMetadataField(self, pName, credDict): :return: S_OK/S_ERROR """ - req = "DROP TABLE FC_Meta_%s" % pName + req = f"DROP TABLE FC_Meta_{pName}" result = self.db._update(req) error = "" if not result["OK"]: error = result["Message"] - req = "DELETE FROM FC_MetaFields WHERE MetaName='%s'" % pName + req = f"DELETE FROM FC_MetaFields WHERE MetaName='{pName}'" result = self.db._update(req) if not result["OK"]: if error: @@ -141,7 +138,7 @@ def addMetadataSet(self, metaSetName, metaSetDict, credDict): # Check the sanity of the metadata set contents for key in metaSetDict: if key not in metaTypeDict: - return S_ERROR("Unknown key %s" % key) + return S_ERROR(f"Unknown key {key}") result = self.db.insertFields("FC_MetaSetNames", ["MetaSetName"], [metaSetName]) if not result["OK"]: @@ -172,7 +169,7 @@ def getMetadataSet(self, metaSetName, expandFlag, credDict): metaTypeDict = result["Value"] req = "SELECT S.MetaKey,S.MetaValue FROM FC_MetaSets as S, FC_MetaSetNames as N " - req += "WHERE N.MetaSetName='%s' AND N.MetaSetID=S.MetaSetID" % metaSetName + req += f"WHERE N.MetaSetName='{metaSetName}' AND N.MetaSetID=S.MetaSetID" result = self.db._query(req) if not result["OK"]: return result @@ -183,7 +180,7 @@ def getMetadataSet(self, metaSetName, expandFlag, credDict): resultDict = {} for key, value in result["Value"]: if key not in metaTypeDict: - return S_ERROR("Unknown key %s" % key) + return S_ERROR(f"Unknown key {key}") if expandFlag: if metaTypeDict[key] == "MetaSet": result = self.getMetadataSet(value, expandFlag, credDict) @@ -220,7 +217,7 @@ def setMetadata(self, dPath, metaDict, credDict): if not result["OK"]: return result if not result["Value"]: - return S_ERROR("Path not found: %s" % dPath) + return S_ERROR(f"Path not found: {dPath}") dirID = result["Value"] dirmeta = self.getDirectoryMetadata(dPath, credDict, ownData=False) @@ -232,7 +229,7 @@ def setMetadata(self, dPath, metaDict, credDict): for metaName, metaValue in metaDict.items(): if metaName not in metaFields: if forceIndex: - return S_ERROR("Field %s not indexed, but ForceIndexedMetadata is set" % metaName, callStack=[]) + return S_ERROR(f"Field {metaName} not indexed, but ForceIndexedMetadata is set", callStack=[]) result = self.setMetaParameter(dPath, metaName, metaValue, credDict) if not result["OK"]: return result @@ -240,7 +237,7 @@ def setMetadata(self, dPath, metaDict, credDict): # Check that the metadata is not defined for the parent directories if metaName in dirmeta["Value"]: return S_ERROR(f"Metadata conflict detected for {metaName} for directory {dPath}") - result = self.db.insertFields("FC_Meta_%s" % metaName, ["DirID", "Value"], [dirID, metaValue]) + result = self.db.insertFields(f"FC_Meta_{metaName}", ["DirID", "Value"], [dirID, metaValue]) if not result["OK"]: if result["Message"].find("Duplicate") != -1: req = "UPDATE FC_Meta_%s SET Value='%s' WHERE DirID=%d" % (metaName, metaValue, dirID) @@ -270,7 +267,7 @@ def removeMetadata(self, dPath, metaData, credDict): if not result["OK"]: return result if not result["Value"]: - return S_ERROR("Path not found: %s" % dPath) + return S_ERROR(f"Path not found: {dPath}") dirID = result["Value"] failedMeta = {} @@ -290,7 +287,7 @@ def removeMetadata(self, dPath, metaData, credDict): if failedMeta: metaExample = list(failedMeta)[0] - result = S_ERROR("Failed to remove %d metadata, e.g. %s" % (len(failedMeta), failedMeta[metaExample])) + result = S_ERROR(f"Failed to remove {len(failedMeta)} metadata, e.g. {failedMeta[metaExample]}") result["FailedMetadata"] = failedMeta else: return S_OK() @@ -310,7 +307,7 @@ def setMetaParameter(self, dPath, metaName, metaValue, credDict): if not result["OK"]: return result if not result["Value"]: - return S_ERROR("Path not found: %s" % dPath) + return S_ERROR(f"Path not found: {dPath}") dirID = result["Value"] result = self.db.insertFields( @@ -337,13 +334,13 @@ def getDirectoryMetaParameters(self, dpath, credDict, inherited=True): if not result["OK"]: return result if not result["Value"]: - return S_ERROR("Path not found: %s" % dpath) + return S_ERROR(f"Path not found: {dpath}") dirID = result["Value"] pathIDs = [dirID] if len(pathIDs) > 1: pathString = ",".join([str(x) for x in pathIDs]) - req = "SELECT DirID,MetaKey,MetaValue from FC_DirMeta where DirID in (%s)" % pathString + req = f"SELECT DirID,MetaKey,MetaValue from FC_DirMeta where DirID in ({pathString})" else: req = "SELECT DirID,MetaKey,MetaValue from FC_DirMeta where DirID=%d " % dirID result = self.db._query(req) @@ -353,11 +350,8 @@ def getDirectoryMetaParameters(self, dpath, credDict, inherited=True): return S_OK({}) metaDict = {} for _dID, key, value in result["Value"]: - if key in metaDict: - if isinstance(metaDict[key], list): - metaDict[key].append(value) - else: - metaDict[key] = [metaDict[key]].append(value) + if isinstance(metaDict.get(key), list): + metaDict[key].append(value) else: metaDict[key] = value @@ -432,7 +426,7 @@ def __transformMetaParameterToData(self, metaName): :return: S_OK/S_ERROR """ - req = "SELECT DirID,MetaValue from FC_DirMeta WHERE MetaKey='%s'" % metaName + req = f"SELECT DirID,MetaValue from FC_DirMeta WHERE MetaKey='{metaName}'" result = self.db._query(req) if not result["OK"]: return result @@ -460,12 +454,12 @@ def __transformMetaParameterToData(self, metaName): for dirID in dirList: insertValueList.append("( %d,'%s' )" % (dirID, dirDict[dirID])) - req = "INSERT INTO FC_Meta_{} (DirID,Value) VALUES {}".format(metaName, ", ".join(insertValueList)) + req = f"INSERT INTO FC_Meta_{metaName} (DirID,Value) VALUES {', '.join(insertValueList)}" result = self.db._update(req) if not result["OK"]: return result - req = "DELETE FROM FC_DirMeta WHERE MetaKey='%s'" % metaName + req = f"DELETE FROM FC_DirMeta WHERE MetaKey='{metaName}'" result = self.db._update(req) return result @@ -537,14 +531,14 @@ def __findSubdirByMeta(self, metaName, value, pathSelection="", subdirFlag=True) return result selectString = result["Value"] - req = " SELECT M.DirID FROM FC_Meta_%s AS M" % metaName + req = f" SELECT M.DirID FROM FC_Meta_{metaName} AS M" if pathSelection: - req += " JOIN ( %s ) AS P WHERE M.DirID=P.DirID" % pathSelection + req += f" JOIN ( {pathSelection} ) AS P WHERE M.DirID=P.DirID" if selectString: if pathSelection: - req += " AND %s" % selectString + req += f" AND {selectString}" else: - req += " WHERE %s" % selectString + req += f" WHERE {selectString}" result = self.db._query(req) if not result["OK"]: @@ -586,7 +580,7 @@ def __findSubdirMissingMeta(self, metaName, pathSelection): if dirList: req = f"SELECT DirID FROM {table} WHERE DirID NOT IN ( {dirString} )" else: - req = "SELECT DirID FROM %s" % table + req = f"SELECT DirID FROM {table}" result = self.db._query(req) if not result["OK"]: return result @@ -625,7 +619,7 @@ def __expandMetaDictionary(self, metaDict, credDict): mDict = result["Value"] for mk, mv in mDict.items(): if mk in resultDict: - return S_ERROR("Contradictory query for key %s" % mk) + return S_ERROR(f"Contradictory query for key {mk}") else: resultDict[mk] = mv @@ -899,9 +893,9 @@ def __findDistinctMetadata(self, metaList, dList): dString = None metaDict = {} for meta in metaList: - req = "SELECT DISTINCT(Value) FROM FC_Meta_%s" % meta + req = f"SELECT DISTINCT(Value) FROM FC_Meta_{meta}" if dString: - req += " WHERE DirID in (%s)" % dString + req += f" WHERE DirID in ({dString})" result = self.db._query(req) if not result["OK"]: return result @@ -928,7 +922,7 @@ def getCompatibleMetadata(self, queryDict, path, credDict): if not result["OK"]: return result if not result["Value"]: - return S_ERROR("Path not found: %s" % path) + return S_ERROR(f"Path not found: {path}") pathDirID = int(result["Value"]) pathDirs = [] if pathDirID: diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryMetadata/MultiVODirectoryMetadata.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryMetadata/MultiVODirectoryMetadata.py index e0aa1d625d7..879bae5d536 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryMetadata/MultiVODirectoryMetadata.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/DirectoryMetadata/MultiVODirectoryMetadata.py @@ -158,17 +158,17 @@ def setMetaParameter(self, dPath, metaName, metaValue, credDict): def getDirectoryMetadata(self, path, credDict, inherited=True, ownData=True): """ - Get metadata for the given directory aggregating metadata for the directory itself - and for all the parent directories if inherited flag is True. Get also the non-indexed - metadata parameters. - - :param str path: directory path - :param dict credDict: client credential dictionary - :param bool inherited: include parent directories if True - :param bool ownData: - :return: standard Dirac result object + additional MetadataOwner \ - and MetadataType dict entries if the operation is successful. - """ + Get metadata for the given directory aggregating metadata for the directory itself + and for all the parent directories if inherited flag is True. Get also the non-indexed + metadata parameters. + + :param str path: directory path + :param dict credDict: client credential dictionary + :param bool inherited: include parent directories if True + :param bool ownData: + :return: standard Dirac result object + additional MetadataOwner \ + and MetadataType dict entries if the operation is successful. + """ result = super().getDirectoryMetadata(path, credDict, inherited, ownData) if not result["OK"]: @@ -181,9 +181,16 @@ def getDirectoryMetadata(self, path, credDict, inherited=True, ownData=True): return result - def findDirIDsByMetadata(self, metaDict, dPath, credDict): - """Find Directories satisfying the given metadata and being subdirectories of - the given path + def findDirectoriesByMetadata(self, queryDict, path, credDict): """ - fMetaDict = _getMetaNameDict(metaDict, credDict) - return super().findDirIDsByMetadata(fMetaDict, dPath, credDict) + Find Directory names satisfying the given metadata and being subdirectories of + the given path. + + :param dict queryDict: dictionary containing query data + :param str path: starting directory path + :param dict credDict: client credential dictionary + :return: S_OK/S_ERROR, Value list of selected directory paths + """ + + fMetaDict = _getMetaNameDict(queryDict, credDict) + return super().findDirectoriesByMetadata(fMetaDict, path, credDict) diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManager.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManager.py index 78851b7a6a3..c2ec3e42b3f 100755 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManager.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManager.py @@ -12,7 +12,6 @@ class FileManager(FileManagerBase): - ###################################################### # # The all important _findFiles and _getDirectoryFiles methods @@ -88,7 +87,7 @@ def _findFileIDs(self, lfns, connection=False): dirID = directoryIDs[dirPath] wheres.append("( DirID=%d AND FileName IN (%s) )" % (dirID, stringListToString(fileNames))) - req = "SELECT FileName,DirID,FileID FROM FC_Files WHERE %s" % " OR ".join(wheres) + req = f"SELECT FileName,DirID,FileID FROM FC_Files WHERE {' OR '.join(wheres)}" result = self.db._query(req, conn=connection) if not result["OK"]: return result @@ -200,17 +199,17 @@ def _getDirectoryFileIDs(self, dirID, requestString=False): dirListString = result["Value"] if requestString: - req = "SELECT FileID FROM FC_Files WHERE DirID IN ( %s )" % dirListString + req = f"SELECT FileID FROM FC_Files WHERE DirID IN ( {dirListString} )" return S_OK(req) - req = "SELECT FileID,DirID,FileName FROM FC_Files WHERE DirID IN ( %s )" % dirListString + req = f"SELECT FileID,DirID,FileName FROM FC_Files WHERE DirID IN ( {dirListString} )" result = self.db._query(req) return result def _getFileMetadataByID(self, fileIDs, connection=False): """Get standard file metadata for a list of files specified by FileID""" - stringIDs = ",".join(["%s" % id_ for id_ in fileIDs]) - req = "SELECT FileID,Size,UID,GID,Status FROM FC_Files WHERE FileID in ( %s )" % stringIDs + stringIDs = ",".join([f"{id_}" for id_ in fileIDs]) + req = f"SELECT FileID,Size,UID,GID,Status FROM FC_Files WHERE FileID in ( {stringIDs} )" result = self.db._query(req, conn=connection) if not result["OK"]: return result @@ -223,7 +222,7 @@ def _getFileMetadataByID(self, fileIDs, connection=False): "Status": self._getIntStatus(status).get("Value", status), } - req = "SELECT FileID,GUID,CreationDate from FC_FileInfo WHERE FileID in ( %s )" % stringIDs + req = f"SELECT FileID,GUID,CreationDate from FC_FileInfo WHERE FileID in ( {stringIDs} )" result = self.db._query(req, conn=connection) if not result["OK"]: return result @@ -239,7 +238,6 @@ def _getFileMetadataByID(self, fileIDs, connection=False): # def _insertFiles(self, lfns, uid, gid, connection=False): - connection = self._getConnection(connection) # Add the files failed = {} @@ -267,7 +265,7 @@ def _insertFiles(self, lfns, uid, gid, connection=False): directorySESizeDict[dirID][0]["Size"] += lfns[lfn]["Size"] directorySESizeDict[dirID][0]["Files"] += 1 - req = "INSERT INTO FC_Files (DirID,Size,UID,GID,Status,FileName) VALUES %s" % (",".join(insertTuples)) + req = f"INSERT INTO FC_Files (DirID,Size,UID,GID,Status,FileName) VALUES {','.join(insertTuples)}" res = self.db._update(req, conn=connection) if not res["OK"]: return res @@ -299,7 +297,7 @@ def _insertFiles(self, lfns, uid, gid, connection=False): ) if insertTuples: fields = "FileID,GUID,Checksum,ChecksumType,CreationDate,ModificationDate,Mode" - req = "INSERT INTO FC_FileInfo ({}) VALUES {}".format(fields, ",".join(insertTuples)) + req = f"INSERT INTO FC_FileInfo ({fields}) VALUES {','.join(insertTuples)}" res = self.db._update(req) if not res["OK"]: self._deleteFiles(toDelete, connection=connection) @@ -320,7 +318,7 @@ def _getFileIDFromGUID(self, guid, connection=False): return S_OK({}) if not isinstance(guid, (list, tuple)): guid = [guid] - req = "SELECT FileID,GUID FROM FC_FileInfo WHERE GUID IN (%s)" % stringListToString(guid) + req = f"SELECT FileID,GUID FROM FC_FileInfo WHERE GUID IN ({stringListToString(guid)})" res = self.db._query(req, conn=connection) if not res["OK"]: return res @@ -338,7 +336,7 @@ def getLFNForGUID(self, guids, connection=False): if not isinstance(guids, (list, tuple)): guids = [guids] req = "SELECT f.FileID, f.FileName, fi.GUID, f.DirID FROM FC_FileInfo fi" - req += " JOIN FC_Files f on fi.FileID = f.FileID WHERE GUID IN (%s)" % stringListToString(guids) + req += f" JOIN FC_Files f on fi.FileID = f.FileID WHERE GUID IN ({stringListToString(guids)})" res = self.db._query(req, conn=connection) if not res["OK"]: return res @@ -401,10 +399,10 @@ def __deleteFiles(self, fileIDs, connection=False): req = f"DELETE FROM {table} WHERE FileID in ({fileIDString})" res = self.db._update(req, conn=connection) if not res["OK"]: - gLogger.error("Failed to remove files from table %s" % table, res["Message"]) + gLogger.error(f"Failed to remove files from table {table}", res["Message"]) failed.append(table) if failed: - return S_ERROR("Failed to remove files from %s" % stringListToString(failed)) + return S_ERROR(f"Failed to remove files from {stringListToString(failed)}") return S_OK() ###################################################### @@ -432,7 +430,7 @@ def _insertReplicas(self, lfns, master=False, connection=False): elif isinstance(seName, list): seList = seName else: - return S_ERROR("Illegal type of SE list: %s" % str(type(seName))) + return S_ERROR(f"Illegal type of SE list: {str(type(seName))}") for seName in seList: res = self.db.seManager.findSE(seName) if not res["OK"]: @@ -507,7 +505,7 @@ def _getRepIDsForReplica(self, replicaTuples, connection=False): queryTuples = [] for fileID, seID in replicaTuples: queryTuples.append("(%d,%d)" % (fileID, seID)) - req = "SELECT RepID,FileID,SEID FROM FC_Replicas WHERE (FileID,SEID) IN (%s)" % intListToString(queryTuples) + req = f"SELECT RepID,FileID,SEID FROM FC_Replicas WHERE (FileID,SEID) IN ({intListToString(queryTuples)})" res = self.db._query(req, conn=connection) if not res["OK"]: return res @@ -586,10 +584,10 @@ def __deleteReplicas(self, repIDs, connection=False): req = f"DELETE FROM {table} WHERE RepID in ({repIDString})" res = self.db._update(req, conn=connection) if not res["OK"]: - gLogger.error("Failed to remove replicas from table %s" % table, res["Message"]) + gLogger.error(f"Failed to remove replicas from table {table}", res["Message"]) failed.append(table) if failed: - return S_ERROR("Failed to remove replicas from %s" % stringListToString(failed)) + return S_ERROR(f"Failed to remove replicas from {stringListToString(failed)}") return S_OK() ###################################################### @@ -600,7 +598,7 @@ def __deleteReplicas(self, repIDs, connection=False): def _setReplicaStatus(self, fileID, se, status, connection=False): if status not in self.db.validReplicaStatus: - return S_ERROR("Invalid replica status %s" % status) + return S_ERROR(f"Invalid replica status {status}") connection = self._getConnection(connection) res = self._getStatusInt(status, connection=connection) if not res["OK"]: @@ -663,15 +661,15 @@ def _setFileParameter(self, fileID, paramName, paramValue, connection=False): tmpreq = "UPDATE FC_Files as FF1, ( %s ) as FF2 %%s WHERE FF1.FileID=FF2.FileID" % fileIDString else: tmpreq = "UPDATE FC_Files %%s WHERE FileID IN (%s)" % fileIDString - req = tmpreq % "SET %s='%s'" % (paramName, paramValue) + req = tmpreq % f"SET {paramName}='{paramValue}'" result = self.db._update(req, conn=connection) if not result["OK"]: return result if "select" in fileIDString.lower(): - req = "UPDATE FC_FileInfo as FF1, ( %s ) as FF2" % fileIDString + req = f"UPDATE FC_FileInfo as FF1, ( {fileIDString} ) as FF2" req += " SET ModificationDate=UTC_TIMESTAMP() WHERE FF1.FileID=FF2.FileID" else: - req = "UPDATE FC_FileInfo SET ModificationDate=UTC_TIMESTAMP() WHERE FileID IN (%s)" % fileIDString + req = f"UPDATE FC_FileInfo SET ModificationDate=UTC_TIMESTAMP() WHERE FileID IN ({fileIDString})" else: # Different statement for the fileIDString with SELECT is for performance optimization # since in this case the MySQL engine manages to use index on FileID. @@ -768,14 +766,14 @@ def __getFileIDReplicas(self, fileIDs, allStatus=False, connection=False): connection = self._getConnection(connection) if not fileIDs: return S_ERROR("No such file or directory") - req = "SELECT FileID,SEID,RepID,Status FROM FC_Replicas WHERE FileID IN (%s)" % (intListToString(fileIDs)) + req = f"SELECT FileID,SEID,RepID,Status FROM FC_Replicas WHERE FileID IN ({intListToString(fileIDs)})" if not allStatus: statusIDs = [] for status in self.db.visibleReplicaStatus: result = self._getStatusInt(status, connection=connection) if result["OK"]: statusIDs.append(result["Value"]) - req += " AND Status in (%s)" % (intListToString(statusIDs)) + req += f" AND Status in ({intListToString(statusIDs)})" res = self.db._query(req, conn=connection) if not res["OK"]: return res @@ -804,17 +802,17 @@ def _getDirectoryReplicas(self, dirID, allStatus=False, connection=False): req += " FC_Replicas as FR, FC_ReplicaInfo as FI" req += " WHERE FF.FileID=FR.FileID AND FR.RepID=FI.RepID AND FF.DirID=%d " % dirID if replicaStatusIDs: - req += " AND FR.Status in (%s)" % intListToString(replicaStatusIDs) + req += f" AND FR.Status in ({intListToString(replicaStatusIDs)})" if fileStatusIDs: - req += " AND FF.Status in (%s)" % intListToString(fileStatusIDs) + req += f" AND FF.Status in ({intListToString(fileStatusIDs)})" else: req = "SELECT FF.FileName,FR.FileID,FR.SEID,'' FROM FC_Files as FF," req += " FC_Replicas as FR" req += " WHERE FF.FileID=FR.FileID AND FF.DirID=%d " % dirID if replicaStatusIDs: - req += " AND FR.Status in (%s)" % intListToString(replicaStatusIDs) + req += f" AND FR.Status in ({intListToString(replicaStatusIDs)})" if fileStatusIDs: - req += " AND FF.Status in (%s)" % intListToString(fileStatusIDs) + req += f" AND FF.Status in ({intListToString(fileStatusIDs)})" result = self.db._query(req, conn=connection) return result @@ -842,7 +840,7 @@ def repairFileTables(self, connection=False): insertTuples.append("(%d,'%s',UTC_TIMESTAMP(),UTC_TIMESTAMP(),%d)" % (int(fileID), guid, self.db.umask)) fields = "FileID,GUID,CreationDate,ModificationDate,Mode" - req = "INSERT INTO FC_FileInfo ({}) VALUES {}".format(fields, ",".join(insertTuples)) + req = f"INSERT INTO FC_FileInfo ({fields}) VALUES {','.join(insertTuples)}" result = self.db._update(req) if not result["OK"]: return result diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerBase.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerBase.py index 58f918cd546..cde118e293a 100755 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerBase.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerBase.py @@ -53,7 +53,7 @@ def getFileCounters(self, connection=False): resultDict["Replicas w/o Files"] = res["Value"][0][0] treeTable = self.db.dtree.getTreeTable() - req = "SELECT COUNT(FileID) FROM FC_Files WHERE DirID NOT IN ( SELECT DirID FROM %s)" % treeTable + req = f"SELECT COUNT(FileID) FROM FC_Files WHERE DirID NOT IN ( SELECT DirID FROM {treeTable})" res = self.db._query(req, conn=connection) if not res["OK"]: return res @@ -382,7 +382,7 @@ def _updateDirectoryUsage(self, directorySEDict, change, connection=False): insertTuples.append("(%d,%d,%d,%d,UTC_TIMESTAMP())" % (dirID, seID, size, files)) req = "INSERT INTO FC_DirectoryUsage (DirID,SEID,SESize,SEFiles,LastUpdate) " - req += "VALUES %s" % ",".join(insertTuples) + req += f"VALUES {','.join(insertTuples)}" req += ( " ON DUPLICATE KEY UPDATE SESize=SESize%s%d, SEFiles=SEFiles%s%d, LastUpdate=UTC_TIMESTAMP() " % (change, size, change, files) @@ -541,7 +541,7 @@ def _getFileRelatives(self, lfns, depths, relation, connection=False): if id_ in relDict: result = self._getFileLFNs(list(relDict[id_])) if not result["OK"]: - failed[inputIDDict[id]] = "Failed to find %s" % relation + failed[inputIDDict[id]] = f"Failed to find {relation}" else: if result["Value"]["Successful"]: resDict = {} @@ -620,7 +620,7 @@ def _checkUniqueGUID(self, lfns, connection=False): return dict.fromkeys(lfns, res["Message"]) for guid, fileID in res["Value"].items(): # resolve this to LFN - failed[guidLFNs[guid]] = "GUID already registered for another file %s" % fileID + failed[guidLFNs[guid]] = f"GUID already registered for another file {fileID}" return failed def removeFile(self, lfns, connection=False): @@ -698,7 +698,7 @@ def setFileStatus(self, lfns, connection=False): status = lfns[lfn] if isinstance(status, str): if status not in self.db.validFileStatus: - failed[lfn] = "Invalid file status %s" % status + failed[lfn] = f"Invalid file status {status}" continue result = self._getStatusInt(status, connection=connection) if not result["OK"]: @@ -738,7 +738,6 @@ def addReplica(self, lfns, connection=False): return S_OK({"Successful": successful, "Failed": failed}) def _addReplicas(self, lfns, connection=False): - connection = self._getConnection(connection) successful = {} res = self._findFiles(list(lfns), ["DirID", "FileID", "Size"], connection=connection) @@ -1057,13 +1056,13 @@ def getReplicaStatus(self, lfns, connection=False): def _getStatusInt(self, status, connection=False): connection = self._getConnection(connection) - req = "SELECT StatusID FROM FC_Statuses WHERE Status = '%s';" % status + req = f"SELECT StatusID FROM FC_Statuses WHERE Status = '{status}';" res = self.db._query(req, conn=connection) if not res["OK"]: return res if res["Value"]: return S_OK(res["Value"][0][0]) - req = "INSERT INTO FC_Statuses (Status) VALUES ('%s');" % status + req = f"INSERT INTO FC_Statuses (Status) VALUES ('{status}');" res = self.db._update(req, conn=connection) if not res["OK"]: return res @@ -1193,7 +1192,7 @@ def _checkInfo(self, info, requiredKeys): return S_ERROR("Missing parameters") for key in requiredKeys: if key not in info: - return S_ERROR("Missing '%s' parameter" % key) + return S_ERROR(f"Missing '{key}' parameter") return S_OK() # def _checkLFNPFNConvention( self, lfn, pfn, se ): diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerFlat.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerFlat.py index 61d4d3e5549..6c9096a2819 100755 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerFlat.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerFlat.py @@ -6,7 +6,6 @@ class FileManagerFlat(FileManagerBase): - ###################################################### # # The all important _findFiles and _getDirectoryFiles methods @@ -94,7 +93,7 @@ def _insertFiles(self, lfns, uid, gid, connection=False): % (dirID, size, uid, gid, statusID, fileName, guid, checksum, checksumtype, self.db.umask) ) fields = "DirID,Size,UID,GID,Status,FileName,GUID,Checksum,ChecksumType,CreationDate,ModificationDate,Mode" - req = "INSERT INTO FC_Files ({}) VALUES {}".format(fields, ",".join(insertTuples)) + req = f"INSERT INTO FC_Files ({fields}) VALUES {','.join(insertTuples)}" res = self.db._update(req, conn=connection) if not res["OK"]: return res @@ -116,7 +115,7 @@ def _getFileIDFromGUID(self, guid, connection=False): return S_OK({}) if not isinstance(guid, (list, tuple)): guid = [guid] - req = "SELECT FileID,GUID FROM FC_Files WHERE GUID IN (%s)" % stringListToString(guid) + req = f"SELECT FileID,GUID FROM FC_Files WHERE GUID IN ({stringListToString(guid)})" res = self.db._query(req, conn=connection) if not res["OK"]: return res @@ -144,14 +143,14 @@ def __deleteFileReplicas(self, fileIDs, connection=False): connection = self._getConnection(connection) if not fileIDs: return S_OK() - req = "DELETE FROM FC_Replicas WHERE FileID in (%s)" % (intListToString(fileIDs)) + req = f"DELETE FROM FC_Replicas WHERE FileID in ({intListToString(fileIDs)})" return self.db._update(req, conn=connection) def __deleteFiles(self, fileIDs, connection=False): connection = self._getConnection(connection) if not fileIDs: return S_OK() - req = "DELETE FROM FC_Files WHERE FileID in (%s)" % (intListToString(fileIDs)) + req = f"DELETE FROM FC_Files WHERE FileID in ({intListToString(fileIDs)})" return self.db._update(req, conn=connection) ###################################################### @@ -207,7 +206,7 @@ def _insertReplicas(self, lfns, master=False, connection=False): deleteTuples.append((fileID, seID)) if insertTuples: fields = "FileID,SEID,Status,RepType,CreationDate,ModificationDate,PFN" - req = "INSERT INTO FC_Replicas ({}) VALUES {}".format(fields, ",".join(insertTuples.values())) + req = f"INSERT INTO FC_Replicas ({fields}) VALUES {','.join(insertTuples.values())}" res = self.db._update(req, conn=connection) if not res["OK"]: self.__deleteReplicas(deleteTuples, connection=connection) @@ -287,7 +286,7 @@ def __deleteReplicas(self, replicaTuples, connection=False): return res seID = res["Value"] deleteTuples.append("(%d,%d)" % (fileID, seID)) - req = "DELETE FROM FC_Replicas WHERE (FileID,SEID) IN (%s)" % intListToString(deleteTuples) + req = f"DELETE FROM FC_Replicas WHERE (FileID,SEID) IN ({intListToString(deleteTuples)})" return self.db._update(req, conn=connection) ###################################################### diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerPs.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerPs.py index 90bddd6c872..e7ed883f521 100755 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerPs.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileManager/FileManagerPs.py @@ -81,7 +81,6 @@ def _findFileIDs(self, lfns, connection=False): successful[lfn] = fileId else: - # We separate the files by directory filesInDirDict = self._getFileDirectories(lfns) @@ -171,7 +170,6 @@ def _getDirectoryFiles(self, dirID, fileNames, metadata_input, allStatus=False, files = {} for row in rows: - rowDict = dict(zip(fieldNames, row)) fileName = rowDict["FileName"] # Returns only the required metadata @@ -271,7 +269,6 @@ def _insertFiles(self, lfns, uid, gid, connection=False): # Prepare each file separately for lfn in lfns: - # Get all the info fileInfo = lfns[lfn] @@ -323,7 +320,6 @@ def _insertFiles(self, lfns, uid, gid, connection=False): if not result["OK"]: failed[lfn] = result["Message"] else: - fileID = result["Value"][0][0] successful[lfn] = lfns[lfn] @@ -506,7 +502,6 @@ def _insertReplicas(self, lfns, master=False, connection=False): # treat each file after each other for lfn in lfns.keys(): - fileID = lfns[lfn]["FileID"] seName = lfns[lfn]["SE"] @@ -515,7 +510,7 @@ def _insertReplicas(self, lfns, master=False, connection=False): elif isinstance(seName, list): seList = seName else: - return S_ERROR("Illegal type of SE list: %s" % str(type(seName))) + return S_ERROR(f"Illegal type of SE list: {str(type(seName))}") replicaType = "Master" if master else "Replica" pfn = lfns[lfn]["PFN"] @@ -523,7 +518,6 @@ def _insertReplicas(self, lfns, master=False, connection=False): # treat each replica of a file after the other # (THIS CANNOT WORK... WE ARE ONLY CAPABLE OF DOING ONE REPLICA PER FILE AT THE TIME) for seName in seList: - # get the SE id res = self.db.seManager.findSE(seName) if not res["OK"]: @@ -659,7 +653,7 @@ def _setReplicaStatus(self, fileID, se, status, connection=False): :returns: S_OK() or S_ERROR(msg) """ if status not in self.db.validReplicaStatus: - return S_ERROR("Invalid replica status %s" % status) + return S_ERROR(f"Invalid replica status {status}") connection = self._getConnection(connection) res = self._getStatusInt(status, connection=connection) if not res["OK"]: @@ -744,7 +738,6 @@ def _setFileParameter(self, fileID, paramName, paramValue, connection=False): # If there is an associated procedure, we go for it if psName: - result = self.db.executeStoredProcedureWithCursor(psName, (fileID, paramValue)) if not result["OK"]: return result diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileMetadata/FileMetadata.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileMetadata/FileMetadata.py index a34e5275a02..3738866aba5 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileMetadata/FileMetadata.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/FileMetadata/FileMetadata.py @@ -14,7 +14,6 @@ class FileMetadata: def __init__(self, database=None): - self.db = database def setDatabase(self, database): @@ -41,7 +40,7 @@ def addMetadataField(self, pName, pType, credDict): if not result["OK"]: return result if pName in result["Value"]: - return S_ERROR("The metadata %s is already defined for Directories" % pName) + return S_ERROR(f"The metadata {pName} is already defined for Directories") result = self._getFileMetadataFields(credDict) if not result["OK"]: return result @@ -84,12 +83,12 @@ def deleteMetadataField(self, pName, credDict): :return: S_OK/S_ERROR """ - req = "DROP TABLE FC_FileMeta_%s" % pName + req = f"DROP TABLE FC_FileMeta_{pName}" result = self.db._update(req) error = "" if not result["OK"]: error = result["Message"] - req = "DELETE FROM FC_FileMetaFields WHERE MetaName='%s'" % pName + req = f"DELETE FROM FC_FileMetaFields WHERE MetaName='{pName}'" result = self.db._update(req) if not result["OK"]: if error: @@ -149,17 +148,17 @@ def setMetadata(self, path, metaDict, credDict): if result["Value"]["Successful"]: fileID = result["Value"]["Successful"][path]["FileID"] else: - return S_ERROR("File %s not found" % path) + return S_ERROR(f"File {path} not found") voName = Registry.getGroupOption(credDict["group"], "VO") forceIndex = Operations(vo=voName).getValue("DataManagement/ForceIndexedMetadata", False) for metaName, metaValue in metaDict.items(): if metaName not in metaFields: if forceIndex: - return S_ERROR("Field %s not indexed, but ForceIndexedMetadata is set" % metaName, callStack=[]) + return S_ERROR(f"Field {metaName} not indexed, but ForceIndexedMetadata is set", callStack=[]) result = self.__setFileMetaParameter(fileID, metaName, metaValue, credDict) else: - result = self.db.insertFields("FC_FileMeta_%s" % metaName, ["FileID", "Value"], [fileID, metaValue]) + result = self.db.insertFields(f"FC_FileMeta_{metaName}", ["FileID", "Value"], [fileID, metaValue]) if not result["OK"]: if result["Message"].find("Duplicate") != -1: req = "UPDATE FC_FileMeta_%s SET Value='%s' WHERE FileID=%d" % (metaName, metaValue, fileID) @@ -191,7 +190,7 @@ def removeMetadata(self, path, metadata, credDict): if result["Value"]["Successful"]: fileID = result["Value"]["Successful"][path]["FileID"] else: - return S_ERROR("File %s not found" % path) + return S_ERROR(f"File {path} not found") failedMeta = {} for meta in metadata: @@ -210,7 +209,7 @@ def removeMetadata(self, path, metadata, credDict): if failedMeta: metaExample = list(failedMeta)[0] - result = S_ERROR("Failed to remove %d metadata, e.g. %s" % (len(failedMeta), failedMeta[metaExample])) + result = S_ERROR(f"Failed to remove {len(failedMeta)} metadata, e.g. {failedMeta[metaExample]}") result["FailedMetadata"] = failedMeta else: return S_OK() @@ -281,7 +280,7 @@ def _getFileUserMetadataByID(self, fileIDList, credDict, connection=False): return result metaFields = result["Value"] - stringIDs = ",".join(["%s" % fId for fId in fileIDList]) + stringIDs = ",".join([f"{fId}" for fId in fileIDList]) metaDict = {} for meta in metaFields: req = f"SELECT Value,FileID FROM FC_FileMeta_{meta} WHERE FileID in ({stringIDs})" @@ -292,7 +291,7 @@ def _getFileUserMetadataByID(self, fileIDList, credDict, connection=False): metaDict.setdefault(fileID, {}) metaDict[fileID][meta] = value - req = "SELECT FileID,MetaKey,MetaValue from FC_FileMeta where FileID in (%s)" % stringIDs + req = f"SELECT FileID,MetaKey,MetaValue from FC_FileMeta where FileID in ({stringIDs})" result = self.db._query(req, conn=connection) if not result["OK"]: return result @@ -359,11 +358,8 @@ def __getFileMetaParameters(self, fileID, credDict): return S_OK({}) metaDict = {} for fileID, key, value in result["Value"]: - if key in metaDict: - if isinstance(metaDict[key], list): - metaDict[key].append(value) - else: - metaDict[key] = [metaDict[key]].append(value) + if isinstance(metaDict.get(key), list): + metaDict[key].append(value) else: metaDict[key] = value @@ -394,7 +390,7 @@ def __transformMetaParameterToData(self, metaName): :return: S_OK/S_ERROR """ - req = "SELECT FileID,MetaValue from FC_FileMeta WHERE MetaKey='%s'" % metaName + req = f"SELECT FileID,MetaValue from FC_FileMeta WHERE MetaKey='{metaName}'" result = self.db._query(req) if not result["OK"]: return result @@ -405,12 +401,12 @@ def __transformMetaParameterToData(self, metaName): for fileID, meta in result["Value"]: insertValueList.append("( %d,'%s' )" % (fileID, meta)) - req = "INSERT INTO FC_FileMeta_{} (FileID,Value) VALUES {}".format(metaName, ", ".join(insertValueList)) + req = f"INSERT INTO FC_FileMeta_{metaName} (FileID,Value) VALUES {', '.join(insertValueList)}" result = self.db._update(req) if not result["OK"]: return result - req = "DELETE FROM FC_FileMeta WHERE MetaKey='%s'" % metaName + req = f"DELETE FROM FC_FileMeta WHERE MetaKey='{metaName}'" result = self.db._update(req) return result @@ -429,7 +425,7 @@ def __createMetaSelection(self, value): """ queryList = [] if isinstance(value, float): - queryList.append(("=", "%f" % value)) + queryList.append(("=", f"{value:f}")) elif isinstance(value, int): queryList.append(("=", "%d" % value)) elif isinstance(value, str): @@ -457,11 +453,10 @@ def __createMetaSelection(self, value): result = self.db._escapeValues(value) if not result["OK"]: return result - query = "( $s )" % ", ".join(result["Value"]) + query = f"( {', '.join(result['Value'])} )" queryList.append(("IN", query)) elif isinstance(value, dict): for operation, operand in value.items(): - # Prepare the escaped operand first if isinstance(operand, list): result = self.db._escapeValues(operand) @@ -471,7 +466,7 @@ def __createMetaSelection(self, value): elif isinstance(operand, int): escapedOperand = "%d" % operand elif isinstance(operand, float): - escapedOperand = "%f" % operand + escapedOperand = f"{operand:f}" else: result = self.db._escapeString(operand) if not result["OK"]: @@ -486,12 +481,12 @@ def __createMetaSelection(self, value): queryList.append((operation, escapedOperand)) elif operation == "in" or operation == "=": if isinstance(operand, list): - queryList.append(("IN", "( %s )" % escapedOperand)) + queryList.append(("IN", f"( {escapedOperand} )")) else: queryList.append(("=", escapedOperand)) elif operation == "nin" or operation == "!=": if isinstance(operand, list): - queryList.append(("NOT IN", "( %s )" % escapedOperand)) + queryList.append(("NOT IN", f"( {escapedOperand} )")) else: queryList.append(("!=", escapedOperand)) @@ -511,7 +506,7 @@ def __buildSEQuery(self, storageElements): for se in storageElements: seID = self.db.seNames.get(se, -1) if seID == -1: - return S_ERROR("Unknown SE %s" % se) + return S_ERROR(f"Unknown SE {se}") seIDList.append(seID) table = "FC_Replicas" seString = intListToString(seIDList) @@ -530,7 +525,7 @@ def __buildUserMetaQuery(self, userMetaDict): resultList = [] leftJoinTables = [] for meta, value in userMetaDict.items(): - table = "FC_FileMeta_%s" % meta + table = f"FC_FileMeta_{meta}" result = self.__createMetaSelection(value) if not result["OK"]: @@ -561,11 +556,11 @@ def __buildStandardMetaQuery(self, standardMetaDict): if infield == "User": value = self.db.users.get(invalue, -1) if value == "-1": - return S_ERROR("Unknown user %s" % invalue) + return S_ERROR(f"Unknown user {invalue}") elif infield == "Group": value = self.db.groups.get(invalue, -1) if value == "-1": - return S_ERROR("Unknown group %s" % invalue) + return S_ERROR(f"Unknown group {invalue}") table = "FC_Files" tableIndex = "F" @@ -585,7 +580,7 @@ def __buildStandardMetaQuery(self, standardMetaDict): for operation, operand in result["Value"]: queriesFileInfo.append(f"{tableIndex}.{field} {operation} {operand}") else: - return S_ERROR("Illegal standard meta key %s" % infield) + return S_ERROR(f"Illegal standard meta key {infield}") resultList = [] if queriesFiles: @@ -650,7 +645,7 @@ def __findFilesByMetadata(self, metaDict, dirList, credDict): if dirList: dirString = intListToString(dirList) - conditions.append("F.DirID in (%s)" % dirString) + conditions.append(f"F.DirID in ({dirString})") counter = 0 for table, condition in tablesAndConditions: @@ -671,7 +666,7 @@ def __findFilesByMetadata(self, metaDict, dirList, credDict): query += " ".join(tables) if conditions: - query += " WHERE %s" % " AND ".join(conditions) + query += f" WHERE {' AND '.join(conditions)}" result = self.db._query(query) if not result["OK"]: diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/SEManager/SEManagerDB.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/SEManager/SEManagerDB.py index 50fafb01562..d497fc87fa6 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/SEManager/SEManagerDB.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/SEManager/SEManagerDB.py @@ -9,7 +9,6 @@ class SEManagerDB(SEManagerBase): def _refreshSEs(self, connection=False): - req = "SELECT SEID,SEName FROM FC_StorageElements;" res = self.db._query(req) if not res["OK"]: @@ -24,7 +23,7 @@ def _refreshSEs(self, connection=False): startTime = time.time() self.lock.acquire() waitTime = time.time() - gLogger.debug("SEManager RefreshSEs lock created. Waited %.3f seconds." % (waitTime - startTime)) + gLogger.debug(f"SEManager RefreshSEs lock created. Waited {waitTime - startTime:.3f} seconds.") # Check once more the lastUpdate because it could have been updated while we were waiting for the lock if (time.time() - self.lastUpdate) < self.seUpdatePeriod: @@ -39,7 +38,7 @@ def _refreshSEs(self, connection=False): for seid, seName in res["Value"]: self.db.seNames[seName] = seid self.db.seids[seid] = seName - gLogger.debug("SEManager RefreshSEs lock released. Used %.3f seconds." % (time.time() - waitTime)) + gLogger.debug(f"SEManager RefreshSEs lock released. Used {time.time() - waitTime:.3f} seconds.") # Update the lastUpdate time self.lastUpdate = time.time() @@ -92,7 +91,7 @@ def __removeSE(self, seName, connection=False): waitTime = time.time() gLogger.debug(f"SEManager RemoveSE lock created. Waited {waitTime - startTime:.3f} seconds. {seName}") seid = self.db.seNames.get(seName, "Missing") - req = "DELETE FROM FC_StorageElements WHERE SEName='%s'" % seName + req = f"DELETE FROM FC_StorageElements WHERE SEName='{seName}'" res = self.db._update(req, conn=connection) if not res["OK"]: gLogger.debug(f"SEManager RemoveSE lock released. Used {time.time() - waitTime:.3f} seconds. {seName}") @@ -129,14 +128,14 @@ def getSEName(self, seID): """ if seID in self.db.seids: return S_OK(self.db.seids[seID]) - gLogger.info("getSEName: seID not found, refreshing", "ID: %s" % seID) + gLogger.info("getSEName: seID not found, refreshing", f"ID: {seID}") result = self._refreshSEs(connection=False) if not result["OK"]: gLogger.error("getSEName: refreshing failed", result["Message"]) return result if seID in self.db.seids: return S_OK(self.db.seids[seID]) - gLogger.error("getSEName: seID not found after refreshing", "ID: %s" % seID) + gLogger.error("getSEName: seID not found after refreshing", f"ID: {seID}") return S_ERROR("SE id %d not found" % seID) def deleteSE(self, seName, force=True): diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/DirectorySecurityManagerWithDelete.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/DirectorySecurityManagerWithDelete.py index f0b8d11f946..2becaf53e27 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/DirectorySecurityManagerWithDelete.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/DirectorySecurityManagerWithDelete.py @@ -76,7 +76,6 @@ def getPathPermissions(self, paths, credDict): # For all the paths that exist, check the write permission if paths: - res = super().getPathPermissions(paths, credDict) if not res["OK"]: return res diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/SecurityManagerBase.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/SecurityManagerBase.py index 9565a8a359c..f5098b7e61d 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/SecurityManagerBase.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/SecurityManagerBase.py @@ -17,6 +17,8 @@ "getDirectoryReplicas", "getDirectorySize", "getDirectoryMetadata", + "getSEDump", + "getDirectoryDump", ] _writeMethods = [ @@ -69,7 +71,7 @@ def hasAccess(self, opType, paths, credDict): successful = {} failed = {} - if not opType.lower() in ["read", "write", "execute"]: + if opType.lower() not in ["read", "write", "execute"]: return S_ERROR("Operation type not known") if self.db.globalReadAccess and (opType.lower() == "read"): for path in paths: diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/VOMSSecurityManager.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/VOMSSecurityManager.py index 2ba0259b4f4..71e898290f3 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/VOMSSecurityManager.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/VOMSSecurityManager.py @@ -79,11 +79,7 @@ def __shareVomsRole(self, grpName, otherGrpName): def __isNotExistError(self, errorMsg): """Returns true if the errorMsg means that the file/directory does not exist""" - for possibleMsg in ["not exist", "not found", "No such file or directory"]: - if possibleMsg in errorMsg: - return True - - return False + return any(possibleMsg in errorMsg for possibleMsg in ["not exist", "not found", "No such file or directory"]) def __getFilePermission(self, path, credDict, noExistStrategy=None): """Checks POSIX permission for a file using the VOMS roles. @@ -529,7 +525,7 @@ def hasAccess(self, opType, paths, credDict): return S_OK({"Successful": dict.fromkeys(paths, True), "Failed": {}}) if opType not in _readMethods + _writeMethods: - return S_ERROR("Operation type not known %s" % opType) + return S_ERROR(f"Operation type not known {opType}") if self.db.globalReadAccess and (opType in _readMethods): return S_OK({"Successful": dict.fromkeys(paths, True), "Failed": {}}) @@ -569,6 +565,7 @@ def hasAccess(self, opType, paths, credDict): "exists", "getFileAncestors", "getFileDescendents", + "getDirectoryDump", ]: policyToExecute = self.__policyReadForFileAndDirectory @@ -583,7 +580,7 @@ def hasAccess(self, opType, paths, credDict): policyToExecute = self.__policyChangePathMode if not policyToExecute: - return S_ERROR("No policy matching operation %s" % opType) + return S_ERROR(f"No policy matching operation {opType}") res = policyToExecute(paths, credDict) diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/test/Test_VOMSSecurityManager.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/test/Test_VOMSSecurityManager.py index d09c097d031..3bb33002504 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/test/Test_VOMSSecurityManager.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/SecurityManager/test/Test_VOMSSecurityManager.py @@ -182,7 +182,6 @@ def getPathPermissions(self, lfns, credDict): successful = {} failed = {} for filename in lfns: - if filename not in fileTree: failed[filename] = "File not found" continue @@ -265,7 +264,6 @@ class BaseCaseMixin: side_effect=mock_getAllGroups, ) def setUp(self, _a, _b): - global directoryTree global fileTree # A dictionary of directories. The keys are path, @@ -311,7 +309,6 @@ def compareResult(self): ("Existing", self.existingRet, self.expectedExistingRet), ("NonExisting", self.nonExistingRet, self.expectedNonExistingRet), ]: - self.assertTrue(real, "The method was not run") self.assertTrue(expected, "No expected results given") @@ -1595,7 +1592,6 @@ def test_addReplica(self): if __name__ == "__main__": - suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestNonExistingUser) suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(TestAdminGrpAnonUser)) suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(TestAdminGrpAdminUser)) diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/UserGroupManager/UserAndGroupManagerDB.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/UserGroupManager/UserAndGroupManagerDB.py index 6736d301640..457630d3fbd 100644 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/UserGroupManager/UserAndGroupManagerDB.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogComponents/UserGroupManager/UserAndGroupManagerDB.py @@ -99,7 +99,7 @@ def __removeUser(self, uname): waitTime = time.time() gLogger.debug(f"UserGroupManager RemoveUser lock created. Waited {waitTime - startTime:.3f} seconds. {uname}") uid = self.db.users.get(uname, "Missing") - req = "DELETE FROM FC_Users WHERE UserName='%s'" % uname + req = f"DELETE FROM FC_Users WHERE UserName='{uname}'" res = self.db._update(req) if not res["OK"]: gLogger.debug( @@ -119,11 +119,11 @@ def _refreshUsers(self): startTime = time.time() self.lock.acquire() waitTime = time.time() - gLogger.debug("UserGroupManager RefreshUsers lock created. Waited %.3f seconds." % (waitTime - startTime)) + gLogger.debug(f"UserGroupManager RefreshUsers lock created. Waited {waitTime - startTime:.3f} seconds.") req = "SELECT UID,UserName from FC_Users" res = self.db._query(req) if not res["OK"]: - gLogger.debug("UserGroupManager RefreshUsers lock released. Used %.3f seconds." % (time.time() - waitTime)) + gLogger.debug(f"UserGroupManager RefreshUsers lock released. Used {time.time() - waitTime:.3f} seconds.") self.lock.release() return res self.db.users = {} @@ -131,7 +131,7 @@ def _refreshUsers(self): for uid, uname in res["Value"]: self.db.users[uname] = uid self.db.uids[uid] = uname - gLogger.debug("UserGroupManager RefreshUsers lock released. Used %.3f seconds." % (time.time() - waitTime)) + gLogger.debug(f"UserGroupManager RefreshUsers lock released. Used {time.time() - waitTime:.3f} seconds.") self.lock.release() return S_OK() @@ -211,7 +211,7 @@ def __removeGroup(self, group): waitTime = time.time() gLogger.debug(f"UserGroupManager RemoveGroup lock created. Waited {waitTime - startTime:.3f} seconds. {group}") gid = self.db.groups.get(group, "Missing") - req = "DELETE FROM FC_Groups WHERE GroupName='%s'" % group + req = f"DELETE FROM FC_Groups WHERE GroupName='{group}'" res = self.db._update(req) if not res["OK"]: gLogger.debug( @@ -232,10 +232,10 @@ def _refreshGroups(self): startTime = time.time() self.lock.acquire() waitTime = time.time() - gLogger.debug("UserGroupManager RefreshGroups lock created. Waited %.3f seconds." % (waitTime - startTime)) + gLogger.debug(f"UserGroupManager RefreshGroups lock created. Waited {waitTime - startTime:.3f} seconds.") res = self.db._query(req) if not res["OK"]: - gLogger.debug("UserGroupManager RefreshGroups lock released. Used %.3f seconds." % (time.time() - waitTime)) + gLogger.debug(f"UserGroupManager RefreshGroups lock released. Used {time.time() - waitTime:.3f} seconds.") self.lock.release() return res self.db.groups = {} @@ -243,6 +243,6 @@ def _refreshGroups(self): for gid, gname in res["Value"]: self.db.groups[gname] = gid self.db.gids[gid] = gname - gLogger.debug("UserGroupManager RefreshGroups lock released. Used %.3f seconds." % (time.time() - waitTime)) + gLogger.debug(f"UserGroupManager RefreshGroups lock released. Used {time.time() - waitTime:.3f} seconds.") self.lock.release() return S_OK() diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogDB.py b/src/DIRAC/DataManagementSystem/DB/FileCatalogDB.py index 4fc41918f24..0ee0c01a14c 100755 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogDB.py +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogDB.py @@ -62,7 +62,6 @@ def setConfig(self, databaseConfig): ("dmeta", "DirectoryMetadata"), ("fmeta", "FileMetadata"), ]: - result = self.__loadCatalogComponent(componentType, databaseConfig[componentType]) if not result["OK"]: return result @@ -75,7 +74,7 @@ def __loadCatalogComponent(self, componentType, componentName): componentModule = f"DataManagementSystem.DB.FileCatalogComponents.{componentType}.{componentName}" result = ObjectLoader().loadObject(componentModule) if not result["OK"]: - gLogger.error("Failed to load catalog component", "{}: {}".format(componentName, result["Message"])) + gLogger.error("Failed to load catalog component", f"{componentName}: {result['Message']}") return result componentClass = result["Value"] component = componentClass(self) @@ -770,6 +769,23 @@ def getFileDescendents(self, lfns, depths, credDict): successful = res["Value"]["Successful"] return S_OK({"Successful": successful, "Failed": failed}) + def getFileDetailsPublic(self, lfns, credDict): + """Return all the metadata, including user defined, for those lfns that exist. + + :return: S_OK with a dictionary of LFNs to detailed information + """ + + res = self._checkPathPermissions("getFileDetailsPublic", lfns, credDict) + if not res["OK"]: + return res + failed = res["Value"]["Failed"] + + # if no successful, just return empty dict + if not res["Value"]["Successful"]: + return S_OK({}) + + return self.getFileDetails(res["Value"]["Successful"], credDict) + def getFileDetails(self, lfnList, credDict): """Get all the metadata for the given files""" connection = False @@ -913,6 +929,33 @@ def listDirectory(self, lfns, credDict, verbose=False): successful = res["Value"]["Successful"] return S_OK({"Successful": successful, "Failed": failed}) + def getDirectoryDump(self, lfns, credDict): + """ + Get a dump of the directories + + :param list lfns: list of directories + :param creDict: credential + + :return: Successful/Failed dict. + The successful values are dictionaries indexed "Files", "Subdirs" + """ + + res = self._checkPathPermissions("getDirectoryDump", lfns, credDict) + if not res["OK"]: + return res + failed = res["Value"]["Failed"] + + # if no successful, just return + if not res["Value"]["Successful"]: + return S_OK({"Successful": {}, "Failed": failed}) + + res = self.dtree.getDirectoryDump(res["Value"]["Successful"]) + if not res["OK"]: + return res + failed.update(res["Value"]["Failed"]) + successful = res["Value"]["Successful"] + return S_OK({"Successful": successful, "Failed": failed}) + def isDirectory(self, lfns, credDict): """ Checks whether a list of LFNS are directories or not @@ -1149,7 +1192,6 @@ def _checkAdminPermission(self, credDict): return self.securityManager.hasAdminAccess(credDict) def _checkPathPermissions(self, operation, lfns, credDict): - res = checkArgumentFormat(lfns) if not res["OK"]: return res diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogDB.sql b/src/DIRAC/DataManagementSystem/DB/FileCatalogDB.sql index 54063a99969..e6a7a8fbae3 100755 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogDB.sql +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogDB.sql @@ -23,7 +23,7 @@ CREATE TABLE FC_Files( UID SMALLINT UNSIGNED NOT NULL, GID TINYINT UNSIGNED NOT NULL, Status SMALLINT UNSIGNED NOT NULL, - FileName VARCHAR(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + FileName VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, INDEX (DirID), INDEX (UID,GID), INDEX (Status), @@ -119,7 +119,7 @@ CREATE TABLE FC_Users ( CREATE TABLE FC_StorageElements ( SEID INTEGER AUTO_INCREMENT PRIMARY KEY, - SEName VARCHAR(127) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + SEName VARCHAR(127) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, AliasName VARCHAR(127) DEFAULT '', UNIQUE KEY (SEName) ) ENGINE = INNODB; @@ -159,7 +159,7 @@ CREATE TABLE FC_DirectoryInfo ( CREATE TABLE FC_DirMeta ( DirID INTEGER NOT NULL, - MetaKey VARCHAR(31) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL DEFAULT 'Noname', + MetaKey VARCHAR(31) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT 'Noname', MetaValue VARCHAR(31) NOT NULL DEFAULT 'Noname', PRIMARY KEY (DirID,MetaKey) ) ENGINE = INNODB; @@ -168,7 +168,7 @@ CREATE TABLE FC_DirMeta ( CREATE TABLE FC_FileMeta ( FileID INTEGER NOT NULL, - MetaKey VARCHAR(31) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL DEFAULT 'Noname', + MetaKey VARCHAR(31) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT 'Noname', MetaValue VARCHAR(31) NOT NULL DEFAULT 'Noname', PRIMARY KEY (FileID,MetaKey) ) ENGINE = INNODB; @@ -177,7 +177,7 @@ CREATE TABLE FC_FileMeta ( CREATE TABLE FC_DirectoryTree ( DirID INT AUTO_INCREMENT PRIMARY KEY, - DirName VARCHAR(1024) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + DirName VARCHAR(1024) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, Parent INT NOT NULL DEFAULT 0, INDEX (Parent), INDEX (DirName) @@ -187,7 +187,7 @@ CREATE TABLE FC_DirectoryTree ( CREATE TABLE FC_DirectoryTreeM ( DirID INT AUTO_INCREMENT PRIMARY KEY, - DirName VARCHAR(255) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + DirName VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, Parent INT NOT NULL DEFAULT 0, Level INT NOT NULL, INDEX (Level), @@ -199,7 +199,7 @@ CREATE TABLE FC_DirectoryTreeM ( CREATE TABLE FC_DirectoryLevelTree ( DirID INT AUTO_INCREMENT PRIMARY KEY, - DirName VARCHAR(255) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + DirName VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, Parent INT NOT NULL DEFAULT 0, Level INT NOT NULL, LPATH1 INT NOT NULL DEFAULT 0, @@ -240,7 +240,7 @@ CREATE TABLE FC_DirectoryUsage( CREATE TABLE FC_MetaFields ( MetaID INT AUTO_INCREMENT PRIMARY KEY, - MetaName VARCHAR(64) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + MetaName VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, MetaType VARCHAR(128) NOT NULL ) ENGINE = INNODB; @@ -248,7 +248,7 @@ CREATE TABLE FC_MetaFields ( CREATE TABLE FC_FileMetaFields ( MetaID INT AUTO_INCREMENT PRIMARY KEY, - MetaName VARCHAR(64) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + MetaName VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, MetaType VARCHAR(128) NOT NULL ) ENGINE = INNODB; diff --git a/src/DIRAC/DataManagementSystem/DB/FileCatalogWithFkAndPsDB.sql b/src/DIRAC/DataManagementSystem/DB/FileCatalogWithFkAndPsDB.sql index db9706967b3..8d65d1b2d6f 100755 --- a/src/DIRAC/DataManagementSystem/DB/FileCatalogWithFkAndPsDB.sql +++ b/src/DIRAC/DataManagementSystem/DB/FileCatalogWithFkAndPsDB.sql @@ -36,7 +36,7 @@ INSERT INTO FC_Statuses (StatusID, Status) values (1, 'FakeStatus'); CREATE TABLE FC_StorageElements ( SEID INTEGER AUTO_INCREMENT, - SEName VARCHAR(127) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + SEName VARCHAR(127) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, AliasName VARCHAR(127) DEFAULT '', PRIMARY KEY (SEID), @@ -79,7 +79,7 @@ INSERT INTO FC_Users (UID, UserName) values (1, 'root'); -- -- create table FC_DirectoryList ( -- DirID INT NOT NULL AUTO_INCREMENT, --- Name varchar(255)CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, +-- Name varchar(255)CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, -- -- PRIMARY KEY (DirID), -- @@ -116,7 +116,7 @@ create table FC_DirectoryList ( ModificationDate DATETIME, Mode SMALLINT UNSIGNED NOT NULL DEFAULT 775, Status INTEGER NOT NULL DEFAULT 0, - Name varchar(255)CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + Name varchar(255)CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, PRIMARY KEY (DirID), FOREIGN KEY (UID) REFERENCES FC_Users(UID), @@ -158,7 +158,7 @@ CREATE TABLE FC_Files( Mode SMALLINT UNSIGNED NOT NULL DEFAULT 775, ChecksumType ENUM('Adler32','MD5'), Checksum VARCHAR(32), - FileName VARCHAR(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + FileName VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, PRIMARY KEY (FileID), FOREIGN KEY (DirID) REFERENCES FC_DirectoryList(DirID) ON DELETE CASCADE, @@ -223,7 +223,7 @@ CREATE TABLE FC_DirectoryUsage( CREATE TABLE FC_DirMeta ( DirID INTEGER NOT NULL, - MetaKey VARCHAR(31) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL DEFAULT 'Noname', + MetaKey VARCHAR(31) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT 'Noname', MetaValue VARCHAR(31) NOT NULL DEFAULT 'Noname', PRIMARY KEY (DirID,MetaKey) ) ENGINE = INNODB; @@ -232,7 +232,7 @@ CREATE TABLE FC_DirMeta ( CREATE TABLE FC_FileMeta ( FileID INTEGER NOT NULL, - MetaKey VARCHAR(31) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL DEFAULT 'Noname', + MetaKey VARCHAR(31) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL DEFAULT 'Noname', MetaValue VARCHAR(31) NOT NULL DEFAULT 'Noname', PRIMARY KEY (FileID,MetaKey) ) ENGINE = INNODB; @@ -242,7 +242,7 @@ CREATE TABLE FC_FileMeta ( CREATE TABLE FC_MetaFields ( MetaID INT AUTO_INCREMENT PRIMARY KEY, - MetaName VARCHAR(64) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + MetaName VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, MetaType VARCHAR(128) NOT NULL ) ENGINE = INNODB; @@ -250,7 +250,7 @@ CREATE TABLE FC_MetaFields ( CREATE TABLE FC_FileMetaFields ( MetaID INT AUTO_INCREMENT PRIMARY KEY, - MetaName VARCHAR(64) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + MetaName VARCHAR(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, MetaType VARCHAR(128) NOT NULL ) ENGINE = INNODB; @@ -280,7 +280,7 @@ CREATE TABLE FC_FileAncestors ( CREATE TABLE FC_MetaDatasets ( DatasetID INT AUTO_INCREMENT, - DatasetName VARCHAR(128) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL, + DatasetName VARCHAR(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL, MetaQuery VARCHAR(512), DirID INT NOT NULL DEFAULT 0, TotalSize BIGINT UNSIGNED NOT NULL, @@ -483,7 +483,7 @@ DELIMITER // CREATE PROCEDURE ps_get_direct_children (IN dir_id INT ) BEGIN - SELECT SQL_NO_CACHE d.DirID from FC_DirectoryList d JOIN FC_DirectoryClosure c on (d.DirID = c.ChildID) where c.ParentID = dir_id and c.Depth = 1; + SELECT SQL_NO_CACHE ChildID FROM FC_DirectoryClosure WHERE ParentID = dir_id and Depth = 1; END // DELIMITER ; @@ -493,6 +493,9 @@ DELIMITER ; -- includeParent: if true, include oneself -- returns (directory id, absolute level) +-- CHRIS: CAN THIS BE REPLACED WITH SOMETHING LIKE +-- select ChildID, Depth + (select max(Depth) from FC_DirectoryClosure where ChildID = dir_id) from FC_DirectoryClosure where ParentID = dir_id; + DROP PROCEDURE IF EXISTS ps_get_sub_directories; DELIMITER // CREATE PROCEDURE ps_get_sub_directories @@ -900,30 +903,30 @@ BEGIN RESIGNAL; END; - START TRANSACTION; - - -- Store the name of the tmp table once for all - - SET @tmpTableName = CONCAT('tmpDirUsageDelRep_',CONNECTION_ID()); - - -- We create the table if it does not exist - SET @sql = CONCAT('CREATE TEMPORARY TABLE IF NOT EXISTS ',@tmpTableName ,' (DirID INT, SEID INT, t_size BIGINT UNSIGNED, t_file INT, INDEX(DirID))'); - - PREPARE stmt FROM @sql; - EXECUTE stmt; - DEALLOCATE PREPARE stmt; - - -- Insert into it the values we will have to substract later on - SET @sql = CONCAT('INSERT INTO ', @tmpTableName, '(DirID, SEID, t_size, t_file) SELECT d1.DirID, d1.SEID, SUM(f.Size) as t_size, count(*) as t_file - FROM FC_DirectoryUsage d1, FC_Files f, FC_Replicas r - WHERE r.FileID = f.FileID AND f.DirID = d1.DirID AND r.SEID = d1.SEID AND f.FileID IN (', file_ids, ') GROUP BY d1.DirID, d1.SEID'); - - PREPARE stmt FROM @sql; - EXECUTE stmt; - DEALLOCATE PREPARE stmt; - - -- perform the update - SET @sql = CONCAT('UPDATE FC_DirectoryUsage d, ',@tmpTableName,' t set d.`SESize` = d.`SESize` - t.t_size, d.`SEFiles` = d.`SEFiles` - t.t_file where d.DirID = t.DirID and d.`SEID`= t.SEID'); + START TRANSACTION; + + + SET @sql = CONCAT('UPDATE FC_DirectoryUsage d, + (SELECT d1.DirID, d1.SEID, SUM(f.Size) as t_size, count(*) as t_file + FROM FC_DirectoryUsage d1, FC_Files f, FC_Replicas r + WHERE r.FileID = f.FileID + AND f.DirID = d1.DirID + AND r.SEID = d1.SEID + AND f.FileID IN (', file_ids, ') + GROUP BY d1.DirID, d1.SEID ) t + SET d.SESize = d.SESize - t.t_size, + d.SEFiles = d.SEFiles - t.t_file + WHERE d.DirID = t.DirID + AND d.SEID = t.SEID'); + + -- This is buggy in case we remove two files that have a replica on the same SE + -- + -- SET @sql = CONCAT('UPDATE FC_DirectoryUsage d, FC_Files f, FC_Replicas r + -- SET d.SESize = d.SESize - f.Size, d.SEFiles = d.SEFiles - 1 + -- WHERE r.FileID = f.FileID + -- AND f.DirID = d.DirID + -- AND r.SEID = d.SEID + -- AND f.FileID IN (', file_ids, ')'); PREPARE stmt FROM @sql; EXECUTE stmt; @@ -931,16 +934,7 @@ BEGIN - -- delete the entries from the temporary table - SET @sql = CONCAT('DELETE t FROM ',@tmpTableName, ' t JOIN FC_Files f ON t.DirID = f.DirID where f.FileID IN (', file_ids, ')'); - - PREPARE stmt FROM @sql; - EXECUTE stmt; - DEALLOCATE PREPARE stmt; - - - -- delete the entry from the FC_Replicas table - SET @sql = CONCAT('DELETE FROM FC_Replicas WHERE FileID IN (', file_ids, ')'); + SET @sql = CONCAT('DELETE FROM FC_Replicas WHERE FileID IN (', file_ids, ')'); PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; @@ -963,60 +957,39 @@ CREATE PROCEDURE ps_delete_files (IN file_ids MEDIUMTEXT) BEGIN + DECLARE exit handler for sqlexception BEGIN ROLLBACK; RESIGNAL; END; - START TRANSACTION; - - -- Store the name of the tmp table once for all - - SET @tmpTableName = CONCAT('tmpDirUsageDelFile_',CONNECTION_ID()); - - -- We create the table if it does not exist - SET @sql = CONCAT('CREATE TEMPORARY TABLE IF NOT EXISTS ',@tmpTableName ,' (DirID INT, t_size BIGINT UNSIGNED, t_file INT, INDEX(DirID))'); - - PREPARE stmt FROM @sql; - EXECUTE stmt; - DEALLOCATE PREPARE stmt; - - - -- Insert into it the values we will have to substract later on - SET @sql = CONCAT('INSERT INTO ', @tmpTableName, '(DirID,t_size, t_file) SELECT d1.DirID, SUM(f.Size) as t_size, count(*) as t_file - FROM FC_DirectoryList d1, FC_Files f - WHERE f.DirID = d1.DirID AND f.FileID IN (', file_ids, ') GROUP BY d1.DirID'); - - PREPARE stmt FROM @sql; - EXECUTE stmt; - DEALLOCATE PREPARE stmt; - - -- perform the update - SET @sql = CONCAT('UPDATE FC_DirectoryUsage d, ',@tmpTableName,' t set d.`SESize` = d.`SESize` - t.t_size, d.`SEFiles` = d.`SEFiles` - t.t_file where d.DirID = t.DirID and d.`SEID`= 1'); - - PREPARE stmt FROM @sql; - EXECUTE stmt; - DEALLOCATE PREPARE stmt; - + START TRANSACTION; + SET @sql = CONCAT('UPDATE FC_DirectoryUsage d, + (SELECT d1.DirID, SUM(f.Size) as t_size, count(*) as t_file + FROM FC_DirectoryList d1, FC_Files f + where f.DirID = d1.DirID + AND f.FileID IN (', file_ids, ') + GROUP BY d1.DirID ) t + SET d.SESize = d.SESize - t.t_size, + d.SEFiles = d.SEFiles - t.t_file + WHERE d.DirID = t.DirID + AND d.SEID = 1' ); - -- delete the entries from the temporary table - SET @sql = CONCAT('DELETE t FROM ',@tmpTableName, ' t JOIN FC_Files f ON t.DirID = f.DirID where f.FileID IN (', file_ids, ')'); + PREPARE stmt FROM @sql; + EXECUTE stmt; + DEALLOCATE PREPARE stmt; - PREPARE stmt FROM @sql; - EXECUTE stmt; - DEALLOCATE PREPARE stmt; + SET @sql = CONCAT('DELETE FROM FC_Files WHERE FileID IN (', file_ids, ')'); + PREPARE stmt FROM @sql; + EXECUTE stmt; + DEALLOCATE PREPARE stmt; + COMMIT; - -- delete the entry from the File table - SET @sql = CONCAT('DELETE FROM FC_Files WHERE FileID IN (', file_ids, ')'); - PREPARE stmt FROM @sql; - EXECUTE stmt; - DEALLOCATE PREPARE stmt; + SELECT 0, 'OK'; - COMMIT; - SELECT 0, 'OK'; END // DELIMITER ; @@ -1966,6 +1939,45 @@ END // DELIMITER ; +-- ps_get_directory_dump : recursively dump all the lfns and subdir in a directory +-- dir_id : directory ID +-- output : +-- LFN, Size, CreationDate for files +-- LFN, NULL, CreationDate for directories + +DROP PROCEDURE IF EXISTS ps_get_directory_dump; +DELIMITER // +CREATE PROCEDURE ps_get_directory_dump +(IN dir_id INT) +BEGIN + + DECLARE exit handler for sqlexception + BEGIN + ROLLBACK; + RESIGNAL; + END; + + + (SELECT d.Name ,NULL, d.CreationDate + FROM FC_DirectoryList d + JOIN FC_DirectoryClosure c + ON d.DirID = c.ChildID + WHERE c.ParentID = dir_id + AND Depth != 0 + ) + UNION ALL + (SELECT CONCAT(d.Name, '/', f.FileName), Size, f.CreationDate + FROM FC_Files f + JOIN FC_DirectoryList d + ON f.DirID = d.DirID + JOIN FC_DirectoryClosure c + ON c.ChildID = f.DirID + WHERE ParentID = dir_id + ); +END // +DELIMITER ; + + -- Consistency checks diff --git a/src/DIRAC/DataManagementSystem/DB/test/FTS3TestUtils.py b/src/DIRAC/DataManagementSystem/DB/test/FTS3TestUtils.py new file mode 100644 index 00000000000..5f1d3879599 --- /dev/null +++ b/src/DIRAC/DataManagementSystem/DB/test/FTS3TestUtils.py @@ -0,0 +1,429 @@ +""" +We define here some tests that are meant to be ran as unit tests +against an sqlite DB and as integration tests against a MySQL DB +""" +import random + +from DIRAC.Core.Security.ProxyInfo import getProxyInfo +from DIRAC.DataManagementSystem.Client.FTS3Operation import FTS3Operation, FTS3TransferOperation, FTS3StagingOperation +from DIRAC.DataManagementSystem.Client.FTS3File import FTS3File +from DIRAC.DataManagementSystem.Client.FTS3Job import FTS3Job + + +def generateOperation(opType, nbFiles, dests, sources=None): + """Generate one FTS3Operation object with FTS3Files in it""" + op = None + if opType == "Transfer": + op = FTS3TransferOperation() + elif opType == "Staging": + op = FTS3StagingOperation() + # Get the username and group from the proxy + # if we are in integration test + try: + proxyInfo = getProxyInfo()["Value"] + op.username = proxyInfo["username"] + op.userGroup = proxyInfo["group"] + except: + op.username = "username" + op.userGroup = "group" + op.sourceSEs = str(sources) + for _i in range(nbFiles * len(dests)): + for dest in dests: + ftsFile = FTS3File() + ftsFile.lfn = f"lfn{random.randint(0,100)}" + ftsFile.targetSE = dest + op.ftsFiles.append(ftsFile) + + return op + + +def base_test_operation(fts3db, fts3Client): + """ + Run basic operation tests + + :param fts3db: an FTS3DB + :param fts3Client: possibly an FTS3Client (integration test) or the same fts3db + """ + op = generateOperation("Transfer", 3, ["Target1", "Target2"], sources=["Source1", "Source2"]) + assert not op.isTotallyProcessed() + + res = fts3Client.persistOperation(op) + assert res["OK"], res + opID = res["Value"] + + res = fts3Client.getOperation(opID) + assert res["OK"] + + op2 = res["Value"] + + assert isinstance(op2, FTS3TransferOperation) + assert not op2.isTotallyProcessed() + + for attr in ["username", "userGroup", "sourceSEs"]: + assert getattr(op, attr) == getattr(op2, attr) + + assert len(op.ftsFiles) == len(op2.ftsFiles) + + assert op2.status == FTS3Operation.INIT_STATE + + fileIds = [] + for ftsFile in op2.ftsFiles: + fileIds.append(ftsFile.fileID) + assert ftsFile.status == FTS3File.INIT_STATE + + # Testing updating the status and error + fileStatusDict = {} + for fId in fileIds: + fileStatusDict[fId] = { + "status": "Finished" if fId % 2 else "Failed", + "error": "" if fId % 2 else "Tough luck", + } + + res = fts3db.updateFileStatus(fileStatusDict) + assert res["OK"] + + res = fts3Client.getOperation(opID) + op3 = res["Value"] + assert res["OK"] + + assert op3.ftsFiles + for ftsFile in op3.ftsFiles: + if ftsFile.fileID % 2: + assert ftsFile.status == "Finished" + assert not ftsFile.error + else: + assert ftsFile.status == "Failed" + assert ftsFile.error == "Tough luck" + + assert not op3.isTotallyProcessed() + + # Testing updating only the status and to final states + fileStatusDict = {} + nbFinalStates = len(FTS3File.FINAL_STATES) + for fId in fileIds: + fileStatusDict[fId] = {"status": FTS3File.FINAL_STATES[fId % nbFinalStates]} + + res = fts3db.updateFileStatus(fileStatusDict) + assert res["OK"] + + res = fts3Client.getOperation(opID) + op4 = res["Value"] + assert res["OK"] + + assert op4.ftsFiles + for ftsFile in op4.ftsFiles: + if ftsFile.fileID % 2: + # Files to finished cannot be changed + assert ftsFile.status == "Finished" + assert not ftsFile.error + else: + assert ftsFile.status == FTS3File.FINAL_STATES[ftsFile.fileID % nbFinalStates] + assert ftsFile.error == "Tough luck" + + # Now it should be considered as totally processed + assert op4.isTotallyProcessed() + res = fts3Client.persistOperation(op4) + + +def base_test_job(fts3db, fts3Client): + """ + Run basic Job tests + + :param fts3db: an FTS3DB + :param fts3Client: possibly an FTS3Client (integration test) or the same fts3db + """ + op = generateOperation("Transfer", 3, ["Target1", "Target2"], sources=["Source1", "Source2"]) + + job1 = FTS3Job() + job1.ftsGUID = "a-random-guid" + job1.ftsServer = "fts3" + + job1.username = op.username + job1.userGroup = op.userGroup + + op.ftsJobs.append(job1) + + res = fts3Client.persistOperation(op) + assert res["OK"], res + opID = res["Value"] + + res = fts3Client.getOperation(opID) + assert res["OK"] + + op2 = res["Value"] + assert len(op2.ftsJobs) == 1 + job2 = op2.ftsJobs[0] + assert job2.operationID == opID + + for attr in ["ftsGUID", "ftsServer", "username", "userGroup"]: + assert getattr(job1, attr) == getattr(job2, attr) + + +def base_test_job_monitoring_racecondition(fts3db, fts3Client): + """We used to have a race condition resulting in duplicated transfers for a file. + This test reproduces the race condition. + + The scenario is as follow. Operation has two files File1 and File2. + Job1 is submitted for File1 and File2. + File1 fails, File2 is still ongoing. + We submit Job2 for File1. + Job1 is monitored again, and we update again File1 to failed (because it is so in Job1) + A Job3 would be created for File1, despite Job2 still running on it. + """ + op = generateOperation("Transfer", 2, ["Target1"]) + + job1 = FTS3Job() + job1.ftsGUID = "03-racecondition-job1" + job1.ftsServer = "fts3" + + job1.username = op.username + job1.userGroup = op.userGroup + + op.ftsJobs.append(job1) + + res = fts3Client.persistOperation(op) + opID = res["Value"] + + # Get back the operation to update all the IDs + res = fts3Client.getOperation(opID) + op = res["Value"] + + fileIds = [] + for ftsFile in op.ftsFiles: + fileIds.append(ftsFile.fileID) + + file1ID = min(fileIds) + file2ID = max(fileIds) + + # Now we monitor Job1, and find that the first file has failed, the second is still ongoing + fileStatusDict = { + file1ID: {"status": "Failed", "error": "Someone made a boo-boo"}, + file2ID: {"status": "Staging"}, + } + + res = fts3db.updateFileStatus(fileStatusDict) + assert res["OK"] + + # We would then submit a second job + job2 = FTS3Job() + job2.ftsGUID = "03-racecondition-job2" + job2.ftsServer = "fts3" + + job2.username = op.username + job2.userGroup = op.userGroup + + op.ftsJobs.append(job2) + res = fts3Client.persistOperation(op) + + # Now we monitor Job2 & Job1 (in this order) + fileStatusDictJob2 = { + file1ID: {"status": "Staging"}, + } + res = fts3db.updateFileStatus(fileStatusDictJob2) + assert res["OK"] + + # And in Job1, File1 is (and will remain) failed, while File2 is still ongoing + fileStatusDictJob1 = { + file1ID: {"status": "Failed", "error": "Someone made a boo-boo"}, + file2ID: {"status": "Staging"}, + } + res = fts3db.updateFileStatus(fileStatusDictJob1) + assert res["OK"] + + # And now this is the problem, because If we check whether this operation still has + # files to submit, it will tell me yes, while all the files are being taken care of + res = fts3Client.getOperation(opID) + op = res["Value"] + + # isTotallyProcessed does not return S_OK struct + filesToSubmit = op._getFilesToSubmit() + assert filesToSubmit == [op.ftsFiles[0]] + + +def base_test_job_monitoring_solve_racecondition(fts3db, fts3Client): + """We used to have a race condition resulting in duplicated transfers for a file. + This test reproduces the race condition to make sure it is fixed. + This test makes sure that the update only happens on files concerned by the job + + The scenario is as follow. Operation has two files File1 and File2. + Job1 is submitted for File1 and File2. + File1 fails, File2 is still ongoing. + We submit Job2 for File1. + Job1 is monitored again, and we update again File1 to failed (because it is so in Job1) + A Job3 would be created for File1, dispite Job2 still runing on it. + """ + op = generateOperation("Transfer", 2, ["Target1"]) + + job1 = FTS3Job() + job1GUID = "04-racecondition-job1" + job1.ftsGUID = job1GUID + job1.ftsServer = "fts3" + + job1.username = op.username + job1.userGroup = op.userGroup + + op.ftsJobs.append(job1) + + # Now, when submitting the job, we specify the ftsGUID to which files are + # assigned + for ftsFile in op.ftsFiles: + ftsFile.ftsGUID = job1GUID + + res = fts3Client.persistOperation(op) + opID = res["Value"] + + # Get back the operation to update all the IDs + res = fts3Client.getOperation(opID) + op = res["Value"] + + fileIds = [] + for ftsFile in op.ftsFiles: + fileIds.append(ftsFile.fileID) + + # Arbitrarilly decide that File1 has the smalled fileID + file1ID = min(fileIds) + file2ID = max(fileIds) + + # Now we monitor Job1, and find that the first file has failed, the second is still ongoing + # And since File1 is in an FTS final status, we set its ftsGUID to None + fileStatusDict = { + file1ID: {"status": "Failed", "error": "Someone made a boo-boo", "ftsGUID": None}, + file2ID: {"status": "Staging"}, + } + + # And when updating, take care of specifying that you are updating for a given GUID + res = fts3db.updateFileStatus(fileStatusDict, ftsGUID=job1GUID) + assert res["OK"] + + # We would then submit a second job + job2 = FTS3Job() + job2GUID = "04-racecondition-job2" + job2.ftsGUID = job2GUID + job2.ftsServer = "fts3" + + job2.username = op.username + job2.userGroup = op.userGroup + + op.ftsJobs.append(job2) + + # And do not forget to add the new FTSGUID to File1 + # assigned + for ftsFile in op.ftsFiles: + if ftsFile.fileID == file1ID: + ftsFile.ftsGUID = job2GUID + + res = fts3Client.persistOperation(op) + + # Now we monitor Job2 & Job1 (in this order) + fileStatusDictJob2 = { + file1ID: {"status": "Staging"}, + } + + # Again specify the GUID + res = fts3db.updateFileStatus(fileStatusDictJob2, ftsGUID=job2GUID) + assert res["OK"] + + # And in Job1, File1 is (and will remain) failed, while File2 is still ongoing + fileStatusDictJob1 = { + file1ID: {"status": "Failed", "error": "Someone made a boo-boo"}, + file2ID: {"status": "Staging"}, + } + + # And thanks to specifying the job GUID, File1 should not be touched ! + res = fts3db.updateFileStatus(fileStatusDictJob1, ftsGUID=job1GUID) + assert res["OK"] + + # And hopefully now there shouldn't be any file to submit + res = fts3Client.getOperation(opID) + op = res["Value"] + + # isTotallyProcessed does not return S_OK struct + filesToSubmit = op._getFilesToSubmit() + assert not filesToSubmit + + +def base_test_delete_operations(fts3db, fts3Client): + """Test operation removals""" + op1 = generateOperation("Transfer", 2, ["Target1"]) + + res = fts3Client.persistOperation(op1) + opID1 = res["Value"] + + # Create two other operations, to test the limit feature + op2 = generateOperation("Transfer", 2, ["Target2"]) + res = fts3Client.persistOperation(op2) + opID2 = res["Value"] + + op3 = generateOperation("Transfer", 2, ["Target3"]) + res = fts3Client.persistOperation(op3) + opID3 = res["Value"] + + # Now, call delete, and make sure that operation is not delete + # Ops is not in a final state, and delay is not passed + res = fts3db.deleteFinalOperations() + assert res["OK"] + + res = fts3Client.getOperation(opID1) + assert res["OK"] + op1 = res["Value"] + + # Try again with no delay, but still not final state + res = fts3db.deleteFinalOperations(deleteDelay=0) + assert res["OK"] + + res = fts3Client.getOperation(opID1) + assert res["OK"] + op1 = res["Value"] + + # Set the final status + op1.status = "Finished" + res = fts3Client.persistOperation(op1) + assert res["OK"] + + # Now try to delete again. + # It should still not work because of the delay + res = fts3db.deleteFinalOperations() + assert res["OK"] + + res = fts3Client.getOperation(opID1) + assert res["OK"] + op1 = res["Value"] + + # Finally, it should work, with no delay and a final status + res = fts3db.deleteFinalOperations(deleteDelay=0) + assert res["OK"] + + res = fts3Client.getOperation(opID1) + assert not res["OK"] + + # op2 and op3 should still be here though ! + + res = fts3Client.getOperation(opID2) + assert res["OK"] + op2 = res["Value"] + res = fts3Client.getOperation(opID3) + assert res["OK"] + op3 = res["Value"] + + # Set them both to a final status + op2.status = "Finished" + res = fts3Client.persistOperation(op2) + assert res["OK"] + op3.status = "Finished" + res = fts3Client.persistOperation(op3) + assert res["OK"] + + # Now try to delete, but only one + res = fts3db.deleteFinalOperations(limit=1, deleteDelay=0) + assert res["OK"] + + # Now only op2 or op3 should be here + + res2 = fts3Client.getOperation(opID2) + res3 = fts3Client.getOperation(opID3) + + assert res2["OK"] ^ res3["OK"] + + +# All base tests defined in this module +allBaseTests = [test_func for testName, test_func in globals().items() if testName.startswith("base_test")] diff --git a/src/DIRAC/DataManagementSystem/DB/test/Test_FTS3DB.py b/src/DIRAC/DataManagementSystem/DB/test/Test_FTS3DB.py index 90cb40ec036..42dcf138922 100644 --- a/src/DIRAC/DataManagementSystem/DB/test/Test_FTS3DB.py +++ b/src/DIRAC/DataManagementSystem/DB/test/Test_FTS3DB.py @@ -9,7 +9,7 @@ from DIRAC.DataManagementSystem.Client.FTS3Operation import FTS3Operation, FTS3TransferOperation, FTS3StagingOperation from DIRAC.DataManagementSystem.Client.FTS3File import FTS3File from DIRAC.DataManagementSystem.Client.FTS3Job import FTS3Job - +import DIRAC.DataManagementSystem.DB.test.FTS3TestUtils as baseTestModule gLogger.setLevel("DEBUG") @@ -189,3 +189,9 @@ def _makeFile(): activeJobs = res["Value"] activeJobIDs = [op.jobID for op in activeJobs] assert activeJobIDs == [1, 6] + + +@pytest.mark.parametrize("baseTest", baseTestModule.allBaseTests) +def test_all_common_tests(fts3db, baseTest): + """Run all the tests in the FTS3TestUtils.""" + baseTest(fts3db, fts3db) diff --git a/src/DIRAC/DataManagementSystem/Service/DataIntegrityHandler.py b/src/DIRAC/DataManagementSystem/Service/DataIntegrityHandler.py index 8892fe03d99..40047840827 100644 --- a/src/DIRAC/DataManagementSystem/Service/DataIntegrityHandler.py +++ b/src/DIRAC/DataManagementSystem/Service/DataIntegrityHandler.py @@ -58,7 +58,7 @@ def export_getProblematic(self): def export_getPrognosisProblematics(self, prognosis): """Get problematic files from the problematics table of the IntegrityDB""" - self.log.info("DataIntegrityHandler.getPrognosisProblematics: Getting files with %s prognosis." % prognosis) + self.log.info(f"DataIntegrityHandler.getPrognosisProblematics: Getting files with {prognosis} prognosis.") res = self.dataIntegrityDB.getPrognosisProblematics(prognosis) if not res["OK"]: self.log.error( @@ -80,7 +80,7 @@ def export_setProblematicStatus(self, fileID, status): def export_incrementProblematicRetry(self, fileID): """Update the retry count for supplied file ID.""" - self.log.info("DataIntegrityHandler.incrementProblematicRetry: Incrementing retries for file %s." % (fileID)) + self.log.info(f"DataIntegrityHandler.incrementProblematicRetry: Incrementing retries for file {fileID}.") res = self.dataIntegrityDB.incrementProblematicRetry(fileID) if not res["OK"]: self.log.error( @@ -126,7 +126,7 @@ def export_getProblematicsSummary(self): res = self.dataIntegrityDB.getProblematicsSummary() if res["OK"]: for prognosis, statusDict in res["Value"].items(): - self.log.info("DataIntegrityHandler.getProblematicsSummary: %s." % prognosis) + self.log.info(f"DataIntegrityHandler.getProblematicsSummary: {prognosis}.") for status, count in statusDict.items(): self.log.info("DataIntegrityHandler.getProblematicsSummary: \t%-10s %-10s." % (status, str(count))) else: @@ -141,7 +141,7 @@ def export_getDistinctPrognosis(self): res = self.dataIntegrityDB.getDistinctPrognosis() if res["OK"]: for prognosis in res["Value"]: - self.log.info("DataIntegrityHandler.getDistinctPrognosis: \t%s." % prognosis) + self.log.info(f"DataIntegrityHandler.getDistinctPrognosis: \t{prognosis}.") else: self.log.error("DataIntegrityHandler.getDistinctPrognosis: Failed to get unique prognosis.", res["Message"]) return res diff --git a/src/DIRAC/DataManagementSystem/Service/FTS3ManagerHandler.py b/src/DIRAC/DataManagementSystem/Service/FTS3ManagerHandler.py index a52968c1b1f..91992f6e387 100644 --- a/src/DIRAC/DataManagementSystem/Service/FTS3ManagerHandler.py +++ b/src/DIRAC/DataManagementSystem/Service/FTS3ManagerHandler.py @@ -8,16 +8,13 @@ :caption: FTS3Manager options """ -from DIRAC import S_OK, S_ERROR, gLogger +from DIRAC import S_ERROR, S_OK, gLogger from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getDNForUsername from DIRAC.Core.DISET.RequestHandler import RequestHandler, getServiceOption from DIRAC.Core.Security.Properties import FULL_DELEGATION, LIMITED_DELEGATION from DIRAC.Core.Utilities import DErrno -from DIRAC.Core.Utilities.JEncode import strToIntDict - - +from DIRAC.Core.Utilities.JEncode import decode, encode, strToIntDict from DIRAC.DataManagementSystem.DB.FTS3DB import FTS3DB -from DIRAC.Core.Utilities.JEncode import encode, decode ######################################################################## @@ -208,5 +205,3 @@ def export_getOperationsFromRMSOpID(cls, rmsOpID): class FTS3ManagerHandler(FTS3ManagerHandlerMixin, RequestHandler): """DISET handler for FTS3Manager""" - - pass diff --git a/src/DIRAC/DataManagementSystem/Service/FileCatalogHandler.py b/src/DIRAC/DataManagementSystem/Service/FileCatalogHandler.py index 02d6ccc6434..ddd7e1296bf 100644 --- a/src/DIRAC/DataManagementSystem/Service/FileCatalogHandler.py +++ b/src/DIRAC/DataManagementSystem/Service/FileCatalogHandler.py @@ -261,6 +261,12 @@ def export_getFileMetadata(self, lfns): """Get the metadata associated to supplied lfns""" return self.fileCatalogDB.getFileMetadata(lfns, self.getRemoteCredentials()) + types_getFileDetails = [[list, dict, str]] + + def export_getFileDetails(self, lfns): + """Get all the metadata associated to supplied lfns, including user metadata""" + return self.fileCatalogDB.getFileDetailsPublic(lfns, self.getRemoteCredentials()) + types_getReplicas = [[list, dict, str], bool] def export_getReplicas(self, lfns, allStatus=False): @@ -336,7 +342,7 @@ def export_isDirectory(self, lfns): types_getDirectoryMetadata = [[list, dict, str]] def export_getDirectoryMetadata(self, lfns): - """Get the size of the supplied directory""" + """Get the metadata of the supplied directory""" return self.fileCatalogDB.getDirectoryMetadata(lfns, self.getRemoteCredentials()) types_getDirectorySize = [[list, dict, str]] @@ -351,6 +357,12 @@ def export_getDirectoryReplicas(self, lfns, allStatus=False): """Get replicas for files in the supplied directory""" return self.fileCatalogDB.getDirectoryReplicas(lfns, allStatus, self.getRemoteCredentials()) + types_getDirectoryDump = [[list, dict, str]] + + def export_getDirectoryDump(self, lfns): + """Recursively list the contents of supplied directories""" + return self.fileCatalogDB.getDirectoryDump(lfns, self.getRemoteCredentials()) + ######################################################################## # # Administrative database operations @@ -387,7 +399,7 @@ def export_addMetadataField(self, fieldName, fieldType, metaType="-d"): elif metaType.lower() == "-f": return self.fileCatalogDB.fmeta.addMetadataField(fieldName, fieldType, self.getRemoteCredentials()) else: - return S_ERROR("Unknown metadata type %s" % metaType) + return S_ERROR(f"Unknown metadata type {metaType}") types_deleteMetadataField = [str] @@ -668,7 +680,7 @@ def transfer_toClient(self, jsonSENames, token, fileHelper): except Exception as e: self.log.exception("Exception while sending seDump", repr(e)) - return S_ERROR("Exception while sending seDump: %s" % repr(e)) + return S_ERROR(f"Exception while sending seDump: {repr(e)}") finally: if csvOutput is not None: csvOutput.close() diff --git a/src/DIRAC/DataManagementSystem/Service/FileCatalogProxyHandler.py b/src/DIRAC/DataManagementSystem/Service/FileCatalogProxyHandler.py deleted file mode 100644 index 56ca1d28117..00000000000 --- a/src/DIRAC/DataManagementSystem/Service/FileCatalogProxyHandler.py +++ /dev/null @@ -1,96 +0,0 @@ -""" -:mod: FileCatalogProxyHandler - -.. module: FileCatalogProxyHandler - :synopsis: This is a service which represents a DISET proxy to the File - Catalog - -""" -# imports -import os - -# from DIRAC -from DIRAC import gLogger, S_OK, S_ERROR -from DIRAC.Core.DISET.RequestHandler import RequestHandler -from DIRAC.Resources.Catalog.FileCatalogFactory import FileCatalogFactory -from DIRAC.Core.Utilities.Subprocess import pythonCall -from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager - - -def initializeFileCatalogProxyHandler(serviceInfo): - """service initalisation""" - return S_OK() - - -class FileCatalogProxyHandler(RequestHandler): - """ - .. class:: FileCatalogProxyHandler - """ - - types_callProxyMethod = [list((str,)), list((str,)), [tuple, list], dict] - - def export_callProxyMethod(self, fcName, methodName, args, kargs): - """A generic method to call methods of the Storage Element.""" - res = pythonCall(120, self.__proxyWrapper, fcName, methodName, args, kargs) - if res["OK"]: - return res["Value"] - else: - return res - - def __proxyWrapper(self, fcName, methodName, args, kwargs): - """The wrapper will obtain the client proxy and set it up in the environment. - The required functionality is then executed and returned to the client. - - :param self: self reference - :param str name: fcn name - :param tuple args: fcn args - :param dict kwargs: fcn keyword args - """ - result = self.__prepareSecurityDetails() - if not result["OK"]: - return result - proxyLocation = result["Value"] - try: - result = FileCatalogFactory().createCatalog(fcName) - if result["OK"]: - fileCatalog = result["Value"] - method = getattr(fileCatalog, methodName) - else: - return result - except AttributeError as error: - errStr = f"{fcName} proxy: no method named {methodName}" - gLogger.exception(errStr, methodName, error) - return S_ERROR(errStr) - try: - result = method(*args, **kwargs) - if os.path.exists(proxyLocation): - os.remove(proxyLocation) - return result - except Exception as error: - if os.path.exists(proxyLocation): - os.remove(proxyLocation) - errStr = f"{fcName} proxy: Exception while performing {methodName}" - gLogger.exception(errStr, error) - return S_ERROR(errStr) - - def __prepareSecurityDetails(self, vomsFlag=True): - """Obtains the connection details for the client""" - try: - credDict = self.getRemoteCredentials() - clientDN = credDict["DN"] - clientUsername = credDict["username"] - clientGroup = credDict["group"] - gLogger.debug(f"Getting proxy for {clientUsername}@{clientGroup} ({clientDN})") - if vomsFlag: - result = gProxyManager.downloadVOMSProxyToFile(clientDN, clientGroup) - else: - result = gProxyManager.downloadProxyToFile(clientDN, clientGroup) - if not result["OK"]: - return result - gLogger.debug("Updating environment.") - os.environ["X509_USER_PROXY"] = result["Value"] - return result - except Exception as error: - exStr = "__getConnectionDetails: Failed to get client connection details." - gLogger.exception(exStr, "", error) - return S_ERROR(exStr) diff --git a/src/DIRAC/DataManagementSystem/Service/S3GatewayHandler.py b/src/DIRAC/DataManagementSystem/Service/S3GatewayHandler.py index 27becd973b2..865d0f45a72 100644 --- a/src/DIRAC/DataManagementSystem/Service/S3GatewayHandler.py +++ b/src/DIRAC/DataManagementSystem/Service/S3GatewayHandler.py @@ -14,8 +14,7 @@ import errno # from DIRAC -from DIRAC import S_OK, S_ERROR, gLogger - +from DIRAC import S_ERROR, S_OK, gLogger from DIRAC.Core.DISET.RequestHandler import RequestHandler from DIRAC.Core.DISET.ThreadConfig import ThreadConfig from DIRAC.Core.Utilities.ReturnValues import returnSingleResult @@ -28,7 +27,7 @@ LOG = gLogger.getSubLogger(__name__) -class S3GatewayHandler(RequestHandler): +class S3GatewayHandlerMixin: """ .. class:: S3GatewayHandler @@ -72,12 +71,11 @@ def initializeHandler(cls, serviceInfoDict): and "Aws_access_key_id" in storageParam and "Aws_secret_access_key" in storageParam ): - cls._S3Storages[seName] = storagePlugin - log.debug("Add %s to the list of usable S3 storages" % seName) + log.debug(f"Add {seName} to the list of usable S3 storages") break - log.info("S3Gateway initialized storages", "%s" % list(cls._S3Storages)) + log.info("S3Gateway initialized storages", f"{list(cls._S3Storages)}") cls._fc = FileCatalog() @@ -88,7 +86,7 @@ def _hasAccess(self, lfn, s3_method): opType = self._s3ToFC_methods.get(s3_method) if not opType: - return S_ERROR(errno.EINVAL, "Unknown S3 method %s" % s3_method) + return S_ERROR(errno.EINVAL, f"Unknown S3 method {s3_method}") return returnSingleResult(self._fc.hasAccess(lfn, opType)) @@ -166,3 +164,7 @@ def export_createPresignedUrl(self, storageName, s3_method, urls, expiration): failed[url] = repr(e) return S_OK({"Successful": successful, "Failed": failed}) + + +class S3GatewayHandler(S3GatewayHandlerMixin, RequestHandler): + pass diff --git a/src/DIRAC/DataManagementSystem/Service/StorageElementHandler.py b/src/DIRAC/DataManagementSystem/Service/StorageElementHandler.py index 90f0e6cd863..d27d89d41af 100755 --- a/src/DIRAC/DataManagementSystem/Service/StorageElementHandler.py +++ b/src/DIRAC/DataManagementSystem/Service/StorageElementHandler.py @@ -64,7 +64,7 @@ def getDiskSpace(path, total=False): result = float(queriedSize * st.f_frsize) except OSError as e: - return S_ERROR(errno.EIO, "Error while getting the available disk space: %s" % repr(e)) + return S_ERROR(errno.EIO, f"Error while getting the available disk space: {repr(e)}") return S_OK(round(result, 4)) @@ -131,7 +131,7 @@ def initializeStorageElementHandler(serviceInfo): MAX_STORAGE_SIZE = convertSizeUnits(getServiceOption(serviceInfo, "MaxStorageSize", MAX_STORAGE_SIZE), "MB", "B") gLogger.info("Starting DIRAC Storage Element") - gLogger.info("Base Path: %s" % BASE_PATH) + gLogger.info(f"Base Path: {BASE_PATH}") gLogger.info("Max size: %d Bytes" % MAX_STORAGE_SIZE) gLogger.info("Use access control tokens: " + str(USE_TOKENS)) return S_OK() @@ -168,10 +168,10 @@ def __resolveFileID(self, fileID): if not port: return "" - if ":%s" % port in fileID: - loc = fileID.find(":%s" % port) + if f":{port}" in fileID: + loc = fileID.find(f":{port}") if loc >= 0: - fileID = fileID[loc + len(":%s" % port) :] + fileID = fileID[loc + len(f":{port}") :] serviceName = self.serviceInfoDict["serviceName"] loc = fileID.find(serviceName) @@ -198,7 +198,7 @@ def __getFileStat(path): if str(x).find("No such file") >= 0: resultDict["Exists"] = False return S_OK(resultDict) - return S_ERROR("Failed to get metadata for %s" % path) + return S_ERROR(f"Failed to get metadata for {path}") resultDict["Exists"] = True mode = statTuple[stat.ST_MODE] @@ -264,13 +264,13 @@ def export_getTotalDiskSpace(): def export_createDirectory(self, dir_path): """Creates the directory on the storage""" path = self.__resolveFileID(dir_path) - gLogger.info("StorageElementHandler.createDirectory: Attempting to create %s." % path) + gLogger.info(f"StorageElementHandler.createDirectory: Attempting to create {path}.") if os.path.exists(path): if os.path.isfile(path): errStr = "Supplied path exists and is a file" - gLogger.error("StorageElementHandler.createDirectory: %s." % errStr, path) + gLogger.error(f"StorageElementHandler.createDirectory: {errStr}.", path) return S_ERROR(errStr) - gLogger.info("StorageElementHandler.createDirectory: %s already exists." % path) + gLogger.info(f"StorageElementHandler.createDirectory: {path} already exists.") return S_OK() # Need to think about permissions. try: @@ -278,7 +278,7 @@ def export_createDirectory(self, dir_path): return S_OK() except Exception as x: errStr = "Exception creating directory." - gLogger.error("StorageElementHandler.createDirectory: %s" % errStr, repr(x)) + gLogger.error(f"StorageElementHandler.createDirectory: {errStr}", repr(x)) return S_ERROR(errStr) types_listDirectory = [str, str] @@ -286,10 +286,12 @@ def export_createDirectory(self, dir_path): def export_listDirectory(self, dir_path, mode): """Return the dir_path directory listing""" is_file = False + fname = None + dirList = None path = self.__resolveFileID(dir_path) if not os.path.exists(path): - return S_ERROR("Directory %s does not exist" % dir_path) - elif os.path.isfile(path): + return S_ERROR(f"Directory {dir_path} does not exist") + if os.path.isfile(path): fname = os.path.basename(path) is_file = True else: @@ -363,7 +365,7 @@ def transfer_toClient(self, fileID, token, fileHelper): result = fileHelper.sendEOF() # check if the file does not really exist if not os.path.exists(file_path): - return S_ERROR("File %s does not exist" % os.path.basename(file_path)) + return S_ERROR(f"File {os.path.basename(file_path)} does not exist") return S_ERROR("Failed to get file descriptor") fileDescriptor = result["Value"] @@ -435,9 +437,9 @@ def __removeFile(self, fileID, token): if str(error).find("No such file") >= 0: # File does not exist anyway return S_OK() - return S_ERROR("Failed to remove file %s" % fileID) + return S_ERROR(f"Failed to remove file {fileID}") else: - return S_ERROR("File removal %s not authorized" % fileID) + return S_ERROR(f"File removal {fileID} not authorized") types_getDirectorySize = [str] @@ -461,7 +463,7 @@ def export_removeDirectory(self, fileID, token): dir_path = self.__resolveFileID(fileID) if not self.__confirmToken(token, fileID, "x"): - return S_ERROR("Directory removal %s not authorized" % fileID) + return S_ERROR(f"Directory removal {fileID} not authorized") else: if not os.path.exists(dir_path): return S_OK() @@ -472,7 +474,7 @@ def export_removeDirectory(self, fileID, token): except Exception as error: gLogger.error("Failed to remove directory", dir_path) gLogger.error(str(error)) - return S_ERROR("Failed to remove directory %s" % dir_path) + return S_ERROR(f"Failed to remove directory {dir_path}") types_removeFileList = [list, str] @@ -519,7 +521,7 @@ def export_getAdminInfo(): @staticmethod def __getDirectorySize(path): """Get the total size of the given directory in bytes""" - comm = "du -sb %s" % path + comm = f"du -sb {path}" result = systemCall(10, shlex.split(comm)) if not result["OK"] or result["Value"][0]: return 0 diff --git a/src/DIRAC/DataManagementSystem/Service/StorageElementProxyHandler.py b/src/DIRAC/DataManagementSystem/Service/StorageElementProxyHandler.py deleted file mode 100644 index 6512195b910..00000000000 --- a/src/DIRAC/DataManagementSystem/Service/StorageElementProxyHandler.py +++ /dev/null @@ -1,339 +0,0 @@ -""" -:mod: StorageElementProxyHandler - -.. module: StorageElementProxyHandler - - :synopsis: This is a service which represents a DISET proxy to the Storage Element component. - -This is used to get and put files from a remote storage. -""" -import os -import shutil -import socketserver -import threading -import socket -import random - -# from DIRAC -from DIRAC import gLogger, gConfig, S_OK, S_ERROR -from DIRAC.Core.Utilities.File import mkDir -from DIRAC.Core.DISET.RequestHandler import RequestHandler -from DIRAC.Resources.Storage.StorageElement import StorageElement -from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager -from DIRAC.Core.Utilities.Subprocess import pythonCall -from DIRAC.Core.Utilities.Os import getDiskSpace -from DIRAC.Core.Utilities.DictCache import DictCache -from DIRAC.DataManagementSystem.private.HttpStorageAccessHandler import HttpStorageAccessHandler -from DIRAC.Core.Utilities.ReturnValues import returnSingleResult -from DIRAC.ConfigurationSystem.Client.Helpers import Registry - -# globals -BASE_PATH = "" -HTTP_FLAG = False -HTTP_PORT = 9180 -HTTP_PATH = "" - - -def purgeCacheDirectory(path): - """del recursively :path:""" - shutil.rmtree(path) - - -gRegister = DictCache(purgeCacheDirectory) - - -def initializeStorageElementProxyHandler(serviceInfo): - """handler initialisation""" - - global BASE_PATH, HTTP_FLAG, HTTP_PORT, HTTP_PATH - cfgPath = serviceInfo["serviceSectionPath"] - - BASE_PATH = gConfig.getValue("%s/BasePath" % cfgPath, BASE_PATH) - if not BASE_PATH: - gLogger.error("Failed to get the base path") - return S_ERROR("Failed to get the base path") - - BASE_PATH = os.path.abspath(BASE_PATH) - gLogger.info("The base path obtained is %s. Checking its existence..." % BASE_PATH) - mkDir(BASE_PATH) - - HTTP_FLAG = gConfig.getValue("%s/HttpAccess" % cfgPath, False) - if HTTP_FLAG: - HTTP_PATH = "%s/httpCache" % BASE_PATH - HTTP_PATH = gConfig.getValue("%s/HttpCache" % cfgPath, HTTP_PATH) - mkDir(HTTP_PATH) - HTTP_PORT = gConfig.getValue("%s/HttpPort" % cfgPath, 9180) - gLogger.info("Creating HTTP server thread, port:%d, path:%s" % (HTTP_PORT, HTTP_PATH)) - _httpThread = HttpThread(HTTP_PORT, HTTP_PATH) - - return S_OK() - - -class ThreadedSocketServer(socketserver.ThreadingMixIn, socketserver.TCPServer): - """bag dummy class to hold ThreadingMixIn and TCPServer""" - - pass - - -class HttpThread(threading.Thread): - """ - .. class:: HttpThread - - Single daemon thread running HttpStorageAccessHandler. - """ - - def __init__(self, port, path): - """c'tor""" - self.port = port - self.path = path - threading.Thread.__init__(self) - self.daemon = True - self.start() - - def run(self): - """thread run""" - global gRegister - handler = HttpStorageAccessHandler - handler.register = gRegister - handler.basePath = self.path - httpd = ThreadedSocketServer(("", self.port), handler) - httpd.serve_forever() - - -class StorageElementProxyHandler(RequestHandler): - """ - .. class:: StorageElementProxyHandler - """ - - types_callProxyMethod = [(str,), (str,), list, dict] - - def export_callProxyMethod(self, se, name, args, kargs): - """A generic method to call methods of the Storage Element.""" - res = pythonCall(200, self.__proxyWrapper, se, name, args, kargs) - if res["OK"]: - return res["Value"] - return res - - def __proxyWrapper(self, se, name, args, kargs): - """The wrapper will obtain the client proxy and set it up in the environment. - - The required functionality is then executed and returned to the client. - """ - res = self.__prepareSecurityDetails() - if not res["OK"]: - return res - credDict = self.getRemoteCredentials() - group = credDict["group"] - vo = Registry.getVOForGroup(group) - if not vo: - return S_ERROR("Can not determine VO of the operation requester") - storageElement = StorageElement(se, vo=vo) - method = getattr(storageElement, name) if hasattr(storageElement, name) else None - if not method: - return S_ERROR("Method '%s' isn't implemented!" % name) - if not callable(getattr(storageElement, name)): - return S_ERROR("Attribute '%s' isn't a method!" % name) - return method(*args, **kargs) - - types_uploadFile = [(str,), (str,)] - - def export_uploadFile(self, se, pfn): - """This method uploads a file present in the local cache to the specified storage element""" - res = pythonCall(300, self.__uploadFile, se, pfn) - if res["OK"]: - return res["Value"] - return res - - def __uploadFile(self, se, pfn): - """proxied upload file""" - res = self.__prepareSecurityDetails() - if not res["OK"]: - return res - - # Put file to the SE - try: - storageElement = StorageElement(se) - except AttributeError as x: - errStr = "__uploadFile: Exception while instantiating the Storage Element." - gLogger.exception(errStr, se, str(x)) - return S_ERROR(errStr) - putFileDir = "%s/putFile" % BASE_PATH - localFileName = f"{putFileDir}/{os.path.basename(pfn)}" - res = returnSingleResult(storageElement.putFile({pfn: localFileName})) - if not res["OK"]: - gLogger.error("prepareFile: Failed to put local file to storage.", res["Message"]) - # Clear the local cache - try: - gLogger.debug("Removing temporary file", localFileName) - os.remove(localFileName) - except Exception as x: - gLogger.exception("Failed to remove local file", localFileName, x) - return res - - types_prepareFile = [(str,), (str,)] - - def export_prepareFile(self, se, pfn): - """This method simply gets the file to the local storage area""" - res = pythonCall(300, self.__prepareFile, se, pfn) - if res["OK"]: - return res["Value"] - return res - - def __prepareFile(self, se, pfn): - """proxied prepare file""" - res = self.__prepareSecurityDetails() - if not res["OK"]: - return res - - # Clear the local cache - getFileDir = "%s/getFile" % BASE_PATH - if not os.path.exists(getFileDir): - os.mkdir(getFileDir) - - # Get the file to the cache - try: - storageElement = StorageElement(se) - except AttributeError as x: - errStr = "prepareFile: Exception while instantiating the Storage Element." - gLogger.exception(errStr, se, str(x)) - return S_ERROR(errStr) - res = returnSingleResult(storageElement.getFile(pfn, localPath="%s/getFile" % BASE_PATH)) - if not res["OK"]: - gLogger.error("prepareFile: Failed to get local copy of file.", res["Message"]) - return res - return S_OK() - - types_prepareFileForHTTP = [list((str,)) + [list]] - - def export_prepareFileForHTTP(self, lfn): - """This method simply gets the file to the local storage area using LFN""" - - # Do clean-up, should be a separate regular thread - gRegister.purgeExpired() - - key = str(random.getrandbits(128)) - result = pythonCall(300, self.__prepareFileForHTTP, lfn, key) - if result["OK"]: - result = result["Value"] - # pylint believes it is a tuple because it is the only possible return type - # it finds in Subprocess.py - if result["OK"]: # pylint: disable=invalid-sequence-index - if HTTP_FLAG: - host = socket.getfqdn() - url = "http://%s:%d/%s" % (host, HTTP_PORT, key) - gRegister.add(key, 1800, result["CachePath"]) - result["HttpKey"] = key - result["HttpURL"] = url - return result - return result - return result - - def __prepareFileForHTTP(self, lfn, key): - """Prepare proxied file for HTTP""" - global HTTP_PATH - - res = self.__prepareSecurityDetails() - if not res["OK"]: - return res - - # Clear the local cache - getFileDir = f"{HTTP_PATH}/{key}" - mkDir(getFileDir) - - # Get the file to the cache - from DIRAC.DataManagementSystem.Client.DataManager import DataManager - - dataMgr = DataManager() - result = dataMgr.getFile(lfn, destinationDir=getFileDir) - result["CachePath"] = getFileDir - return result - - ############################################################ - # - # This is the method to setup the proxy and configure the environment with the client credential - # - - def __prepareSecurityDetails(self): - """Obtains the connection details for the client""" - try: - credDict = self.getRemoteCredentials() - clientDN = credDict["DN"] - clientUsername = credDict["username"] - clientGroup = credDict["group"] - gLogger.debug(f"Getting proxy for {clientUsername}@{clientGroup} ({clientDN})") - res = gProxyManager.downloadVOMSProxy(clientDN, clientGroup) - if not res["OK"]: - return res - chain = res["Value"] - proxyBase = "%s/proxies" % BASE_PATH - mkDir(proxyBase) - proxyLocation = f"{BASE_PATH}/proxies/{clientUsername}-{clientGroup}" - gLogger.debug("Obtained proxy chain, dumping to %s." % proxyLocation) - res = gProxyManager.dumpProxyToFile(chain, proxyLocation) - if not res["OK"]: - return res - gLogger.debug("Updating environment.") - os.environ["X509_USER_PROXY"] = res["Value"] - return res - except Exception as error: - exStr = "__getConnectionDetails: Failed to get client connection details." - gLogger.exception(exStr, "", error) - return S_ERROR(exStr) - - ############################################################ - # - # These are the methods that are for actual file transfer - # - - def transfer_toClient(self, fileID, token, fileHelper): - """Method to send files to clients. - fileID is the local file name in the SE. - token is used for access rights confirmation. - """ - file_path = f"{BASE_PATH}/{fileID}" - result = fileHelper.getFileDescriptor(file_path, "r") - if not result["OK"]: - result = fileHelper.sendEOF() - # check if the file does not really exist - if not os.path.exists(file_path): - return S_ERROR("File %s does not exist" % os.path.basename(file_path)) - else: - return S_ERROR("Failed to get file descriptor") - - fileDescriptor = result["Value"] - result = fileHelper.FDToNetwork(fileDescriptor) - if not result["OK"]: - return S_ERROR("Failed to get file %s" % fileID) - - if os.path.exists(file_path): - os.remove(file_path) - - return result - - def transfer_fromClient(self, fileID, token, fileSize, fileHelper): - """Method to receive file from clients. - fileID is the local file name in the SE. - fileSize can be Xbytes or -1 if unknown. - token is used for access rights confirmation. - """ - if not self.__checkForDiskSpace(BASE_PATH, fileSize): - return S_ERROR("Not enough disk space") - - file_path = f"{BASE_PATH}/{fileID}" - mkDir(os.path.dirname(file_path)) - result = fileHelper.getFileDescriptor(file_path, "w") - if not result["OK"]: - return S_ERROR("Failed to get file descriptor") - - fileDescriptor = result["Value"] - result = fileHelper.networkToFD(fileDescriptor) - if not result["OK"]: - return S_ERROR("Failed to put file %s" % fileID) - return result - - @staticmethod - def __checkForDiskSpace(dpath, size): - """Check if the directory dpath can accomodate 'size' volume of data""" - dsize = (getDiskSpace(dpath) - 1) * 1024 * 1024 - maxStorageSizeBytes = 1024 * 1024 * 1024 - return min(dsize, maxStorageSizeBytes) > size diff --git a/src/DIRAC/DataManagementSystem/Service/TornadoFTS3ManagerHandler.py b/src/DIRAC/DataManagementSystem/Service/TornadoFTS3ManagerHandler.py index c9c51f2f287..3c67659ed25 100644 --- a/src/DIRAC/DataManagementSystem/Service/TornadoFTS3ManagerHandler.py +++ b/src/DIRAC/DataManagementSystem/Service/TornadoFTS3ManagerHandler.py @@ -1,8 +1,17 @@ +""" +Service handler for FTS3DB using Tornado + +.. literalinclude:: ../ConfigTemplate.cfg + :start-after: ##BEGIN TornadoFTS3Manager + :end-before: ##END + :dedent: 2 + :caption: TornadoFTS3Manager options + +""" + from DIRAC.Core.Tornado.Server.TornadoService import TornadoService from DIRAC.DataManagementSystem.Service.FTS3ManagerHandler import FTS3ManagerHandlerMixin class TornadoFTS3ManagerHandler(FTS3ManagerHandlerMixin, TornadoService): """Tornado handler for the FTS3Manager""" - - pass diff --git a/src/DIRAC/DataManagementSystem/Service/TornadoFileCatalogHandler.py b/src/DIRAC/DataManagementSystem/Service/TornadoFileCatalogHandler.py index 49342bddfc9..fff7f694438 100644 --- a/src/DIRAC/DataManagementSystem/Service/TornadoFileCatalogHandler.py +++ b/src/DIRAC/DataManagementSystem/Service/TornadoFileCatalogHandler.py @@ -56,7 +56,7 @@ def export_streamToClient(self, jsonSENames): except Exception as e: self.log.exception("Exception while sending seDump", repr(e)) - return S_ERROR("Exception while sendind seDump: %s" % repr(e)) + return S_ERROR(f"Exception while sendind seDump: {repr(e)}") finally: if csvOutput is not None: csvOutput.close() diff --git a/src/DIRAC/DataManagementSystem/Service/TornadoS3GatewayHandler.py b/src/DIRAC/DataManagementSystem/Service/TornadoS3GatewayHandler.py new file mode 100644 index 00000000000..139961f6342 --- /dev/null +++ b/src/DIRAC/DataManagementSystem/Service/TornadoS3GatewayHandler.py @@ -0,0 +1,14 @@ +""" TornadoS3Gateway is the implementation of the S3Gateway service in HTTPS + + .. literalinclude:: ../ConfigTemplate.cfg + :start-after: ##BEGIN TornadoS3Gateway + :end-before: ##END + :dedent: 2 + :caption: TornadoS3Gateway options +""" +from DIRAC.Core.Tornado.Server.TornadoService import TornadoService +from DIRAC.DataManagementSystem.Service.S3GatewayHandler import S3GatewayHandlerMixin + + +class TornadoS3GatewayHandler(S3GatewayHandlerMixin, TornadoService): + pass diff --git a/src/DIRAC/DataManagementSystem/Utilities/DMSHelpers.py b/src/DIRAC/DataManagementSystem/Utilities/DMSHelpers.py index 7705964b0af..ea469f20125 100644 --- a/src/DIRAC/DataManagementSystem/Utilities/DMSHelpers.py +++ b/src/DIRAC/DataManagementSystem/Utilities/DMSHelpers.py @@ -11,6 +11,8 @@ PROTOCOL = LOCAL + 1 DOWNLOAD = PROTOCOL + 1 +sLog = gLogger.getSubLogger(__name__) + def resolveSEGroup(seGroupList, allSEs=None): """ @@ -26,14 +28,14 @@ def resolveSEGroup(seGroupList, allSEs=None): if allSEs is None: res = gConfig.getSections("/Resources/StorageElements") if not res["OK"]: - gLogger.fatal("Error getting list of SEs from CS", res["Message"]) + sLog.fatal("Error getting list of SEs from CS", res["Message"]) return [] allSEs = res["Value"] seList = [] if isinstance(seGroupList, str): seGroupList = [se.strip() for se in seGroupList.split(",") if se.strip()] for se in seGroupList: - seConfig = gConfig.getValue("/Resources/StorageElementGroups/%s" % se, se) + seConfig = gConfig.getValue(f"/Resources/StorageElementGroups/{se}", se) if seConfig != se: newSEs = [se1.strip() for se1 in seConfig.split(",") if se1.strip()] # print seList @@ -43,7 +45,7 @@ def resolveSEGroup(seGroupList, allSEs=None): if se1 not in allSEs: # Here means se is not a group and is not an SE either, fatal! if se1 == se: - gLogger.fatal("%s is not a valid SE" % se1) + sLog.fatal(f"{se1} is not a valid SE") return [] # If not an SE, it may be a group recursive = resolveSEGroup(se1, allSEs=allSEs) @@ -112,7 +114,7 @@ def getSiteSEMapping(self): # BaseSE storageElements = gConfig.getSections("Resources/StorageElements") if not storageElements["OK"]: - gLogger.warn("Problem retrieving storage elements", storageElements["Message"]) + sLog.warn("Problem retrieving storage elements", storageElements["Message"]) return storageElements storageElements = storageElements["Value"] equivalentSEs = {} @@ -126,20 +128,20 @@ def getSiteSEMapping(self): siteSEMapping = {} gridTypes = gConfig.getSections("Resources/Sites/") if not gridTypes["OK"]: - gLogger.warn("Problem retrieving sections in /Resources/Sites", gridTypes["Message"]) + sLog.warn("Problem retrieving sections in /Resources/Sites", gridTypes["Message"]) return gridTypes gridTypes = gridTypes["Value"] - gLogger.debug("Grid Types are: %s" % (", ".join(gridTypes))) + sLog.debug(f"Grid Types are: {', '.join(gridTypes)}") # Get a list of sites and their local SEs siteSet = set() storageElementSet = set() siteSEMapping[LOCAL] = {} for grid in gridTypes: - result = gConfig.getSections("/Resources/Sites/%s" % grid) + result = gConfig.getSections(f"/Resources/Sites/{grid}") if not result["OK"]: - gLogger.warn("Problem retrieving /Resources/Sites/%s section" % grid) + sLog.warn(f"Problem retrieving /Resources/Sites/{grid} section") return result sites = result["Value"] siteSet.update(sites) @@ -188,7 +190,7 @@ def getSiteSEMapping(self): # Add storage elements that may not be associated with a site result = gConfig.getSections("/Resources/StorageElements") if not result["OK"]: - gLogger.warn("Problem retrieving /Resources/StorageElements section", result["Message"]) + sLog.warn("Problem retrieving /Resources/StorageElements section", result["Message"]) return result self.storageElementSet = storageElementSet | set(result["Value"]) self.siteSet = siteSet @@ -335,7 +337,7 @@ def getSEsForSite(self, site, connectionLevel=None): if not self.siteSet: self.getSiteSEMapping() if site not in self.siteSet: - siteList = [s for s in self.siteSet if ".%s." % site in s] + siteList = [s for s in self.siteSet if f".{site}." in s] else: siteList = [site] if not siteList: @@ -400,7 +402,7 @@ def getAllSEsInGroupAtSite(self, seGroup, site): return sesAtSite foundSEs = set(seList) & set(sesAtSite["Value"]) if not foundSEs: - gLogger.warn("No SE found at that site", f"in group {seGroup} at {site}") + sLog.warn("No SE found at that site", f"in group {seGroup} at {site}") return S_OK() return S_OK(sorted(foundSEs)) diff --git a/src/DIRAC/DataManagementSystem/Utilities/ResolveSE.py b/src/DIRAC/DataManagementSystem/Utilities/ResolveSE.py new file mode 100644 index 00000000000..0eccbec8fdb --- /dev/null +++ b/src/DIRAC/DataManagementSystem/Utilities/ResolveSE.py @@ -0,0 +1,118 @@ +""" This module allows to resolve output SEs for Job based +on SE and site/country association +""" + +from random import shuffle + +from DIRAC import gLogger, gConfig +from DIRAC.Core.Utilities.SiteSEMapping import getSEsForSite +from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup + +sLog = gLogger.getSubLogger(__name__) + + +def _setLocalFirst(seList, localSEs): + """return a shuffled list of SEs from seList, localSEs being first.""" + # Make a copy not to change the original order + seList = list(seList) + shuffle(seList) + # localSEs are put first in the list + return sorted(seList, key=lambda x: x not in localSEs) + + +def getDestinationSEList(outputSE, site, outputmode="Any"): + """Evaluate the output SE list from a workflow and return the concrete list + of SEs to upload output data. The resolution order goes as follow: + + * outputSE as a normal StorageElement + * outputSE as an alias of one SE defined in the ``site`` AssociatedSEs (return local first) + * outputSE as an alias of multiple SE (local SE should come first) + * outputSE as a StorageElementGroup + + Moreover, if output mode is `Local`: + + * return ONLY local SE within the SEGroup if they exist (i.e. in the ``/SE>`` config) + * look at associated countries and countries association + + + :param str outputSE: name of the SE or SEGroup we want to resolve + :param str site: site on which we are running + :param str outputmode: (default "Any") resolution mode + + :returns: list of string + + :raises: + RuntimeError if anything is wrong + + """ + + if not outputSE: + return [] + + if outputmode.lower() not in ("any", "local"): + raise RuntimeError("Unexpected outputmode") + + # Add output SE defined in the job description + sLog.info("Resolving workflow output SE description", str(outputSE)) + + # Check if the SE is defined explicitly for the site + prefix = site.split(".")[0] + country = site.split(".")[-1] + + # Concrete SE name + result = gConfig.getOptions(f"/Resources/StorageElements/{outputSE}") + if result["OK"]: + sLog.info("Found concrete SE", str(outputSE)) + return [outputSE] + + # Get local SEs + localSEs = getSEsForSite(site) + if not localSEs["OK"]: + raise RuntimeError(localSEs["Message"]) + localSEs = localSEs["Value"] + sLog.verbose("Local SE list is:", ", ".join(localSEs)) + # There is an alias defined for this Site + associatedSEs = gConfig.getValue(f"/Resources/Sites/{prefix}/{site}/AssociatedSEs/{outputSE}", []) + if associatedSEs: + associatedSEs = _setLocalFirst(associatedSEs, localSEs) + sLog.info("Found associated SE for site", f"{associatedSEs} associated to {site}") + return associatedSEs + + groupSEs = resolveSEGroup(outputSE) + if not groupSEs: + raise RuntimeError(f"Failed to resolve SE {outputSE}") + sLog.verbose("Group SE list is:", str(groupSEs)) + + # Find a local SE or an SE considered as local because the country is associated to it + if outputmode.lower() == "local": + # First, check if one SE in the group is local + ses = list(set(localSEs) & set(groupSEs)) + if ses: + sLog.info("Found eligible local SE", str(ses)) + return ses + + # Final check for country associated SE + assignedCountry = country + while True: + # check if country is already one with associated SEs + section = f"/Resources/Countries/{assignedCountry}/AssociatedSEs/{outputSE}" + associatedSEs = gConfig.getValue(section, []) + if associatedSEs: + associatedSEs = _setLocalFirst(associatedSEs, localSEs) + sLog.info("Found associated SEs", f"{associatedSEs} in {section}") + return associatedSEs + + opt = gConfig.getOption(f"/Resources/Countries/{assignedCountry}/AssignedTo") + if opt["OK"] and opt["Value"]: + assignedCountry = opt["Value"] + else: + # No associated SE and no assigned country, give up + raise RuntimeError( + f"Could not establish associated SE nor assigned country for country {assignedCountry}" + ) + + # For collective Any and All modes return the whole group + # Make sure that local SEs are passing first + orderedSEs = _setLocalFirst(groupSEs, localSEs) + sLog.info("Found SEs, local first:", str(orderedSEs)) + return orderedSEs diff --git a/src/DIRAC/DataManagementSystem/Utilities/test/Test_resolveSE.py b/src/DIRAC/DataManagementSystem/Utilities/test/Test_resolveSE.py new file mode 100644 index 00000000000..5bedf465ef6 --- /dev/null +++ b/src/DIRAC/DataManagementSystem/Utilities/test/Test_resolveSE.py @@ -0,0 +1,306 @@ +import os +import pytest +import tempfile + + +from DIRAC.tests.Utilities.utils import generateDIRACConfig + +from DIRAC.DataManagementSystem.Utilities.ResolveSE import getDestinationSEList +from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup + +from DIRAC import gLogger + +gLogger.setLevel("DEBUG") + + +CFG_CONTENT = """ +Resources +{ + StorageElements + { + LogSE + { + } + CERN-BUFFER + { + } + CERN-FAILOVER + { + } + CERN-DST + { + } + CNAF-BUFFER + { + } + CNAF-FAILOVER + { + } + CNAF-DST + { + } + GRIDKA-BUFFER + { + } + GRIDKA-FAILOVER + { + } + GRIDKA-DST + { + } + } + StorageElementGroups + { + # Define a StorageElementGroup which is not + # overwriten in any associatedSEs + Tier1-Failover = CERN-FAILOVER, CNAF-FAILOVER + + # Define a SEGroup which will be + # overwriten at CERN only + Tier1-DST = CERN-DST, CNAF-DST + } + Sites + { + LCG + { + LCG.CERN.cern + { + # Local SEs + SE = CERN-BUFFER, CERN-LogSE, CERN-FAILOVER, CNAF-DST + + AssociatedSEs + { + + # This should have no impact because + # LogSE exists as an SE, so we do not + # check associated SE + + LogSE = CERN-LogSE + + # Here define a name that is used as an alias + + LocalLogSE = CERN-LogSE + + # Tier1-Buffer is defined as associated site + # for all Sites. + # It's not a real use case. + # We would normally define a StorageElementGroup + # and overwrite it if needed in some sites + # (like what we do for Tier1-DST) + Tier1-Buffer = CERN-BUFFER, CNAF-BUFFER + + # Overwite the definition of the Tier1-DST group + # for this site only + Tier1-DST = CERN-DST + } + } + + LCG.CNAF.it + { + SE = CNAF-BUFFER, CNAF-FAILOVER, CNAF-DST + AssociatedSEs + { + # We do NOT define CNAF-LogSE as local, yet it is still returned + # when using ``outputmode = local`` because ``local`` only impacts + # ``StorageElementGroups`` + LocalLogSE = CNAF-LogSE + + # We anyway expect CNAF-BUFFER to be returned first, as it is local + Tier1-Buffer = CERN-BUFFER, CNAF-BUFFER + } + } + + # Do not update anything for this site + LCG.IN2P3.fr + { + } + + # Do not update anything for this site + # but define some aliases in the ``de`` country + LCG.GRIDKA.de + { + } + } + } + + # Countries are taking into account only when using ``Local`` mode + Countries + { + pl + { + # Associate Polish sites to german sites + AssignedTo = de + } + de + { + AssociatedSEs + { + + # This alias will never be used + # because ``LocalLogSE`` is not a + # ``StorageElementGroup``, so the resolution + # does not take place, even when used with ``Local`` + LocalLogSE = GRIDKA-LogSE + + # As for LocalLogSE, we will never reach that + # alias + Tier1-Buffer = GRIDKA-BUFFER + + Tier1-DST = GRIDKA-DST + + } + } + } +} +""" + + +@pytest.fixture(scope="module", autouse=True) +def loadCS(): + """Load the CFG_CONTENT as a DIRAC Configuration for this module""" + with generateDIRACConfig(CFG_CONTENT, "test_resolveSE.cfg"): + yield + + +def test_directSEName(): + """Should return the LogSE as is, full stop""" + # Asking for a specific SE should return that SE, no matter what + for site in ("LCG.CERN.cern", "LCG.CNAF.it", "LCG.IN2P3.fr", "LCG.GRIDKA.de", "LCG.NCBJ.pl", "AnySite"): + assert getDestinationSEList("LogSE", site) == ["LogSE"] + assert getDestinationSEList("LogSE", site, outputmode="Local") == ["LogSE"] + + +def test_emptyInput(): + """Should return empty list""" + + assert getDestinationSEList("", "site") == [] + assert getDestinationSEList("", "site", outputmode="Local") == [] + assert getDestinationSEList([], "site") == [] + assert getDestinationSEList([], "site", outputmode="Local") == [] + + +def test_directSEName_redefined(): + """Redifining an existing SEName has no impact""" + # CERN redefines LogSE in the associatedSEs + # but it should be ignored + assert getDestinationSEList("LogSE", "LCG.CERN.cern") == ["LogSE"] + assert getDestinationSEList("LogSE", "LCG.CERN.cern", outputmode="Local") == ["LogSE"] + + +def test_associatedSE_singleSE(): + """Map a given name to an SE in the sites definition""" + # CERN defines LocalLogSE as "CERN-LogSE" + assert getDestinationSEList("LocalLogSE", "LCG.CERN.cern") == ["CERN-LogSE"] + assert getDestinationSEList("LocalLogSE", "LCG.CERN.cern", outputmode="Local") == ["CERN-LogSE"] + # CNAF defines LocalLogSE as "CNAF-LogSE + assert getDestinationSEList("LocalLogSE", "LCG.CNAF.it") == ["CNAF-LogSE"] + # CNAF-LogSE is NOT defined as local, but it is still returned as + # ``Local`` only impacts StorageElementGroups resolution + assert getDestinationSEList("LocalLogSE", "LCG.CNAF.it", outputmode="Local") == ["CNAF-LogSE"] + + # LocalLogSE does not exist for these sites + + for site in ("LCG.IN2P3.fr", "LCG.GRIDKA.de", "LCG.NCBJ.pl"): + with pytest.raises(RuntimeError): + assert getDestinationSEList("LocalLogSE", site) + + # LocalLogSE is NOT a StorageElementGroup, so ``Local`` + # will not help, even if ``LocalLogSE`` is defined in the + # ``de`` country + # NOTE: That is quite counter intuitive and could be revisited + for site in ("LCG.IN2P3.fr", "LCG.GRIDKA.de", "LCG.NCBJ.pl"): + with pytest.raises(RuntimeError): + assert getDestinationSEList("LocalLogSE", site, outputmode="Local") + + +def test_associatedSE_group(): + """Map a given name to multiple SEs. The order should be different + because the local storages are returned first""" + + # CERN defines Tier1-Buffer as CERN-BUFFER, CNAF-BUFFER + assert getDestinationSEList("Tier1-Buffer", "LCG.CERN.cern") == ["CERN-BUFFER", "CNAF-BUFFER"] + # We could expect ``Local`` to reduce the output to CERN-BUFFER, but it does not + # because Tier1-Buffer is NOT a StorageElement + assert getDestinationSEList("Tier1-Buffer", "LCG.CERN.cern", outputmode="Local") == ["CERN-BUFFER", "CNAF-BUFFER"] + + # CNAF defines Tier1-Buffer as CERN-BUFFER, CNAF-BUFFER but the order is changed + # because local SEs go first + assert getDestinationSEList("Tier1-Buffer", "LCG.CNAF.it") == ["CNAF-BUFFER", "CERN-BUFFER"] + # Same as CERN, with local SEs first + assert getDestinationSEList("Tier1-Buffer", "LCG.CNAF.it") == ["CNAF-BUFFER", "CERN-BUFFER"] + + # Tier1-Buffer does not exist for these sites + for site in ("LCG.IN2P3.fr", "LCG.GRIDKA.de", "LCG.NCBJ.pl"): + with pytest.raises(RuntimeError): + assert getDestinationSEList("Tier1-Buffer", site) + + # Tier1-Buffer does not exist for these sites, and since it is not + # a StorageElementGroup, we do not do the country resolution + for site in ("LCG.IN2P3.fr", "LCG.GRIDKA.de", "LCG.NCBJ.pl"): + with pytest.raises(RuntimeError): + assert getDestinationSEList("Tier1-Buffer", site, outputmode="Local") + + +def test_seGroup(): + """Test resolving a StorageElementGroup not redefined anywhere""" + + # Tier1-Failover is not redifined anywhere + + # Retrieve the full list, sorting local SEs first + assert getDestinationSEList("Tier1-Failover", "LCG.CERN.cern") == ["CERN-FAILOVER", "CNAF-FAILOVER"] + assert getDestinationSEList("Tier1-Failover", "LCG.CNAF.it") == ["CNAF-FAILOVER", "CERN-FAILOVER"] + + # In ``Local`` mode, we ONLY get local SE + assert getDestinationSEList("Tier1-Failover", "LCG.CERN.cern", outputmode="Local") == ["CERN-FAILOVER"] + assert getDestinationSEList("Tier1-Failover", "LCG.CNAF.it", outputmode="Local") == ["CNAF-FAILOVER"] + + # Here we get the full Tier1-Failover list + # We have to compare with ``sorted`` because the order is shuffled + for site in ("LCG.IN2P3.fr", "LCG.GRIDKA.de", "LCG.NCBJ.pl"): + assert sorted(getDestinationSEList("Tier1-Failover", site)) == sorted(["CERN-FAILOVER", "CNAF-FAILOVER"]) + + # When using the ``Local`` mode, we get an error because none of + # the SE is local to any of the site. + for site in ("LCG.IN2P3.fr", "LCG.GRIDKA.de", "LCG.NCBJ.pl"): + with pytest.raises(RuntimeError): + getDestinationSEList("Tier1-Failover", site, outputmode="Local") + + +def test_seGroup_overwrite(): + """Test resolving a StorageElementGroup which is redefined in some places""" + + # CERN redefineds Tier1-DST + assert getDestinationSEList("Tier1-DST", "LCG.CERN.cern") == ["CERN-DST"] + # CERN-DST is local to CERN, so the same when using ``Local`` mode + assert getDestinationSEList("Tier1-DST", "LCG.CERN.cern", outputmode="Local") == ["CERN-DST"] + + # CNAF does NOT redefine Tier1-DST, but it puts local SE first + assert getDestinationSEList("Tier1-DST", "LCG.CNAF.it") == ["CNAF-DST", "CERN-DST"] + # CNAF-DST is local to CNAF, so get only that if using ``Local`` mode + assert getDestinationSEList("Tier1-DST", "LCG.CNAF.it", outputmode="Local") == ["CNAF-DST"] + + # Sites do not redefine Tier1-DST, so we get the full + # Tier1-DST list in a random order + for site in ("LCG.IN2P3.fr", "LCG.GRIDKA.de", "LCG.NCBJ.pl"): + assert sorted(getDestinationSEList("Tier1-DST", site)) == sorted(["CNAF-DST", "CERN-DST"]) + + # There are no SE in Tier1-DST which are ``Local`` to IN2P3, + # and IN2P3 is not associated to any country, + # so we don't find anything + with pytest.raises(RuntimeError): + getDestinationSEList("Tier1-DST", "LCG.IN2P3.fr", outputmode="Local") + + # Here we redefine the Tier1-DST at the country level: + # de has GRIDKA-DST as Tier1-DST + # pl points to de + for site in ("LCG.GRIDKA.de", "LCG.NCBJ.pl"): + assert getDestinationSEList("Tier1-DST", site, outputmode="Local") == ["GRIDKA-DST"] + + +def test_compat_resolveSEGroup(): + """We want to make sure that resolveSEGroup produces the same output + when we do not overwrite the SEGroup definition and use the default outputMode + """ + + for site in ("LCG.CERN.cern", "LCG.CNAF.it", "LCG.IN2P3.fr", "LCG.GRIDKA.de", "LCG.NCBJ.pl", "AnySite"): + assert sorted(getDestinationSEList("Tier1-Failover", site=site)) == sorted(resolveSEGroup("Tier1-Failover")) diff --git a/src/DIRAC/Resources/Cloud/__init__.py b/src/DIRAC/DataManagementSystem/Utilities/test/__init__.py similarity index 100% rename from src/DIRAC/Resources/Cloud/__init__.py rename to src/DIRAC/DataManagementSystem/Utilities/test/__init__.py diff --git a/src/DIRAC/DataManagementSystem/private/FTS3Plugins/test/Test_DefaultFTS3Plugin.py b/src/DIRAC/DataManagementSystem/private/FTS3Plugins/test/Test_DefaultFTS3Plugin.py index 26e96e164a1..2762f622471 100644 --- a/src/DIRAC/DataManagementSystem/private/FTS3Plugins/test/Test_DefaultFTS3Plugin.py +++ b/src/DIRAC/DataManagementSystem/private/FTS3Plugins/test/Test_DefaultFTS3Plugin.py @@ -2,18 +2,17 @@ import os import tempfile -import pytest +from unittest import mock +import pytest from diraccfg import CFG import DIRAC - -from DIRAC.ConfigurationSystem.private.ConfigurationClient import ConfigurationClient +from DIRAC import S_OK from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData - -from DIRAC.Resources.Storage.StorageBase import StorageBase +from DIRAC.ConfigurationSystem.private.ConfigurationClient import ConfigurationClient from DIRAC.DataManagementSystem.private.FTS3Plugins.DefaultFTS3Plugin import DefaultFTS3Plugin -from DIRAC import S_OK +from DIRAC.Resources.Storage.StorageBase import StorageBase # pylint: disable=redefined-outer-name @@ -318,6 +317,21 @@ def fts3Plugin(monkeypatch): monkeypatch.setattr( DIRAC.Resources.Storage.StorageElement.StorageElementItem, "addAccountingOperation", lambda: None ) + + def mock_init(self, useProxy=False, vo=None): + self.proxy = False + self.proxy = useProxy + self.resourceStatus = mock.MagicMock() + self.vo = vo + self.remoteProtocolSections = [] + self.localProtocolSections = [] + self.name = "" + self.options = {} + self.protocols = {} + self.storages = {} + + monkeypatch.setattr(DIRAC.Resources.Storage.StorageFactory.StorageFactory, "__init__", mock_init) + fts3Plugin = DefaultFTS3Plugin() return fts3Plugin diff --git a/src/DIRAC/DataManagementSystem/private/FTS3Utilities.py b/src/DIRAC/DataManagementSystem/private/FTS3Utilities.py index 18ba26add6e..9aa60074258 100644 --- a/src/DIRAC/DataManagementSystem/private/FTS3Utilities.py +++ b/src/DIRAC/DataManagementSystem/private/FTS3Utilities.py @@ -59,7 +59,6 @@ def selectUniqueSource(ftsFiles, fts3Plugin, allowedSources=None): failedFiles = {} for ftsFile in ftsFiles: - # If we failed to get the replicas, add the FTS3File to the dictionary if ftsFile.lfn in filteredReplicas["Failed"]: errMsg = filteredReplicas["Failed"][ftsFile.lfn] @@ -107,7 +106,7 @@ def getFTS3Plugin(vo=None): objLoader = ObjectLoader() _class = objLoader.loadObject( - "DataManagementSystem.private.FTS3Plugins.%sFTS3Plugin" % pluginName, "%sFTS3Plugin" % pluginName + f"DataManagementSystem.private.FTS3Plugins.{pluginName}FTS3Plugin", f"{pluginName}FTS3Plugin" ) if not _class["OK"]: @@ -138,9 +137,9 @@ def __init__(self, serverDict, serverPolicy="Random"): self._nextServerID = 0 self._resourceStatus = ResourceStatus() - methName = "_%sServerPolicy" % serverPolicy.lower() + methName = f"_{serverPolicy.lower()}ServerPolicy" if not hasattr(self, methName): - self.log.error("Unknown server policy %s. Using Random instead" % serverPolicy) + self.log.error(f"Unknown server policy {serverPolicy}. Using Random instead") methName = "_randomServerPolicy" self._policyMethod = getattr(self, methName) @@ -189,7 +188,7 @@ def _getFTSServerStatus(self, ftsServer): result = res["Value"] if ftsServer not in result: - return S_ERROR("No FTS Server %s known to RSS" % ftsServer) + return S_ERROR(f"No FTS Server {ftsServer} known to RSS") if result[ftsServer]["all"] == "Active": return S_OK(True) @@ -205,7 +204,6 @@ def chooseFTS3Server(self): attempt = 0 while not fts3Server and attempt < self._maxAttempts: - fts3Server = self._policyMethod(attempt) res = self._getFTSServerStatus(fts3Server) @@ -218,7 +216,7 @@ def chooseFTS3Server(self): ftsServerStatus = res["Value"] if not ftsServerStatus: - self.log.warn("FTS server %s is not in good shape. Choose another one" % fts3Server) + self.log.warn(f"FTS server {fts3Server} is not in good shape. Choose another one") fts3Server = None attempt += 1 diff --git a/src/DIRAC/DataManagementSystem/private/HttpStorageAccessHandler.py b/src/DIRAC/DataManagementSystem/private/HttpStorageAccessHandler.py index f870e0224b9..64f0eb2cf8f 100644 --- a/src/DIRAC/DataManagementSystem/private/HttpStorageAccessHandler.py +++ b/src/DIRAC/DataManagementSystem/private/HttpStorageAccessHandler.py @@ -4,7 +4,7 @@ ######################################################################## """ The HttpStorageAccessHandler is a http server request handler to provide a secure http - access to the DIRAC StorageElement and StorageElementProxy. It is derived from the + access to the DIRAC StorageElement. It is derived from the SimpleHTTPRequestHandler standard python handler """ import os @@ -17,7 +17,6 @@ class HttpStorageAccessHandler(server.BaseHTTPRequestHandler): - register = DictCache() basePath = "" @@ -39,7 +38,7 @@ def do_GET(self): unique = str(random.getrandbits(24)) fileString = " ".join(fileList) os.system(f"tar -cf {cache_path}/dirac_data_{unique}.tar --remove-files -C {cache_path} {fileString}") - path = os.path.join(cache_path, "dirac_data_%s.tar" % unique) + path = os.path.join(cache_path, f"dirac_data_{unique}.tar") f = self.send_head(path) if f: @@ -66,6 +65,6 @@ def send_head(self, path): self.send_header("Last-Modified", self.date_time_string(fs.st_mtime)) fname = os.path.basename(path) self.send_header("Last-Modified", self.date_time_string(fs.st_mtime)) - self.send_header("Content-Disposition", "filename=%s" % fname) + self.send_header("Content-Disposition", f"filename={fname}") self.end_headers() return f diff --git a/src/DIRAC/DataManagementSystem/private/test/Test_FTS3Utilities.py b/src/DIRAC/DataManagementSystem/private/test/Test_FTS3Utilities.py index a654a8136cb..5e6437f6858 100644 --- a/src/DIRAC/DataManagementSystem/private/test/Test_FTS3Utilities.py +++ b/src/DIRAC/DataManagementSystem/private/test/Test_FTS3Utilities.py @@ -50,7 +50,6 @@ def setUp(self): self.allFiles = [self.f1, self.f2, self.f3, self.f4] def test_01_groupFilesByTarget(self): - # empty input self.assertTrue(groupFilesByTarget([])["Value"] == {}) diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_admin_allow_se.py b/src/DIRAC/DataManagementSystem/scripts/dirac_admin_allow_se.py index 038f3680463..4150b2a4c89 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_admin_allow_se.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_admin_allow_se.py @@ -17,6 +17,7 @@ def main(): remove = False site = "" mute = False + userName = "" Script.registerSwitch("r", "AllowRead", " Allow only reading from the storage element") Script.registerSwitch("w", "AllowWrite", " Allow only writing to the storage element") @@ -25,6 +26,7 @@ def main(): Script.registerSwitch("a", "All", " Allow all access to the storage element") Script.registerSwitch("m", "Mute", " Do not send email") Script.registerSwitch("S:", "Site=", " Allow all SEs associated to site") + Script.registerSwitch("t:", "tokenOwner=", " Optional Name of the token owner") # Registering arguments will automatically add their description to the help menu Script.registerArgument(["seGroupList: list of SEs or comma-separated SEs"]) @@ -48,9 +50,11 @@ def main(): mute = True if switch[0].lower() in ("s", "site"): site = switch[1] + if switch[0] in ("t", "tokenOwner"): + userName = switch[1] # imports - from DIRAC import gConfig, gLogger + from DIRAC import gLogger from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSites from DIRAC.Core.Security.ProxyInfo import getProxyInfo @@ -68,21 +72,17 @@ def main(): ses = resolveSEGroup(ses) diracAdmin = DiracAdmin() - errorList = [] - setup = gConfig.getValue("/DIRAC/Setup", "") - if not setup: - print("ERROR: Could not contact Configuration Service") - DIRAC.exit(2) - res = getProxyInfo() - if not res["OK"]: - gLogger.error("Failed to get proxy information", res["Message"]) - DIRAC.exit(2) - - userName = res["Value"].get("username") if not userName: - gLogger.error("Failed to get username for proxy") - DIRAC.exit(2) + res = getProxyInfo() + if not res["OK"]: + gLogger.error("Failed to get proxy information", res["Message"]) + DIRAC.exit(2) + + userName = res["Value"].get("username") + if not userName: + gLogger.error("Failed to get username for proxy") + DIRAC.exit(2) if site: res = getSites() @@ -90,7 +90,7 @@ def main(): gLogger.error(res["Message"]) DIRAC.exit(-1) if site not in res["Value"]: - gLogger.error("The provided site (%s) is not known." % site) + gLogger.error(f"The provided site ({site}) is not known.") DIRAC.exit(-1) ses.extend(res["Value"]["SE"].replace(" ", "").split(",")) if not ses: @@ -98,7 +98,7 @@ def main(): DIRAC.exit() STATUS_TYPES = ["ReadAccess", "WriteAccess", "CheckAccess", "RemoveAccess"] - ALLOWED_STATUSES = ["Unknown", "InActive", "Banned", "Probing", "Degraded"] + ALLOWED_STATUSES = ["Unknown", "InActive", "Banned", "Probing", "Degraded", "Error"] statusAllowedDict = {} for statusType in STATUS_TYPES: @@ -114,10 +114,10 @@ def main(): res = resourceStatus.getElementStatus(ses, "StorageElement") if not res["OK"]: - gLogger.error("Storage Element %s does not exist" % ses) + gLogger.error(f"Storage Element {ses} does not exist") DIRAC.exit(-1) - reason = "Forced with dirac-admin-allow-se by %s" % userName + reason = f"Forced with dirac-admin-allow-se by {userName}" for se, seOptions in res["Value"].items(): # InActive is used on the CS model, Banned is the equivalent in RSS @@ -159,34 +159,35 @@ def main(): gLogger.notice("Email is muted by script switch") DIRAC.exit(0) - subject = "%s storage elements allowed for use" % len(totalAllowedSEs) + subject = f"{len(totalAllowedSEs)} storage elements allowed for use" addressPath = "EMail/Production" address = Operations().getValue(addressPath, "") + fromAddress = Operations().getValue("ResourceStatus/Config/FromAddress", "") body = "" if read: - body = "%s\n\nThe following storage elements were allowed for reading:" % body + body = f"{body}\n\nThe following storage elements were allowed for reading:" for se in statusAllowedDict["ReadAccess"]: body = f"{body}\n{se}" if write: - body = "%s\n\nThe following storage elements were allowed for writing:" % body + body = f"{body}\n\nThe following storage elements were allowed for writing:" for se in statusAllowedDict["WriteAccess"]: body = f"{body}\n{se}" if check: - body = "%s\n\nThe following storage elements were allowed for checking:" % body + body = f"{body}\n\nThe following storage elements were allowed for checking:" for se in statusAllowedDict["CheckAccess"]: body = f"{body}\n{se}" if remove: - body = "%s\n\nThe following storage elements were allowed for removing:" % body + body = f"{body}\n\nThe following storage elements were allowed for removing:" for se in statusAllowedDict["RemoveAccess"]: body = f"{body}\n{se}" if not address: - gLogger.notice("'%s' not defined in Operations, can not send Mail\n" % addressPath, body) + gLogger.notice(f"'{addressPath}' not defined in Operations, can not send Mail\n", body) DIRAC.exit(0) - res = diracAdmin.sendMail(address, subject, body) - gLogger.notice("Notifying %s" % address) + res = diracAdmin.sendMail(address, subject, body, fromAddress=fromAddress) + gLogger.notice(f"Notifying {address}") if res["OK"]: gLogger.notice(res["Value"]) else: diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_admin_ban_se.py b/src/DIRAC/DataManagementSystem/scripts/dirac_admin_ban_se.py index 9787254d19d..91553dc2a06 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_admin_ban_se.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_admin_ban_se.py @@ -18,6 +18,7 @@ def main(): remove = True sites = [] mute = False + userName = "" Script.registerSwitch("r", "BanRead", " Ban only reading from the storage element") Script.registerSwitch("w", "BanWrite", " Ban writing to the storage element") @@ -28,6 +29,7 @@ def main(): Script.registerSwitch( "S:", "Site=", " Ban all SEs associate to site (note that if writing is allowed, check is always allowed)" ) + Script.registerSwitch("t:", "tokenOwner=", " Optional Name of the token owner") # Registering arguments will automatically add their description to the help menu Script.registerArgument(["seGroupList: list of SEs or comma-separated SEs"]) @@ -56,31 +58,30 @@ def main(): mute = True if switch[0].lower() in ("s", "site"): sites = switch[1].split(",") + if switch[0] in ("t", "tokenOwner"): + userName = switch[1] # from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI - from DIRAC import gConfig, gLogger + from DIRAC import gLogger from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations from DIRAC.Core.Security.ProxyInfo import getProxyInfo + from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers, resolveSEGroup from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus - from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup, DMSHelpers ses = resolveSEGroup(ses) diracAdmin = DiracAdmin() - setup = gConfig.getValue("/DIRAC/Setup", "") - if not setup: - print("ERROR: Could not contact Configuration Service") - DIRAC.exit(2) - res = getProxyInfo() - if not res["OK"]: - gLogger.error("Failed to get proxy information", res["Message"]) - DIRAC.exit(2) - - userName = res["Value"].get("username") if not userName: - gLogger.error("Failed to get username for proxy") - DIRAC.exit(2) + res = getProxyInfo() + if not res["OK"]: + gLogger.error("Failed to get proxy information", res["Message"]) + DIRAC.exit(2) + + userName = res["Value"].get("username") + if not userName: + gLogger.error("Failed to get username for proxy") + DIRAC.exit(2) for site in sites: res = DMSHelpers().getSEsForSite(site) @@ -102,40 +103,36 @@ def main(): res = resourceStatus.getElementStatus(ses, "StorageElement") if not res["OK"]: - gLogger.error("Storage Element %s does not exist" % ses) + gLogger.error(f"Storage Element {ses} does not exist") DIRAC.exit(-1) - reason = "Forced with dirac-admin-ban-se by %s" % userName + reason = f"Forced with dirac-admin-ban-se by {userName}" for se, seOptions in res["Value"].items(): - resW = resC = resR = {"OK": False} # Eventually, we will get rid of the notion of InActive, as we always write Banned. if read and "ReadAccess" in seOptions: - if seOptions["ReadAccess"] == "Banned": gLogger.notice("Read access already banned", se) resR["OK"] = True - elif not seOptions["ReadAccess"] in ["Active", "Degraded", "Probing"]: + elif not seOptions["ReadAccess"] in ["Active", "Degraded", "Probing", "Error"]: gLogger.notice( "Read option for %s is %s, instead of %s" - % (se, seOptions["ReadAccess"], ["Active", "Degraded", "Probing"]) + % (se, seOptions["ReadAccess"], ["Active", "Degraded", "Probing", "Error"]) ) gLogger.notice("Try specifying the command switches") else: - resR = resourceStatus.setElementStatus(se, "StorageElement", "ReadAccess", "Banned", reason, userName) # res = csAPI.setOption( "%s/%s/ReadAccess" % ( storageCFGBase, se ), "InActive" ) if not resR["OK"]: - gLogger.error("Failed to update %s read access to Banned" % se) + gLogger.error(f"Failed to update {se} read access to Banned") else: - gLogger.notice("Successfully updated %s read access to Banned" % se) + gLogger.notice(f"Successfully updated {se} read access to Banned") readBanned.append(se) # Eventually, we will get rid of the notion of InActive, as we always write Banned. if write and "WriteAccess" in seOptions: - if seOptions["WriteAccess"] == "Banned": gLogger.notice("Write access already banned", se) resW["OK"] = True @@ -146,18 +143,16 @@ def main(): ) gLogger.notice("Try specifying the command switches") else: - resW = resourceStatus.setElementStatus(se, "StorageElement", "WriteAccess", "Banned", reason, userName) # res = csAPI.setOption( "%s/%s/WriteAccess" % ( storageCFGBase, se ), "InActive" ) if not resW["OK"]: - gLogger.error("Failed to update %s write access to Banned" % se) + gLogger.error(f"Failed to update {se} write access to Banned") else: - gLogger.notice("Successfully updated %s write access to Banned" % se) + gLogger.notice(f"Successfully updated {se} write access to Banned") writeBanned.append(se) # Eventually, we will get rid of the notion of InActive, as we always write Banned. if check and "CheckAccess" in seOptions: - if seOptions["CheckAccess"] == "Banned": gLogger.notice("Check access already banned", se) resC["OK"] = True @@ -168,18 +163,16 @@ def main(): ) gLogger.notice("Try specifying the command switches") else: - resC = resourceStatus.setElementStatus(se, "StorageElement", "CheckAccess", "Banned", reason, userName) # res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" ) if not resC["OK"]: - gLogger.error("Failed to update %s check access to Banned" % se) + gLogger.error(f"Failed to update {se} check access to Banned") else: - gLogger.notice("Successfully updated %s check access to Banned" % se) + gLogger.notice(f"Successfully updated {se} check access to Banned") checkBanned.append(se) # Eventually, we will get rid of the notion of InActive, as we always write Banned. if remove and "RemoveAccess" in seOptions: - if seOptions["RemoveAccess"] == "Banned": gLogger.notice("Remove access already banned", se) resC["OK"] = True @@ -190,13 +183,12 @@ def main(): ) gLogger.notice("Try specifying the command switches") else: - resC = resourceStatus.setElementStatus(se, "StorageElement", "RemoveAccess", "Banned", reason, userName) # res = csAPI.setOption( "%s/%s/CheckAccess" % ( storageCFGBase, se ), "InActive" ) if not resC["OK"]: - gLogger.error("Failed to update %s remove access to Banned" % se) + gLogger.error(f"Failed to update {se} remove access to Banned") else: - gLogger.notice("Successfully updated %s remove access to Banned" % se) + gLogger.notice(f"Successfully updated {se} remove access to Banned") removeBanned.append(se) if not (resR["OK"] or resW["OK"] or resC["OK"]): @@ -210,34 +202,35 @@ def main(): gLogger.notice("Email is muted by script switch") DIRAC.exit(0) - subject = "%s storage elements banned for use" % len(writeBanned + readBanned + checkBanned + removeBanned) + subject = f"{len(writeBanned + readBanned + checkBanned + removeBanned)} storage elements banned for use" addressPath = "EMail/Production" address = Operations().getValue(addressPath, "") + fromAddress = Operations().getValue("ResourceStatus/Config/FromAddress", "") body = "" if read: - body = "%s\n\nThe following storage elements were banned for reading:" % body + body = f"{body}\n\nThe following storage elements were banned for reading:" for se in readBanned: body = f"{body}\n{se}" if write: - body = "%s\n\nThe following storage elements were banned for writing:" % body + body = f"{body}\n\nThe following storage elements were banned for writing:" for se in writeBanned: body = f"{body}\n{se}" if check: - body = "%s\n\nThe following storage elements were banned for check access:" % body + body = f"{body}\n\nThe following storage elements were banned for check access:" for se in checkBanned: body = f"{body}\n{se}" if remove: - body = "%s\n\nThe following storage elements were banned for remove access:" % body + body = f"{body}\n\nThe following storage elements were banned for remove access:" for se in removeBanned: body = f"{body}\n{se}" if not address: - gLogger.notice("'%s' not defined in Operations, can not send Mail\n" % addressPath, body) + gLogger.notice(f"'{addressPath}' not defined in Operations, can not send Mail\n", body) DIRAC.exit(0) - res = diracAdmin.sendMail(address, subject, body) - gLogger.notice("Notifying %s" % address) + res = diracAdmin.sendMail(address, subject, body, fromAddress=fromAddress) + gLogger.notice(f"Notifying {address}") if res["OK"]: gLogger.notice(res["Value"]) else: diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_admin_user_quota.py b/src/DIRAC/DataManagementSystem/scripts/dirac_admin_user_quota.py index 3372c92c960..1498772ccea 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_admin_user_quota.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_admin_user_quota.py @@ -33,10 +33,10 @@ def main(): users = res["Value"] gLogger.notice("-" * 30) - gLogger.notice("{}|{}".format("Username".ljust(15), "Quota (GB)".rjust(15))) + gLogger.notice(f"{'Username'.ljust(15)}|{'Quota (GB)'.rjust(15)}") gLogger.notice("-" * 30) for user in sorted(users): - quota = gConfig.getValue("/Registry/Users/%s/Quota" % user, 0) + quota = gConfig.getValue(f"/Registry/Users/{user}/Quota", 0) if not quota: quota = gConfig.getValue("/Registry/DefaultStorageQuota") gLogger.notice(f"{user.ljust(15)}|{str(quota).rjust(15)}") diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_add_file.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_add_file.py index 5f7f9f79702..4aee253342a 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_add_file.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_add_file.py @@ -87,7 +87,7 @@ def main(): lfns.append(getDict(items)) inputFile.close() else: - gLogger.error("Error: LFN list '%s' missing." % inputFileName) + gLogger.error(f"Error: LFN list '{inputFileName}' missing.") exitCode = 4 else: lfns.append(getDict(args)) @@ -95,22 +95,22 @@ def main(): dm = DataManager() for lfn in lfns: if not os.path.exists(lfn["localfile"]): - gLogger.error("File %s must exist locally" % lfn["localfile"]) + gLogger.error(f"File {lfn['localfile']} must exist locally") exitCode = 1 continue if not os.path.isfile(lfn["localfile"]): - gLogger.error("%s is not a file" % lfn["localfile"]) + gLogger.error(f"{lfn['localfile']} is not a file") exitCode = 2 continue - gLogger.notice("\nUploading %s" % lfn["lfn"]) + gLogger.notice(f"\nUploading {lfn['lfn']}") res = dm.putAndRegister(lfn["lfn"], lfn["localfile"], lfn["SE"], lfn["guid"], overwrite=overwrite) if not res["OK"]: exitCode = 3 - gLogger.error("Error: failed to upload {} to {}: {}".format(lfn["lfn"], lfn["SE"], res)) + gLogger.error(f"Error: failed to upload {lfn['lfn']} to {lfn['SE']}: {res}") continue else: - gLogger.notice("Successfully uploaded file to %s" % lfn["SE"]) + gLogger.notice(f"Successfully uploaded file to {lfn['SE']}") DIRAC.exit(exitCode) diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_change_replica_status.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_change_replica_status.py index 58abee92b07..167a397eb79 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_change_replica_status.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_change_replica_status.py @@ -47,9 +47,9 @@ def main(): if not res["OK"]: print("ERROR:", res["Message"]) if res["Value"]["Failed"]: - print("Failed to update %d replica status" % len(res["Value"]["Failed"])) + print(f"Failed to update {len(res['Value']['Failed'])} replica status") if res["Value"]["Successful"]: - print("Successfully updated %d replica status" % len(res["Value"]["Successful"])) + print(f"Successfully updated {len(res['Value']['Successful'])} replica status") if __name__ == "__main__": diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_clean_directory.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_clean_directory.py index d59c1c92519..d00fca2e0a3 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_clean_directory.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_clean_directory.py @@ -34,7 +34,7 @@ def main(): dm = DataManager() retVal = 0 for lfn in [lfn for lfn in lfns if lfn]: - gLogger.notice("Cleaning directory %r ... " % lfn) + gLogger.notice(f"Cleaning directory {lfn!r} ... ") result = dm.cleanLogicalDirectory(lfn) if not result["OK"]: gLogger.error("Failed to clean directory", result["Message"]) diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_archive_request.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_archive_request.py index 067b3e8d19b..d5b23650b36 100644 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_archive_request.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_archive_request.py @@ -155,12 +155,12 @@ def registerSwitchesAndParseCommandLine(self): sLog.error('The "ArchiveFiles" operation is not enabled, contact your administrator!') DIRAC.exit(1) for _short, longOption, _doc in self.options: - defaultValue = ops.getValue("DataManagement/ArchiveFiles/%s" % longOption, None) + defaultValue = ops.getValue(f"DataManagement/ArchiveFiles/{longOption}", None) if defaultValue: sLog.verbose(f"Found default value in the CS for {longOption!r} with value {defaultValue!r}") self.switches[longOption] = defaultValue for _short, longOption, _doc in self.flags: - defaultValue = ops.getValue("DataManagement/ArchiveFiles/%s" % longOption, False) + defaultValue = ops.getValue(f"DataManagement/ArchiveFiles/{longOption}", False) if defaultValue: sLog.verbose(f"Found default value in the CS for {longOption!r} with value {defaultValue!r}") self.switches[longOption] = defaultValue @@ -193,16 +193,16 @@ def getLFNList(self): if os.path.exists(self.switches.get("List")): self.lfnList = list({line.split()[0] for line in open(self.switches.get("List")).read().splitlines()}) else: - raise ValueError("%s not a file" % self.switches.get("List")) + raise ValueError(f"{self.switches.get('List')} not a file") elif self.lfnFolderPath: path = self.lfnFolderPath - sLog.debug("Check if %r is a directory" % path) + sLog.debug(f"Check if {path!r} is a directory") isDir = returnSingleResult(self.fcClient.isDirectory(path)) - sLog.debug("Result: %r" % isDir) + sLog.debug(f"Result: {isDir!r}") if not isDir["OK"] or not isDir["Value"]: sLog.error("Path is not a directory", isDir.get("Message", "")) - raise RuntimeError("Path %r is not a directory" % path) - sLog.notice("Looking for files in %r" % path) + raise RuntimeError(f"Path {path!r} is not a directory") + sLog.notice(f"Looking for files in {path!r}") metaDict = {"SE": self.sourceSEs[0]} if self.switches.get("SourceOnly") else {} lfns = self.fcClient.findFilesByMetadata(metaDict=metaDict, path=path) @@ -212,7 +212,7 @@ def getLFNList(self): self.lfnList = lfns["Value"] if self.lfnList: - sLog.notice("Will create request(s) with %d lfns" % len(self.lfnList)) + sLog.notice(f"Will create request(s) with {len(self.lfnList)} lfns") if len(self.lfnList) == 1: raise RuntimeError("Only 1 file in the list, aborting!") return @@ -224,7 +224,7 @@ def putOrRunRequests(self): requestIDs = [] if self.dryRun: - sLog.notice("Would have created %d requests" % len(self.requests)) + sLog.notice(f"Would have created {len(self.requests)} requests") for reqID, req in enumerate(self.requests): sLog.notice("Request %d:" % reqID) for opID, op in enumerate(req): @@ -233,14 +233,14 @@ def putOrRunRequests(self): for request in self.requests: putRequest = self.reqClient.putRequest(request) if not putRequest["OK"]: - sLog.error("unable to put request {!r}: {}".format(request.RequestName, putRequest["Message"])) + sLog.error(f"unable to put request {request.RequestName!r}: {putRequest['Message']}") continue requestIDs.append(str(putRequest["Value"])) - sLog.always("Request %r has been put to ReqDB for execution." % request.RequestName) + sLog.always(f"Request {request.RequestName!r} has been put to ReqDB for execution.") if requestIDs: - sLog.always("%d requests have been put to ReqDB for execution" % len(requestIDs)) - sLog.always("RequestID(s): %s" % " ".join(requestIDs)) + sLog.always(f"{len(requestIDs)} requests have been put to ReqDB for execution") + sLog.always(f"RequestID(s): {' '.join(requestIDs)}") sLog.always("You can monitor the request status using the command: dirac-rms-request ") return 0 @@ -257,7 +257,7 @@ def checkSwitches(self): self.switches["AutoName"] = os.path.join( os.path.dirname(self.lfnFolderPath), os.path.basename(self.lfnFolderPath) + ".tar" ) - sLog.notice("Using %r for tarball" % self.switches.get("AutoName")) + sLog.notice(f"Using {self.switches.get('AutoName')!r} for tarball") if self.switches.get("List") and not self.name: raise RuntimeError('Have to set "Name" with "List"') @@ -308,7 +308,7 @@ def run(self): else: baseArchiveLFN = archiveLFN = self.name tarballName = os.path.basename(archiveLFN) - baseRequestName = requestName = "Archive_%s" % tarballName.rsplit(".", 1)[0] + baseRequestName = requestName = f"Archive_{tarballName.rsplit('.', 1)[0]}" from DIRAC.RequestManagementSystem.private.RequestValidator import RequestValidator @@ -330,7 +330,7 @@ def run(self): valid = RequestValidator().validate(request) if not valid["OK"]: - sLog.error("putRequest: request not valid", "%s" % valid["Message"]) + sLog.error("putRequest: request not valid", f"{valid['Message']}") return 1 else: self.requests.append(request) @@ -366,8 +366,8 @@ def getLFNMetadata(self): metaData = self.fcClient.getFileMetadata(self.lfnList) error = False if not metaData["OK"]: - sLog.error("Unable to read metadata for lfns: %s" % metaData["Message"]) - raise RuntimeError("Could not read metadata: %s" % metaData["Message"]) + sLog.error(f"Unable to read metadata for lfns: {metaData['Message']}") + raise RuntimeError(f"Could not read metadata: {metaData['Message']}") self.metaData = metaData["Value"] for failedLFN, reason in self.metaData["Failed"].items(): @@ -377,7 +377,7 @@ def getLFNMetadata(self): raise RuntimeError("Could not read all metadata") for lfn in self.metaData["Successful"].keys(): - sLog.verbose("found %s" % lfn) + sLog.verbose(f"found {lfn}") def createRequest(self, requestName, archiveLFN, lfnChunk): """Create the Request.""" @@ -455,17 +455,17 @@ def createRequest(self, requestName, archiveLFN, lfnChunk): def checkArchive(self, archiveLFN): """Check that archiveLFN does not exist yet.""" - sLog.notice("Using Tarball: %s" % archiveLFN) + sLog.notice(f"Using Tarball: {archiveLFN}") exists = returnSingleResult(self.fcClient.isFile(archiveLFN)) - sLog.debug("Checking for Tarball existence %r" % exists) + sLog.debug(f"Checking for Tarball existence {exists!r}") if exists["OK"] and exists["Value"]: - raise RuntimeError("Tarball %r already exists" % archiveLFN) + raise RuntimeError(f"Tarball {archiveLFN!r} already exists") - sLog.debug("Checking permissions for %r" % archiveLFN) + sLog.debug(f"Checking permissions for {archiveLFN!r}") hasAccess = returnSingleResult(self.fcClient.hasAccess(archiveLFN, "addFile")) if not archiveLFN or not hasAccess["OK"] or not hasAccess["Value"]: - sLog.error("Error checking tarball location: %r" % hasAccess) - raise ValueError('%s is not a valid path, parameter "Name" must be correct' % archiveLFN) + sLog.error(f"Error checking tarball location: {hasAccess!r}") + raise ValueError(f'{archiveLFN} is not a valid path, parameter "Name" must be correct') def _checkReplicaSites(self, request, lfnChunk): """Ensure that all lfns can be found at the SourceSE, otherwise add replication operation to request. @@ -486,7 +486,7 @@ def _checkReplicaSites(self, request, lfnChunk): if sourceSE in replInfo: atSource.append(lfn) else: - sLog.notice("WARN: LFN {!r} not found at source, only at: {}".format(lfn, ",".join(replInfo.keys()))) + sLog.notice(f"WARN: LFN {lfn!r} not found at source, only at: {','.join(replInfo.keys())}") notAt.append(lfn) for lfn, errorMessage in resReplica["Value"]["Failed"].items(): @@ -495,7 +495,7 @@ def _checkReplicaSites(self, request, lfnChunk): if failed: raise RuntimeError("Failed to get replica information") - sLog.notice("Found %d files to replicate" % len(notAt)) + sLog.notice(f"Found {len(notAt)} files to replicate") if not notAt: return if notAt and self.switches.get("AllowReplication"): diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_moving_request.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_moving_request.py index 8a8c3182673..4985210a9e7 100644 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_moving_request.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_moving_request.py @@ -136,16 +136,16 @@ def getLFNList(self): if os.path.exists(self.switches.get("List")): self.lfnList = list({line.split()[0] for line in open(self.switches.get("List")).read().splitlines()}) else: - raise ValueError("%s not a file" % self.switches.get("List")) + raise ValueError(f"{self.switches.get('List')} not a file") elif self.lfnFolderPath: path = self.lfnFolderPath - sLog.debug("Check if %r is a directory" % path) + sLog.debug(f"Check if {path!r} is a directory") isDir = returnSingleResult(self.fcClient.isDirectory(path)) - sLog.debug("Result: %r" % isDir) + sLog.debug(f"Result: {isDir!r}") if not isDir["OK"] or not isDir["Value"]: sLog.error("Path is not a directory", isDir.get("Message", "")) - raise RuntimeError("Path %r is not a directory" % path) - sLog.notice("Looking for files in %r" % path) + raise RuntimeError(f"Path {path!r} is not a directory") + sLog.notice(f"Looking for files in {path!r}") metaDict = {"SE": self.sourceSEs[0]} if self.switches.get("SourceOnly") else {} lfns = self.fcClient.findFilesByMetadata(metaDict=metaDict, path=path) @@ -155,7 +155,7 @@ def getLFNList(self): self.lfnList = lfns["Value"] if self.lfnList: - sLog.notice("Will create request(s) with %d lfns" % len(self.lfnList)) + sLog.notice(f"Will create request(s) with {len(self.lfnList)} lfns") if len(self.lfnList) == 1: raise RuntimeError("Only 1 file in the list, aborting!") return @@ -167,8 +167,8 @@ def getLFNMetadata(self): metaData = self.fcClient.getFileMetadata(self.lfnList) error = False if not metaData["OK"]: - sLog.error("Unable to read metadata for lfns: %s" % metaData["Message"]) - raise RuntimeError("Could not read metadata: %s" % metaData["Message"]) + sLog.error(f"Unable to read metadata for lfns: {metaData['Message']}") + raise RuntimeError(f"Could not read metadata: {metaData['Message']}") self.metaData = metaData["Value"] for failedLFN, reason in self.metaData["Failed"].items(): @@ -178,7 +178,7 @@ def getLFNMetadata(self): raise RuntimeError("Could not read all metadata") for lfn in self.metaData["Successful"].keys(): - sLog.verbose("found %s" % lfn) + sLog.verbose(f"found {lfn}") def run(self): """Perform checks and create the request.""" @@ -193,7 +193,7 @@ def run(self): request = self.createRequest(requestName, lfnChunk) valid = RequestValidator().validate(request) if not valid["OK"]: - sLog.error("putRequest: request not valid", "%s" % valid["Message"]) + sLog.error("putRequest: request not valid", f"{valid['Message']}") return 1 else: self.requests.append(request) @@ -255,7 +255,7 @@ def putOrRunRequests(self): requestIDs = [] if self.dryRun: - sLog.notice("Would have created %d requests" % len(self.requests)) + sLog.notice(f"Would have created {len(self.requests)} requests") for reqID, req in enumerate(self.requests): sLog.notice("Request %d:" % reqID) for opID, op in enumerate(req): @@ -264,14 +264,14 @@ def putOrRunRequests(self): for request in self.requests: putRequest = self.reqClient.putRequest(request) if not putRequest["OK"]: - sLog.error("unable to put request {!r}: {}".format(request.RequestName, putRequest["Message"])) + sLog.error(f"unable to put request {request.RequestName!r}: {putRequest['Message']}") continue requestIDs.append(str(putRequest["Value"])) - sLog.always("Request %r has been put to ReqDB for execution." % request.RequestName) + sLog.always(f"Request {request.RequestName!r} has been put to ReqDB for execution.") if requestIDs: - sLog.always("%d requests have been put to ReqDB for execution" % len(requestIDs)) - sLog.always("RequestID(s): %s" % " ".join(requestIDs)) + sLog.always(f"{len(requestIDs)} requests have been put to ReqDB for execution") + sLog.always(f"RequestID(s): {' '.join(requestIDs)}") sLog.always("You can monitor the request status using the command: dirac-rms-request ") return 0 diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_removal_request.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_removal_request.py index 76cd5303240..013182c5505 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_removal_request.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_create_removal_request.py @@ -57,7 +57,6 @@ def main(): requestOperation = "RemoveFile" for lfnList in breakListIntoChunks(lfns, 100): - oRequest = Request() requestName = "{}_{}".format( md5(repr(time.time()).encode()).hexdigest()[:16], @@ -71,7 +70,7 @@ def main(): res = fc.getFileMetadata(lfnList) if not res["OK"]: - print("Can't get file metadata: %s" % res["Message"]) + print(f"Can't get file metadata: {res['Message']}") DIRAC.exit(1) if res["Value"]["Failed"]: print("Could not get the file metadata of the following, so skipping them:") diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_data_size.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_data_size.py index 625ecbb29d4..cc959ff4f40 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_data_size.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_data_size.py @@ -19,7 +19,7 @@ @Script() def main(): unit = "GB" - Script.registerSwitch("u:", "Unit=", " Unit to use [default %s] (MB,GB,TB,PB)" % unit) + Script.registerSwitch("u:", "Unit=", f" Unit to use [default {unit}] (MB,GB,TB,PB)") # Registering arguments will automatically add their description to the help menu Script.registerArgument(("LocalFile: Path to local file containing LFNs", "LFN: Logical File Name")) Script.registerArgument(["LFN: Logical File Name"], mandatory=False) @@ -56,16 +56,16 @@ def main(): gLogger.error("Failed to get size of data", res["Message"]) DIRAC.exit(-2) for lfn, reason in res["Value"]["Failed"].items(): - gLogger.error("Failed to get size for %s" % lfn, reason) + gLogger.error(f"Failed to get size for {lfn}", reason) totalSize = 0 totalFiles = 0 for lfn, size in res["Value"]["Successful"].items(): totalFiles += 1 totalSize += size gLogger.notice("-" * 30) - gLogger.notice("{}|{}".format("Files".ljust(15), ("Size (%s)" % unit).rjust(15))) + gLogger.notice(f"{'Files'.ljust(15)}|{('Size (%s)' % unit).rjust(15)}") gLogger.notice("-" * 30) - gLogger.notice("{}|{}".format(str(totalFiles).ljust(15), str("%.1f" % (totalSize / scaleFactor)).rjust(15))) + gLogger.notice(f"{str(totalFiles).ljust(15)}|{str('%.1f' % (totalSize / scaleFactor)).rjust(15)}") gLogger.notice("-" * 30) DIRAC.exit(0) diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_directory_sync.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_directory_sync.py index 55e2acc4d9d..0fac8f143b2 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_directory_sync.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_directory_sync.py @@ -81,7 +81,7 @@ def getSetOfRemoteSubDirectoriesAndFiles(path, fc, directories, files): return S_ERROR("Error: " + res["Message"]) return S_OK() else: - return S_ERROR("Error: %s" % result["Value"]) + return S_ERROR(f"Error: {result['Value']}") else: return S_ERROR("Error:" + result["Message"]) @@ -273,7 +273,7 @@ def doUpload(fc, dm, result, source_dir, dest_dir, storage, delete, nthreads): if len(lfns) > 0: res = removeRemoteFiles(dm, lfns) if not res["OK"]: - gLogger.fatal("Deleting of files: " + lfns + " -X- [FAILED]" + res["Message"]) + gLogger.fatal("Deleting of files: " + str(lfns) + " -X- [FAILED]" + res["Message"]) DIRAC.exit(1) else: gLogger.notice("Deleting " + ", ".join(lfns) + " -> [DONE]") @@ -316,14 +316,14 @@ def uploadListOfFiles(dm, source_dir, dest_dir, storage, listOfFiles, tID): """ Wrapper for multithreaded uploading of a list of files """ - log = gLogger.getLocalSubLogger("[Thread %s] " % tID) - threadLine = "[Thread %s]" % tID + log = gLogger.getLocalSubLogger(f"[Thread {tID}] ") + threadLine = f"[Thread {tID}]" for filename in listOfFiles: destLFN = os.path.join(dest_dir, filename) res = returnSingleResult(dm.putAndRegister(destLFN, source_dir + "/" + filename, storage, None)) if not res["OK"]: log.fatal(threadLine + " Uploading " + filename + " -X- [FAILED] " + res["Message"]) - listOfFailedFiles.append("{}: {}".format(destLFN, res["Message"])) + listOfFailedFiles.append(f"{destLFN}: {res['Message']}") else: log.notice(threadLine + " Uploading " + filename + " -> [DONE]") @@ -397,14 +397,14 @@ def downloadListOfFiles(dm, source_dir, dest_dir, listOfFiles, tID): """ Wrapper for multithreaded downloading of a list of files """ - log = gLogger.getLocalSubLogger("[Thread %s] " % tID) - threadLine = "[Thread %s]" % tID + log = gLogger.getLocalSubLogger(f"[Thread {tID}] ") + threadLine = f"[Thread {tID}]" for filename in listOfFiles: sourceLFN = os.path.join(source_dir, filename) res = returnSingleResult(dm.getFile(sourceLFN, dest_dir + ("/" + filename).rsplit("/", 1)[0])) if not res["OK"]: log.fatal(threadLine + " Downloading " + filename + " -X- [FAILED] " + res["Message"]) - listOfFailedFiles.append("{}: {}".format(sourceLFN, res["Message"])) + listOfFailedFiles.append(f"{sourceLFN}: {res['Message']}") else: log.notice(threadLine + " Downloading " + filename + " -> [DONE]") diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_filecatalog_cli.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_filecatalog_cli.py index 7b042f0924e..9035736f073 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_filecatalog_cli.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_filecatalog_cli.py @@ -20,52 +20,75 @@ FC:/> """ -import sys from DIRAC.Core.Base.Script import Script @Script() def main(): - fcType = "FileCatalog" - Script.registerSwitch("f:", "file-catalog=", " Catalog client type to use (default %s)" % fcType) + fcType = None + catalog = None + Script.registerSwitch("f:", "file-catalog=", f" Catalog to use (default - Catalog defined for the users' VO)") Script.parseCommandLine(ignoreErrors=False) - from DIRAC import gConfig, exit as dexit - from DIRAC.Resources.Catalog.FileCatalogFactory import FileCatalogFactory - - fcType = gConfig.getValue("/LocalSite/FileCatalog", "") - - res = gConfig.getSections("/Resources/FileCatalogs", listOrdered=True) - if not res["OK"]: - dexit(1) - fcList = res["Value"] - if not fcType: - if res["OK"]: - fcType = res["Value"][0] + from DIRAC import exit as dexit for switch in Script.getUnprocessedSwitches(): if switch[0].lower() == "f" or switch[0].lower() == "file-catalog": fcType = switch[1] if not fcType: - print("No file catalog given and defaults could not be obtained") - sys.exit(1) + # A particular catalog is not specified, try to instantiate the catalog container + from DIRAC.Resources.Catalog.FileCatalog import FileCatalog + + catalog = FileCatalog() + if not catalog.valid: + print("Failed to create the FileCatalog container. Try to use a specific catalog with -f option") + dexit(-1) + result = catalog.getMasterCatalogNames() + if not result["OK"]: + print("Failed to get the Master catalog name for the FileCatalog container") + dexit(-1) + masterCatalog = result["Value"][0] + readCatalogs = [c[0] for c in catalog.getReadCatalogs()] + writeCatalogs = [c[0] for c in catalog.getWriteCatalogs()] + allCatalogs = list(set([masterCatalog] + readCatalogs + writeCatalogs)) + + if len(allCatalogs) == 1: + # If we have a single catalog in the container, let's use this catalog directly + fcType = allCatalogs[0] + catalog = None + else: + print( + "\nStarting FileCatalog container console. \n" + "Note that you will access several catalogs at the same time:" + ) + print(f" {masterCatalog} - Master") + for cat in allCatalogs: + if cat != masterCatalog: + cTypes = ["Write"] if cat in writeCatalogs else [] + cTypes.extend(["Read"] if cat in readCatalogs else []) + print(f" {cat} - {'-'.join(cTypes)}") + print("If you want to work with a single catalog, specify it with the -f option\n") + + if fcType: + # We have to use a specific File Catalog, create it now + from DIRAC.Resources.Catalog.FileCatalogFactory import FileCatalogFactory + + result = FileCatalogFactory().createCatalog(fcType) + if not result["OK"]: + print(result["Message"]) + dexit(-1) + catalog = result["Value"] + print(f"Starting {fcType} console") from DIRAC.DataManagementSystem.Client.FileCatalogClientCLI import FileCatalogClientCLI - result = FileCatalogFactory().createCatalog(fcType) - if not result["OK"]: - print(result["Message"]) - if fcList: - print("Possible choices are:") - for fc in fcList: - print(" " * 5, fc) - sys.exit(1) - print("Starting %s client" % fcType) - catalog = result["Value"] - cli = FileCatalogClientCLI(catalog) - cli.cmdloop() + if catalog: + cli = FileCatalogClientCLI(catalog) + cli.cmdloop() + else: + print(f"Failed to access the catalog {fcType if fcType else 'container'}") if __name__ == "__main__": diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_find_lfns.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_find_lfns.py index 844eea2f636..877e7b0a452 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_find_lfns.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_find_lfns.py @@ -11,7 +11,7 @@ @Script() def main(): - Script.registerSwitch("", "Path=", " Path to search for") + Script.registerSwitch("", "Path=", " Directory path to search for") Script.registerSwitch("", "SE=", " (comma-separated list of) SEs/SE-groups to be searched") # Registering arguments will automatically add their description to the help menu Script.registerArgument( @@ -36,7 +36,7 @@ def main(): seList = resolveSEGroup(val.split(",")) if seList: - args.append("SE=%s" % ",".join(seList)) + args.append(f"SE={','.join(seList)}") fc = FileCatalog() result = fc.getMetadataFields() if not result["OK"]: @@ -48,8 +48,8 @@ def main(): typeDict.update(FILE_STANDARD_METAKEYS) if len(args) < 1: - print("Error: No argument provided\n%s:" % Script.scriptName) - gLogger.notice("MetaDataDictionary: \n%s" % str(typeDict)) + print(f"Error: No argument provided\n{Script.scriptName}:") + gLogger.notice(f"MetaDataDictionary: \n{str(typeDict)}") Script.showHelp(exitCode=1) mq = MetaQuery(typeDict=typeDict) @@ -59,7 +59,17 @@ def main(): DIRAC.exit(-1) metaDict = result["Value"] path = metaDict.pop("Path", path) - + # check if path exists and is a directory + result = fc.isDirectory(path) + if not result["OK"]: + gLogger.error("Can not access File Catalog:", result["Message"]) + DIRAC.exit(-1) + if path not in result["Value"]["Successful"]: + gLogger.error("Failed to query path status in file catalogue.", result["Message"]) + DIRAC.exit(-1) + if not result["Value"]["Successful"][path]: + gLogger.error(f"{path} does not exist or is not a directory.") + DIRAC.exit(-1) result = fc.findFilesByMetadata(metaDict, path) if not result["OK"]: gLogger.error("Can not access File Catalog:", result["Message"]) diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_move_replica_request.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_move_replica_request.py index 3b15647d981..7cfc3d25650 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_move_replica_request.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_move_replica_request.py @@ -59,7 +59,7 @@ def main(): for lfnChunk in lfnChunks: metaDatas = fc.getFileMetadata(lfnChunk) if not metaDatas["OK"]: - gLogger.error("unable to read metadata for lfns: %s" % metaDatas["Message"]) + gLogger.error(f"unable to read metadata for lfns: {metaDatas['Message']}") error = -1 continue metaDatas = metaDatas["Value"] @@ -73,7 +73,7 @@ def main(): continue if len(lfnChunk) > Operation.MAX_FILES: - gLogger.error("too many LFNs, max number of files per operation is %s" % Operation.MAX_FILES) + gLogger.error(f"too many LFNs, max number of files per operation is {Operation.MAX_FILES}") error = -1 continue @@ -106,7 +106,7 @@ def main(): result = reqClient.putRequest(request) if not result["OK"]: - gLogger.error("Failed to submit Request: %s" % (result["Message"])) + gLogger.error(f"Failed to submit Request: {result['Message']}") error = -1 continue diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_protocol_matrix.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_protocol_matrix.py index 48baf1713b2..904bc3ed93e 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_protocol_matrix.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_protocol_matrix.py @@ -116,7 +116,7 @@ def main(): # for each baseSE an entry corresponding to one real storage (the first one) # and itself for each real non inherited SE for se in allSEs: - baseSE = gConfig.getOption("/Resources/StorageElements/%s/BaseSE" % se).get("Value") + baseSE = gConfig.getOption(f"/Resources/StorageElements/{se}/BaseSE").get("Value") if baseSE and not fullOutput: if baseSE not in seForSeBases: seForSeBases[baseSE] = se @@ -131,7 +131,6 @@ def main(): fromSE = list(seForSeBases) targetSE = list(seForSeBases) else: # he specified at least source of dest - # if bidirection, source and target should be the same if bidirection: if not fromSE and targetSE: # we gave target, but no source @@ -151,8 +150,8 @@ def main(): fromSE = sorted(fromSE) targetSE = sorted(targetSE) - gLogger.notice("Using sources: %s" % ",".join(fromSE)) - gLogger.notice("Using target: %s" % ",".join(targetSE)) + gLogger.notice(f"Using sources: {','.join(fromSE)}") + gLogger.notice(f"Using target: {','.join(targetSE)}") # Now we construct the SE object for each SE that we want to appear ses = {} @@ -166,7 +165,7 @@ def main(): vo = ret["Value"] gLogger.notice("Using the Virtual Organization:", vo) # dummy LFN, still has to follow lfn convention - lfn = "/%s/toto.xml" % vo + lfn = f"/{vo}/toto.xml" # Create a matrix of protocol src/dest @@ -176,7 +175,6 @@ def main(): # For each source and destination, generate the url pair, and the compatible third party protocols for src, dst in ((x, y) for x in fromSE for y in targetSE): - if ftsTab: try: # breakpoint() @@ -186,20 +184,18 @@ def main(): res = S_ERROR(str(e)) if not res["OK"]: surls = "Error" - gLogger.notice( - "Could not generate transfer URLS", "src:{}, dst:{}, error:{}".format(src, dst, res["Message"]) - ) + gLogger.notice("Could not generate transfer URLS", f"src:{src}, dst:{dst}, error:{res['Message']}") else: # We only keep the protocol part of the url surls = "/".join(res["Value"]["Protocols"]) - ftsMatrix[src][dst] = "%s" % surls + ftsMatrix[src][dst] = f"{surls}" gLogger.verbose(f"{src} -> {dst}: {surls}") # Add also the third party protocols if tpcTab: proto = ",".join(ses[dst].negociateProtocolWithOtherSE(ses[src], thirdPartyProtocols)["Value"]) - tpMatrix[src][dst] = "%s" % proto + tpMatrix[src][dst] = f"{proto}" gLogger.verbose(f"{src} -> {dst}: {proto}") diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_put_and_register_request.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_put_and_register_request.py index c2ed212461d..dc70c542fce 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_put_and_register_request.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_put_and_register_request.py @@ -41,10 +41,10 @@ def main(): from DIRAC.Core.Utilities.Adler import fileAdler if not os.path.exists(PFN): - gLogger.error("%s does not exist" % PFN) + gLogger.error(f"{PFN} does not exist") DIRAC.exit(-1) if not os.path.isfile(PFN): - gLogger.error("%s is not a file" % PFN) + gLogger.error(f"{PFN} is not a file") DIRAC.exit(-1) PFN = os.path.abspath(PFN) @@ -68,11 +68,11 @@ def main(): reqClient = ReqClient() putRequest = reqClient.putRequest(request) if not putRequest["OK"]: - gLogger.error("unable to put request '{}': {}".format(requestName, putRequest["Message"])) + gLogger.error(f"unable to put request '{requestName}': {putRequest['Message']}") DIRAC.exit(-1) - gLogger.always("Request '%s' has been put to ReqDB for execution." % requestName) - gLogger.always("You can monitor its status using command: 'dirac-rms-request %s'" % requestName) + gLogger.always(f"Request '{requestName}' has been put to ReqDB for execution.") + gLogger.always(f"You can monitor its status using command: 'dirac-rms-request {requestName}'") DIRAC.exit(0) diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_catalog_files.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_catalog_files.py index ac6f814d9bd..380d5074bd1 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_catalog_files.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_catalog_files.py @@ -62,7 +62,7 @@ def main(): for lfn in sorted(res["Value"]["Failed"].keys()): message = res["Value"]["Failed"][lfn] print(f"Error: failed to remove {lfn}: {message}") - print("Successfully removed %d catalog files." % (len(res["Value"]["Successful"]))) + print(f"Successfully removed {len(res['Value']['Successful'])} catalog files.") if __name__ == "__main__": diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_catalog_replicas.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_catalog_replicas.py index 9d2d79a2d8b..0edc97fd467 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_catalog_replicas.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_catalog_replicas.py @@ -56,7 +56,7 @@ def main(): for lfn in sorted(res["Value"]["Failed"]): message = res["Value"]["Failed"][lfn] print(f"Failed to remove {storageElementName} replica of {lfn}: {message}") - print("Successfully remove %d catalog replicas at %s" % (len(res["Value"]["Successful"]), storageElementName)) + print(f"Successfully remove {len(res['Value']['Successful'])} catalog replicas at {storageElementName}") if __name__ == "__main__": diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_files.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_files.py index 68ebca71260..5f7cecfd172 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_files.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_remove_files.py @@ -47,7 +47,7 @@ def main(): successfullyRemoved += len(res["Value"]["Successful"]) for reason, lfns in errorReasons.items(): - gLogger.notice("Failed to remove %d files with error: %s" % (len(lfns), reason)) + gLogger.notice(f"Failed to remove {len(lfns)} files with error: {reason}") if successfullyRemoved > 0: gLogger.notice("Successfully removed %d files" % successfullyRemoved) DIRAC.exit(0) diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_replica_metadata.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_replica_metadata.py index 0022ab17931..4a80e7d3ff2 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_replica_metadata.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_replica_metadata.py @@ -34,7 +34,7 @@ def main(): print("Error:", res["Message"]) DIRACExit(1) - print("{} {} {} {}".format("File".ljust(100), "Migrated".ljust(8), "Cached".ljust(8), "Size (bytes)".ljust(10))) + print(f"{'File'.ljust(100)} {'Migrated'.ljust(8)} {'Cached'.ljust(8)} {'Size (bytes)'.ljust(10)}") for lfn, metadata in res["Value"]["Successful"].items(): print( "%s %s %s %s" diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_replicate_and_register_request.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_replicate_and_register_request.py index 334a7354b2b..82bbf151897 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_replicate_and_register_request.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_replicate_and_register_request.py @@ -22,15 +22,19 @@ def getLFNList(arg): def main(): catalog = None Script.registerSwitch("C:", "Catalog=", "Catalog to use") + Script.registerSwitch("N:", "ChunkSize=", "Number of files per request") # Registering arguments will automatically add their description to the help menu Script.registerArgument(" requestName: a request name") Script.registerArgument(" LFNs: single LFN or file with LFNs") Script.registerArgument(["targetSE: target SE"]) Script.parseCommandLine() + + chunksize = 100 for switch in Script.getUnprocessedSwitches(): if switch[0] == "C" or switch[0].lower() == "catalog": catalog = switch[1] - + if switch[0] == "N" or switch[0].lower() == "chunksize": + chunksize = int(switch[1]) args = Script.getPositionalArgs() requestName = None @@ -54,7 +58,7 @@ def main(): from DIRAC.Resources.Catalog.FileCatalog import FileCatalog from DIRAC.Core.Utilities.List import breakListIntoChunks - lfnChunks = breakListIntoChunks(lfnList, 100) + lfnChunks = breakListIntoChunks(lfnList, chunksize) multiRequests = len(lfnChunks) > 1 error = 0 @@ -65,7 +69,7 @@ def main(): for lfnChunk in lfnChunks: metaDatas = fc.getFileMetadata(lfnChunk) if not metaDatas["OK"]: - gLogger.error("unable to read metadata for lfns: %s" % metaDatas["Message"]) + gLogger.error(f"unable to read metadata for lfns: {metaDatas['Message']}") error = -1 continue metaDatas = metaDatas["Value"] @@ -79,7 +83,7 @@ def main(): continue if len(lfnChunk) > Operation.MAX_FILES: - gLogger.error("too many LFNs, max number of files per operation is %s" % Operation.MAX_FILES) + gLogger.error(f"too many LFNs, max number of files per operation is {Operation.MAX_FILES}") error = -1 continue @@ -109,17 +113,17 @@ def main(): putRequest = reqClient.putRequest(request) if not putRequest["OK"]: - gLogger.error("unable to put request '{}': {}".format(request.RequestName, putRequest["Message"])) + gLogger.error(f"unable to put request '{request.RequestName}': {putRequest['Message']}") error = -1 continue requestIDs.append(str(putRequest["Value"])) if not multiRequests: - gLogger.always("Request '%s' has been put to ReqDB for execution." % request.RequestName) + gLogger.always(f"Request '{request.RequestName}' has been put to ReqDB for execution.") if multiRequests: gLogger.always("%d requests have been put to ReqDB for execution, with name %s_" % (count, requestName)) if requestIDs: - gLogger.always("RequestID(s): %s" % " ".join(requestIDs)) + gLogger.always(f"RequestID(s): {' '.join(requestIDs)}") gLogger.always("You can monitor requests' status using command: 'dirac-rms-request '") DIRAC.exit(error) diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_set_replica_status.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_set_replica_status.py index 4b2e5f3456a..6b9814322f6 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_set_replica_status.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_set_replica_status.py @@ -54,7 +54,7 @@ def main(): for lfn, error in res["Value"]["Failed"].items(): gLogger.error("Failed to set replica status for file.", f"{lfn}:{error}") gLogger.notice( - "Successfully updated the status of %d files at %s." % (len(res["Value"]["Successful"].keys()), storageElement) + f"Successfully updated the status of {len(res['Value']['Successful'].keys())} files at {storageElement}." ) DIRAC.exit(0) diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_show_se_status.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_show_se_status.py index ca284d1200c..fa475bedde2 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_show_se_status.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_show_se_status.py @@ -71,7 +71,7 @@ def main(): gLogger.error(res["Message"]) DIRACexit(1) - gLogger.info("{} {} {}".format("Storage Element".ljust(25), "Read Status".rjust(15), "Write Status".rjust(15))) + gLogger.info(f"{'Storage Element'.ljust(25)} {'Read Status'.rjust(15)} {'Write Status'.rjust(15)}") seList = sorted(res["Value"]) @@ -79,7 +79,7 @@ def main(): res = resourceStatus.getElementStatus(seList, "StorageElement") if not res["OK"]: - gLogger.error("Failed to get StorageElement status for %s" % str(seList)) + gLogger.error(f"Failed to get StorageElement status for {str(seList)}") DIRACexit(1) fields = ["SE", "ReadAccess", "WriteAccess", "RemoveAccess", "CheckAccess"] @@ -93,10 +93,9 @@ def main(): vo = result["Value"] for se, statusDict in res["Value"].items(): - # Check if the SE is allowed for the user VO if not allVOsFlag: - voList = gConfig.getValue("/Resources/StorageElements/%s/VO" % se, []) + voList = gConfig.getValue(f"/Resources/StorageElements/{se}/VO", []) if noVOFlag and voList: continue if voList and vo not in voList: diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_user_lfns.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_user_lfns.py index 918182d53a9..b6bb517b083 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_user_lfns.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_user_lfns.py @@ -15,6 +15,10 @@ /formation/user/v/vhamar/0/20: 1 files, 0 sub-directories 16 matched files have been put in formation-user-v-vhamar.lfns """ + +import fnmatch +from datetime import datetime, timedelta + from DIRAC.Core.Base.Script import Script @@ -26,9 +30,9 @@ def main(): wildcard = None baseDir = "" emptyDirsFlag = False - Script.registerSwitch("D:", "Days=", "Match files older than number of days [%s]" % days) - Script.registerSwitch("M:", "Months=", "Match files older than number of months [%s]" % months) - Script.registerSwitch("Y:", "Years=", "Match files older than number of years [%s]" % years) + Script.registerSwitch("D:", "Days=", f"Match files older than number of days [{days}]") + Script.registerSwitch("M:", "Months=", f"Match files older than number of months [{months}]") + Script.registerSwitch("Y:", "Years=", f"Match files older than number of years [{years}]") Script.registerSwitch("w:", "Wildcard=", "Wildcard for matching filenames [All]") Script.registerSwitch("b:", "BaseDir=", "Base directory to begin search (default /[vo]/user/[initial]/[username])") Script.registerSwitch("e", "EmptyDirs", "Create a list of empty directories") @@ -54,11 +58,6 @@ def main(): from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup from DIRAC.Core.Security.ProxyInfo import getProxyInfo from DIRAC.Resources.Catalog.FileCatalog import FileCatalog - from datetime import datetime, timedelta - import sys - import os - import time - import fnmatch fc = FileCatalog() @@ -100,53 +99,47 @@ def isOlderThan(cTimeStruct, days): baseDir = baseDir.rstrip("/") - gLogger.notice("Will search for files in {}{}".format(baseDir, (" matching %s" % wildcard) if wildcard else "")) - activeDirs = [baseDir] + gLogger.notice(f"Will search for files in {baseDir}{f' matching {wildcard}' if wildcard else ''}") allFiles = [] emptyDirs = [] - while len(activeDirs) > 0: - currentDir = activeDirs.pop() - res = fc.listDirectory(currentDir, withMetadata, timeout=360) - if not res["OK"]: - gLogger.error("Error retrieving directory contents", "{} {}".format(currentDir, res["Message"])) - elif currentDir in res["Value"]["Failed"]: - gLogger.error( - "Error retrieving directory contents", "{} {}".format(currentDir, res["Value"]["Failed"][currentDir]) - ) + res = fc.getDirectoryDump(baseDir, timeout=360) + if not res["OK"]: + gLogger.error("Error retrieving directory contents", f"{baseDir} {res['Message']}") + DIRAC.exit(1) + elif baseDir in res["Value"]["Failed"]: + gLogger.error("Error retrieving directory contents", f"{baseDir} {res['Value']['Failed'][baseDir]}") + DIRAC.exit(1) + else: + dirContents = res["Value"]["Successful"][baseDir] + subdirs = dirContents["SubDirs"] + files = dirContents["Files"] + if not subdirs and not files: + emptyDirs.append(baseDir) + gLogger.notice(f"{baseDir}: empty directory") else: - dirContents = res["Value"]["Successful"][currentDir] - subdirs = dirContents["SubDirs"] - files = dirContents["Files"] - if not subdirs and not files: - emptyDirs.append(currentDir) - gLogger.notice("%s: empty directory" % currentDir) - else: - for subdir in sorted(subdirs, reverse=True): - if (not withMetadata) or isOlderThan(subdirs[subdir]["CreationDate"], totalDays): - activeDirs.append(subdir) - for filename in sorted(files): - fileOK = False - if (not withMetadata) or isOlderThan(files[filename]["MetaData"]["CreationDate"], totalDays): - if wildcard is None or fnmatch.fnmatch(filename, wildcard): - fileOK = True - if not fileOK: - files.pop(filename) - allFiles += sorted(files) - - if len(files) or len(subdirs): - gLogger.notice( - "%s: %d files%s, %d sub-directories" - % (currentDir, len(files), " matching" if withMetadata or wildcard else "", len(subdirs)) - ) + for filename in sorted(files): + fileOK = False + if (not withMetadata) or isOlderThan(files[filename]["CreationDate"], totalDays): + if wildcard is None or fnmatch.fnmatch(filename, wildcard): + fileOK = True + if not fileOK: + files.pop(filename) + allFiles += sorted(files) + + if len(files) or len(subdirs): + gLogger.notice( + "%s: %d files%s, %d sub-directories" + % (baseDir, len(files), " matching" if withMetadata or wildcard else "", len(subdirs)) + ) outputFileName = "%s.lfns" % baseDir.replace("/%s" % vo, "%s" % vo).replace("/", "-") outputFile = open(outputFileName, "w") for lfn in sorted(allFiles): outputFile.write(lfn + "\n") outputFile.close() - gLogger.notice("%d matched files have been put in %s" % (len(allFiles), outputFileName)) + gLogger.notice(f"{len(allFiles)} matched files have been put in {outputFileName}") if emptyDirsFlag: outputFileName = "%s.emptydirs" % baseDir.replace("/%s" % vo, "%s" % vo).replace("/", "-") @@ -154,7 +147,7 @@ def isOlderThan(cTimeStruct, days): for dir in sorted(emptyDirs): outputFile.write(dir + "\n") outputFile.close() - gLogger.notice("%d empty directories have been put in %s" % (len(emptyDirs), outputFileName)) + gLogger.notice(f"{len(emptyDirs)} empty directories have been put in {outputFileName}") DIRAC.exit(0) diff --git a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_user_quota.py b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_user_quota.py index a89db1732a4..1b5e6ae2d13 100755 --- a/src/DIRAC/DataManagementSystem/scripts/dirac_dms_user_quota.py +++ b/src/DIRAC/DataManagementSystem/scripts/dirac_dms_user_quota.py @@ -29,8 +29,8 @@ def main(): try: quota = gConfig.getValue("/Registry/DefaultStorageQuota", 0.0) - quota = gConfig.getValue("/Registry/Users/%s/Quota" % username, quota) - gLogger.notice("Current quota found to be %.1f GB" % quota) + quota = gConfig.getValue(f"/Registry/Users/{username}/Quota", quota) + gLogger.notice(f"Current quota found to be {quota:.1f} GB") DIRAC.exit(0) except Exception as x: gLogger.exception("Failed to convert retrieved quota", "", x) diff --git a/src/DIRAC/FrameworkSystem/API/AuthHandler.py b/src/DIRAC/FrameworkSystem/API/AuthHandler.py index 1bef460b7e9..3d7710a3daa 100644 --- a/src/DIRAC/FrameworkSystem/API/AuthHandler.py +++ b/src/DIRAC/FrameworkSystem/API/AuthHandler.py @@ -80,10 +80,7 @@ def get_index(self): } """ if self.request.method == "GET": - resDict = dict( - setups=gConfig.getSections("DIRAC/Setups").get("Value", []), - configuration_server=gConfig.getValue("/DIRAC/Configuration/MasterServer", ""), - ) + resDict = {"configuration_server": gConfig.getValue("/DIRAC/Configuration/MasterServer", "")} resDict.update(self.server.metadata) resDict.pop("Clients", None) return resDict @@ -239,7 +236,7 @@ def get_device(self, provider=None, user_code=None, client_id=None): "session is expired.", theme="warning", body=result.get("Message"), - info="Seems device code flow authorization session %s expired." % user_code, + info=f"Seems device code flow authorization session {user_code} expired.", ) session = result["Value"] # Get original request from session @@ -247,7 +244,7 @@ def get_device(self, provider=None, user_code=None, client_id=None): req.setQueryArguments(id=session["id"], user_code=user_code) # Save session to cookie and redirect to authorization endpoint - authURL = "{}?{}".format(req.path.replace("device", "authorization"), req.query) + authURL = f"{req.path.replace('device', 'authorization')}?{req.query}" return self.server.handle_response(302, {}, [("Location", authURL)], session) # If received a request without a user code, then send a form to enter the user code @@ -331,7 +328,7 @@ def get_redirect(self, state, error=None, error_description="", chooseScope=[]): "session is expired.", theme="warning", state=400, - info="Seems %s session is expired, please, try again." % state, + info=f"Seems {state} session is expired, please, try again.", ), delSession=True, ) @@ -343,7 +340,7 @@ def get_redirect(self, state, error=None, error_description="", chooseScope=[]): "session is expired.", theme="warning", state=400, - info="Seems %s session is expired, please, try again." % state, + info=f"Seems {state} session is expired, please, try again.", ), delSession=True, ) @@ -363,7 +360,7 @@ def get_redirect(self, state, error=None, error_description="", chooseScope=[]): if not sessionWithExtIdP.get("authed"): # Parse result of the second authentication flow - self.log.info("%s session, parsing authorization response:\n" % state, self.request.uri) + self.log.info(f"{state} session, parsing authorization response:\n", self.request.uri) result = self.server.parseIdPAuthorizationResponse(self.request, sessionWithExtIdP) if not result["OK"]: @@ -420,12 +417,12 @@ def post_token(self): return self.server.create_token_response(self.request) def __researchDIRACGroup(self, extSession, chooseScope, state): - """Research DIRAC groups for authorized user + """Look for DIRAC groups of a user already authorized by an Identity Provider :param dict extSession: ended authorized external IdP session :return: -- will return (None, response) to provide error or group selector - will return (grant_user, request) to contionue authorization with choosed group + will return (grant_user, request) to continue authorization with chosen group """ # Base DIRAC client auth session firstRequest = createOAuth2Request(extSession["firstRequest"]) @@ -435,9 +432,13 @@ def __researchDIRACGroup(self, extSession, chooseScope, state): username = extSession["authed"]["username"] # Requested arguments in first request provider = firstRequest.provider - self.log.debug("Next groups has been found for %s:" % username, ", ".join(firstRequest.groups)) + self.log.debug("The following groups found", f"for {username}: {', '.join(firstRequest.groups)}") + + # If group is already defined in the first request, just return it as it was already validated + if firstRequest.groups: + return extSession["authed"], firstRequest - # Researche Group + # Look for DIRAC groups valid for the user result = getGroupsForUser(username) if not result["OK"]: return None, self.server.handle_response( @@ -458,18 +459,14 @@ def __researchDIRACGroup(self, extSession, chooseScope, state): delSession=True, ) - self.log.debug("The state of %s user groups has been checked:" % username, pprint.pformat(validGroups)) - - # If group already defined in first request, just return it - if firstRequest.groups: - return extSession["authed"], firstRequest + self.log.debug(f"The state of {username} user groups has been checked:", pprint.pformat(validGroups)) # If not and we found only one valid group, apply this group if len(validGroups) == 1: - firstRequest.addScopes(["g:%s" % validGroups[0]]) + firstRequest.addScopes([f"g:{validGroups[0]}"]) return extSession["authed"], firstRequest - # Else give user chanse to choose group in browser + # Else give user a chance to choose a group in the browser with dom.div(cls="row mt-5 justify-content-md-center align-items-center") as tag: for group in sorted(validGroups): vo, gr = group.split("_") diff --git a/src/DIRAC/FrameworkSystem/Agent/CAUpdateAgent.py b/src/DIRAC/FrameworkSystem/Agent/CAUpdateAgent.py deleted file mode 100644 index 8436c2c3151..00000000000 --- a/src/DIRAC/FrameworkSystem/Agent/CAUpdateAgent.py +++ /dev/null @@ -1,35 +0,0 @@ -""" CAUpdateAgent is meant to be used in a multi-server installations - where one server has some machinery of keeping up to date the CA's data - and other servers are just synchronized with the master one without "official" CA installations locally. -""" -from DIRAC import S_OK -from DIRAC.Core.Base.AgentModule import AgentModule -from DIRAC.FrameworkSystem.Client.BundleDeliveryClient import BundleDeliveryClient - - -class CAUpdateAgent(AgentModule): - """just routinely calls BundleDeliveryClient.syncCAs()/syncCRLs()""" - - def initialize(self): - self.am_setOption("PollingTime", 3600 * 6) - return S_OK() - - def execute(self): - """The main agent execution method""" - bdc = BundleDeliveryClient() - result = bdc.syncCAs() - if not result["OK"]: - self.log.error("Error while updating CAs", result["Message"]) - elif result["Value"]: - self.log.info("CAs got updated") - else: - self.log.info("CAs are already synchronized") - result = bdc.syncCRLs() - if not result["OK"]: - self.log.error("Error while updating CRLs", result["Message"]) - elif result["Value"]: - self.log.info("CRLs got updated") - else: - self.log.info("CRLs are already synchronized") - - return S_OK() diff --git a/src/DIRAC/FrameworkSystem/Agent/ComponentSupervisionAgent.py b/src/DIRAC/FrameworkSystem/Agent/ComponentSupervisionAgent.py index 78b7b96e739..83eb1bde83b 100644 --- a/src/DIRAC/FrameworkSystem/Agent/ComponentSupervisionAgent.py +++ b/src/DIRAC/FrameworkSystem/Agent/ComponentSupervisionAgent.py @@ -89,7 +89,6 @@ def __init__(self, *args, **kwargs): """Initialize the agent, clients, default values.""" AgentModule.__init__(self, *args, **kwargs) self.name = "ComponentSupervisionAgent" - self.setup = "DIRAC-Production" self.enabled = False self.restartAgents = False self.restartExecutors = False @@ -112,7 +111,7 @@ def __init__(self, *args, **kwargs): self.addressTo = [] self.addressFrom = "" - self.emailSubject = "ComponentSupervisionAgent on %s" % socket.getfqdn() + self.emailSubject = f"ComponentSupervisionAgent on {socket.getfqdn()}" def logError(self, errStr, varMsg=""): """Append errors to a list, which is sent in email notification.""" @@ -121,7 +120,6 @@ def logError(self, errStr, varMsg=""): def beginExecution(self): """Reload the configurations before every cycle.""" - self.setup = self.am_getOption("Setup", self.setup) self.enabled = self.am_getOption("EnableFlag", self.enabled) self.restartAgents = self.am_getOption("RestartAgents", self.restartAgents) self.restartExecutors = self.am_getOption("RestartExecutors", self.restartExecutors) @@ -197,30 +195,31 @@ def getRunningInstances(self, instanceType="Agents", runitStatus="Run"): """ res = self.sysAdminClient.getOverallStatus() if not res["OK"]: - self.logError("Failure to get %s from system administrator client" % instanceType, res["Message"]) + self.logError(f"Failure to get {instanceType} from system administrator client", res["Message"]) return res val = res["Value"][instanceType] runningComponents = defaultdict(dict) for system, components in val.items(): for componentName, componentInfo in components.items(): - if componentInfo["Setup"] and componentInfo["Installed"]: + fullName = f"{system}__{componentName}" + if componentInfo["Installed"]: if runitStatus != "All" and componentInfo["RunitStatus"] != runitStatus: continue for option, default in (("PollingTime", HOUR), ("Port", None), ("Protocol", None)): - runningComponents[componentName][option] = self._getComponentOption( + runningComponents[fullName][option] = self._getComponentOption( instanceType, system, componentName, option, default ) # remove empty values so we can use defaults in _getURL - if not runningComponents[componentName][option]: - runningComponents[componentName].pop(option) - runningComponents[componentName]["LogFileLocation"] = os.path.join( + if not runningComponents[fullName][option]: + runningComponents[fullName].pop(option) + runningComponents[fullName]["LogFileLocation"] = os.path.join( self.diracLocation, "runit", system, componentName, "log", "current" ) - runningComponents[componentName]["PID"] = componentInfo["PID"] - runningComponents[componentName]["Module"] = componentInfo["Module"] - runningComponents[componentName]["RunitStatus"] = componentInfo["RunitStatus"] - runningComponents[componentName]["System"] = system + runningComponents[fullName]["PID"] = componentInfo["PID"] + runningComponents[fullName]["Module"] = componentInfo["Module"] + runningComponents[fullName]["RunitStatus"] = componentInfo["RunitStatus"] + runningComponents[fullName]["System"] = system return S_OK(runningComponents) @@ -229,7 +228,6 @@ def _getComponentOption(self, instanceType, system, componentName, option, defau componentPath = PathFinder.getComponentSection( system=system, component=componentName, - setup=self.setup, componentCategory=instanceType, ) if instanceType != "Agents": @@ -251,7 +249,7 @@ def execute(self): # call checkAgent, checkExecutor, checkService res = getattr(self, "check" + instanceType.capitalize())(name, options) if not res["OK"]: - self.logError("Failure when checking %s" % instanceType, "{}, {}".format(name, res["Message"])) + self.logError(f"Failure when checking {instanceType}", f"{name}, {res['Message']}") res = self.componentControl() if not res["OK"]: @@ -289,12 +287,12 @@ def getLastAccessTime(logFileLocation): def restartInstance(self, pid, instanceName, enabled): """Kill a process which is then restarted automatically.""" if not (self.enabled and enabled): - self.log.info("Restarting is disabled, please restart %s manually" % instanceName) + self.log.info(f"Restarting is disabled, please restart {instanceName} manually") self.accounting[instanceName]["Treatment"] = "Please restart it manually" return S_OK(NO_RESTART) if any(pattern in instanceName for pattern in self.doNotRestartInstancePattern): - self.log.info("Restarting for %s is disabled, please restart it manually" % instanceName) + self.log.info(f"Restarting for {instanceName} is disabled, please restart it manually") self.accounting[instanceName]["Treatment"] = "Please restart it manually" return S_OK(NO_RESTART) @@ -310,13 +308,13 @@ def restartInstance(self, pid, instanceName, enabled): processesToTerminate, timeout=5, callback=partial(self.on_terminate, instanceName) ) for proc in alive: - self.log.info("Forcefully killing process %s" % proc.pid) + self.log.info(f"Forcefully killing process {proc.pid}") proc.kill() return S_OK() except psutil.Error as err: - self.logError("Exception occurred in terminating processes", "%s" % err) + self.logError("Exception occurred in terminating processes", f"{err}") return S_ERROR() def checkService(self, serviceName, options): @@ -327,22 +325,22 @@ def checkService(self, serviceName, options): self.log.info("Pinging service", url) pingRes = Client().ping(url=url) if not pingRes["OK"]: - self.log.warn("Failure pinging service", ": {}: {}".format(url, pingRes["Message"])) + self.log.warn("Failure pinging service", f": {url}: {pingRes['Message']}") res = self.restartInstance(int(options["PID"]), serviceName, self.restartServices) if not res["OK"]: return res if res["Value"] != NO_RESTART: self.accounting[serviceName]["Treatment"] = "Successfully Restarted" - self.log.info("Service %s has been successfully restarted" % serviceName) + self.log.info(f"Service {serviceName} has been successfully restarted") self.log.info("Service responded OK") return S_OK() def checkAgent(self, agentName, options): """Check the age of agent's log file, if it is too old then restart the agent.""" pollingTime, currentLogLocation, pid = (options["PollingTime"], options["LogFileLocation"], options["PID"]) - self.log.info("Checking Agent: %s" % agentName) - self.log.info("Polling Time: %s" % pollingTime) - self.log.info("Current Log File location: %s" % currentLogLocation) + self.log.info(f"Checking Agent: {agentName}") + self.log.info(f"Polling Time: {pollingTime}") + self.log.info(f"Current Log File location: {currentLogLocation}") res = self.getLastAccessTime(currentLogLocation) if not res["OK"]: @@ -355,7 +353,7 @@ def checkAgent(self, agentName, options): if age.seconds < maxLogAge: return S_OK() - self.log.info("Current log file is too old for Agent %s" % agentName) + self.log.info(f"Current log file is too old for Agent {agentName}") self.accounting[agentName]["LogAge"] = age.seconds / MINUTES res = self.restartInstance(int(pid), agentName, self.restartAgents) @@ -363,7 +361,7 @@ def checkAgent(self, agentName, options): return res if res["Value"] != NO_RESTART: self.accounting[agentName]["Treatment"] = "Successfully Restarted" - self.log.info("Agent %s has been successfully restarted" % agentName) + self.log.info(f"Agent {agentName} has been successfully restarted") return S_OK() @@ -371,8 +369,8 @@ def checkExecutor(self, executor, options): """Check the age of executor log file, if too old check for jobs in checking status, then restart the executors.""" currentLogLocation = options["LogFileLocation"] pid = options["PID"] - self.log.info("Checking executor: %s" % executor) - self.log.info("Current Log File location: %s" % currentLogLocation) + self.log.info(f"Checking executor: {executor}") + self.log.info(f"Current Log File location: {currentLogLocation}") res = self.getLastAccessTime(currentLogLocation) if not res["OK"]: @@ -384,7 +382,7 @@ def checkExecutor(self, executor, options): if age.seconds < 2 * HOUR: return S_OK() - self.log.info("Current log file is too old for Executor %s" % executor) + self.log.info(f"Current log file is too old for Executor {executor}") self.accounting[executor]["LogAge"] = age.seconds / MINUTES res = self.checkForCheckingJobs(executor) @@ -399,7 +397,7 @@ def checkExecutor(self, executor, options): return res elif res["OK"] and res["Value"] != NO_RESTART: self.accounting[executor]["Treatment"] = "Successfully Restarted" - self.log.info("Executor %s has been successfully restarted" % executor) + self.log.info(f"Executor {executor} has been successfully restarted") return S_OK() @@ -410,12 +408,12 @@ def checkForCheckingJobs(self, executorName): # returns list of jobs IDs resJobs = self.jobMonClient.getJobs(attrDict) if not resJobs["OK"]: - self.logError("Could not get jobs for this executor", "{}: {}".format(executorName, resJobs["Message"])) + self.logError("Could not get jobs for this executor", f"{executorName}: {resJobs['Message']}") return resJobs if resJobs["Value"]: - self.log.info('Found %d jobs in "Checking" status for %s' % (len(resJobs["Value"]), executorName)) + self.log.info(f"Found {len(resJobs['Value'])} jobs in \"Checking\" status for {executorName}") return S_OK(CHECKING_JOBS) - self.log.info('Found no jobs in "Checking" status for %s' % executorName) + self.log.info(f'Found no jobs in "Checking" status for {executorName}') return S_OK(NO_CHECKING_JOBS) def componentControl(self): @@ -449,7 +447,7 @@ def componentControl(self): self._ensureComponentDown(shouldBe["Down"]) for instance in shouldBe["Unknown"]: - self.logError("Unknown instance", "%r, either uninstall or add to config" % instance) + self.logError("Unknown instance", f"{instance!r}, either uninstall or add to config") return S_OK() @@ -494,12 +492,12 @@ def _getDefaultComponentStatus(self): def _ensureComponentRunning(self, shouldBeRunning): """Ensure the correct components are running.""" for instance in shouldBeRunning: - self.log.info("Starting instance %s" % instance) + self.log.info(f"Starting instance {instance}") system, name = instance.split("__") if self.controlComponents: res = self.sysAdminClient.startComponent(system, name) if not res["OK"]: - self.logError("Failed to start component:", "{}: {}".format(instance, res["Message"])) + self.logError("Failed to start component:", f"{instance}: {res['Message']}") else: self.accounting[instance]["Treatment"] = "Instance was down, started instance" else: @@ -508,12 +506,12 @@ def _ensureComponentRunning(self, shouldBeRunning): def _ensureComponentDown(self, shouldBeDown): """Ensure the correct components are not running.""" for instance in shouldBeDown: - self.log.info("Stopping instance %s" % instance) + self.log.info(f"Stopping instance {instance}") system, name = instance.split("__") if self.controlComponents: res = self.sysAdminClient.stopComponent(system, name) if not res["OK"]: - self.logError("Failed to stop component:", "{}: {}".format(instance, res["Message"])) + self.logError("Failed to stop component:", f"{instance}: {res['Message']}") else: self.accounting[instance]["Treatment"] = "Instance was running, stopped instance" else: @@ -527,12 +525,8 @@ def checkURLs(self): # get port used for https based services try: - tornadoSystemInstance = PathFinder.getSystemInstance( - system="Tornado", - setup=self.setup, - ) self._tornadoPort = gConfig.getValue( - Path.cfgPath("/System/Tornado/", tornadoSystemInstance, "Port"), + Path.cfgPath("/System/Tornado/", "Port"), self._tornadoPort, ) except RuntimeError: @@ -565,10 +559,10 @@ def _checkServiceURL(self, serviceName, options): system = options["System"] module = options["Module"] self.log.info(f"Checking URLs for {system}/{module}") - urlsConfigPath = Path.cfgPath(PathFinder.getSystemURLSection(system=system, setup=self.setup), module) + urlsConfigPath = Path.cfgPath(PathFinder.getSystemURLSection(system=system), module) urls = gConfig.getValue(urlsConfigPath, []) self.log.debug(f"Found configured URLs for {module}: {urls}") - self.log.debug("This URL is %s" % url) + self.log.debug(f"This URL is {url}") runitStatus = options["RunitStatus"] wouldHave = "Would have " if not self.commitURLs else "" if runitStatus == "Run" and url not in urls: @@ -586,6 +580,7 @@ def _checkServiceURL(self, serviceName, options): def _getURL(self, serviceName, options): """Return URL for the service.""" + serviceName = serviceName.rsplit("__")[-1] system = options["System"] port = options.get("Port", self._tornadoPort) host = socket.getfqdn() diff --git a/src/DIRAC/FrameworkSystem/Agent/ProxyRenewalAgent.py b/src/DIRAC/FrameworkSystem/Agent/ProxyRenewalAgent.py index 488d882d658..296ebc21592 100644 --- a/src/DIRAC/FrameworkSystem/Agent/ProxyRenewalAgent.py +++ b/src/DIRAC/FrameworkSystem/Agent/ProxyRenewalAgent.py @@ -6,8 +6,6 @@ :dedent: 2 :caption: ProxyRenewalAgent options """ -import concurrent.futures - from DIRAC import S_OK from DIRAC.Core.Base.AgentModule import AgentModule @@ -18,30 +16,16 @@ class ProxyRenewalAgent(AgentModule): def initialize(self): - requiredLifeTime = self.am_getOption("MinimumLifeTime", 3600) renewedLifeTime = self.am_getOption("RenewedLifeTime", 54000) mailFrom = self.am_getOption("MailFrom", DEFAULT_MAIL_FROM) - self.useMyProxy = self.am_getOption("UseMyProxy", False) - self.proxyDB = ProxyDB(useMyProxy=self.useMyProxy, mailFrom=mailFrom) + self.proxyDB = ProxyDB(mailFrom=mailFrom) self.log.info(f"Minimum Life time : {requiredLifeTime}") self.log.info(f"Life time on renew : {renewedLifeTime}") - if self.useMyProxy: - self.log.info(f"MyProxy server : {self.proxyDB.getMyProxyServer()}") - self.log.info(f"MyProxy max proxy time : {self.proxyDB.getMyProxyMaxLifeTime()}") return S_OK() - def __renewProxyForCredentials(self, userDN, userGroup): - lifeTime = self.am_getOption("RenewedLifeTime", 54000) - self.log.info(f"Renewing for {userDN}@{userGroup} {lifeTime} secs") - res = self.proxyDB.renewFromMyProxy(userDN, userGroup, lifeTime=lifeTime) - if not res["OK"]: - self.log.error("Failed to renew proxy", f"for {userDN}@{userGroup} : {res['Message']}") - else: - self.log.info(f"Renewed proxy for {userDN}@{userGroup}") - def execute(self): """The main agent execution method""" self.log.verbose("Purging expired requests") @@ -51,13 +35,6 @@ def execute(self): else: self.log.info(f"Purged {res['Value']} requests") - self.log.verbose("Purging expired tokens") - res = self.proxyDB.purgeExpiredTokens() - if not res["OK"]: - self.log.error(res["Message"]) - else: - self.log.info(f"Purged {res['Value']} tokens") - self.log.verbose("Purging expired proxies") res = self.proxyDB.purgeExpiredProxies() if not res["OK"]: @@ -70,17 +47,4 @@ def execute(self): if not res["OK"]: self.log.error(res["Message"]) - if self.useMyProxy: - res = self.proxyDB.getCredentialsAboutToExpire(self.am_getOption("MinimumLifeTime", 3600)) - if not res["OK"]: - return res - data = res["Value"] - self.log.info(f"Renewing {len(data)} proxies...") - with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: - futures = [] - for record in data: - userDN = record[0] - userGroup = record[1] - futures.append(executor.submit(self.__renewProxyForCredentials, userDN, userGroup)) - return S_OK() diff --git a/src/DIRAC/FrameworkSystem/Agent/test/Test_ComponentSupervisionAgent.py b/src/DIRAC/FrameworkSystem/Agent/test/Test_ComponentSupervisionAgent.py index 72a593d1cdb..4e5d6744f3a 100644 --- a/src/DIRAC/FrameworkSystem/Agent/test/Test_ComponentSupervisionAgent.py +++ b/src/DIRAC/FrameworkSystem/Agent/test/Test_ComponentSupervisionAgent.py @@ -22,19 +22,6 @@ def clientMock(ret): return clientModuleMock -def mockComponentSection(*_args, **kwargs): - """Mock the PathFinder.getComponentSection to return individual componentSections.""" - system = kwargs.get("system") - component = kwargs.get("component") - return f"/Systems/{system}/Production/Services/{component}" - - -def mockURLSection(*_args, **kwargs): - """Mock the PathFinder.getSystemURLSection to return individual componentSections.""" - system = kwargs.get("system") - return "/Systems/%s/Production/URLs/" % system - - class TestComponentSupervisionAgent(unittest.TestCase): """TestComponentSupervisionAgent class.""" @@ -84,7 +71,6 @@ def test_begin_execution(self): self.restartAgent.accounting["Junk"]["Funk"] = 1 self.restartAgent.am_getOption = MagicMock() getOptionCalls = [ - call("Setup", self.restartAgent.setup), call("EnableFlag", True), call("MailTo", self.restartAgent.addressTo), call("MailFrom", self.restartAgent.addressFrom), @@ -149,7 +135,6 @@ def test_get_running_instances(self): "DataManagement": { "FTS3Agent": { "MEM": "0.3", - "Setup": True, "PID": "18128", "RunitStatus": "Run", "Module": "CleanFTSDBAgent", @@ -163,7 +148,6 @@ def test_get_running_instances(self): "Framework": { "ErrorMessageMonitor": { "MEM": "0.3", - "Setup": True, "PID": "2303", "RunitStatus": "Run", "Module": "ErrorMessageMonitor", @@ -177,7 +161,6 @@ def test_get_running_instances(self): "System": { "Off": { "MEM": "0.3", - "Setup": True, "PID": "---", "RunitStatus": "Down", "Module": "ErrorMessageMonitor", @@ -191,7 +174,6 @@ def test_get_running_instances(self): } } agents["Agents"]["DataManagement"]["FTSAgent"] = { - "Setup": False, "PID": 0, "RunitStatus": "Unknown", "Module": "FTSAgent", @@ -202,10 +184,10 @@ def test_get_running_instances(self): self.restartAgent.sysAdminClient.getOverallStatus.return_value = S_OK(agents) res = self.restartAgent.getRunningInstances(instanceType="Agents") - # only insalled agents with RunitStatus RUN should be returned - self.assertTrue("FTSAgent" not in res["Value"]) - self.assertTrue("FTS3Agent" in res["Value"]) - self.assertTrue("ErrorMessageMonitor" in res["Value"]) + # only installed agents with RunitStatus RUN should be returned + self.assertTrue("DataManagement__FTSAgent" not in res["Value"]) + self.assertTrue("DataManagement__FTS3Agent" in res["Value"]) + self.assertTrue("Framework__ErrorMessageMonitor" in res["Value"]) for agent in res["Value"]: self.assertTrue("PollingTime" in res["Value"][agent]) self.assertTrue("LogFileLocation" in res["Value"][agent]) @@ -617,14 +599,16 @@ def test_checkURLs_1(self): port = ["", "1001", "1002", theTornadoPort] urls, tempurls, newurls = [], [], [] for i in [1, 2]: - urls.append("%(prot)s://%(host)s:%(port)s/Sys/Serv%(i)s" % dict(i=i, host=host, port=port[i], prot=prot[i])) + urls.append( + f"{dict(i=i, host=host, port=port[i], prot=prot[i])['prot']}://{dict(i=i, host=host, port=port[i], prot=prot[i])['host']}:{dict(i=i, host=host, port=port[i], prot=prot[i])['port']}/Sys/Serv{dict(i=i, host=host, port=port[i], prot=prot[i])['i']}" + ) for i in [1]: tempurls.append( - "%(prot)s://%(host)s:%(port)s/Sys/Serv%(i)s" % dict(i=i, host=host, port=port[i], prot=prot[i]) + f"{dict(i=i, host=host, port=port[i], prot=prot[i])['prot']}://{dict(i=i, host=host, port=port[i], prot=prot[i])['host']}:{dict(i=i, host=host, port=port[i], prot=prot[i])['port']}/Sys/Serv{dict(i=i, host=host, port=port[i], prot=prot[i])['i']}" ) for i in [1, 3]: newurls.append( - "%(prot)s://%(host)s:%(port)s/Sys/Serv%(i)s" % dict(i=i, host=host, port=port[i], prot=prot[i]) + f"{dict(i=i, host=host, port=port[i], prot=prot[i])['prot']}://{dict(i=i, host=host, port=port[i], prot=prot[i])['host']}:{dict(i=i, host=host, port=port[i], prot=prot[i])['port']}/Sys/Serv{dict(i=i, host=host, port=port[i], prot=prot[i])['i']}" ) def gVal(*args, **_kwargs): @@ -640,7 +624,7 @@ def gVal(*args, **_kwargs): if "Protocol" in args[0]: return "https" if "Serv3" in args[0] else args[1] else: - assert False, "Unknown config option requested %s" % args[0] + assert False, f"Unknown config option requested {args[0]}" gConfigMock = MagicMock() gConfigMock.getValue.side_effect = gVal @@ -648,7 +632,6 @@ def gVal(*args, **_kwargs): "Services": { "Sys": { "Serv1": { - "Setup": True, "PID": "18128", "Port": "1001", "RunitStatus": "Run", @@ -656,7 +639,6 @@ def gVal(*args, **_kwargs): "Installed": True, }, "Serv2": { - "Setup": True, "PID": "18128", "Port": "1002", "RunitStatus": "Down", @@ -664,7 +646,6 @@ def gVal(*args, **_kwargs): "Installed": True, }, "Serv3": { - "Setup": True, "PID": "18128", "RunitStatus": "Run", "Protocol": "https", @@ -672,7 +653,6 @@ def gVal(*args, **_kwargs): "Installed": True, }, "SystemAdministrator": { - "Setup": True, "PID": "18128", "Port": "1003", "RunitStatus": "Run", @@ -687,22 +667,13 @@ def gVal(*args, **_kwargs): with patch("DIRAC.FrameworkSystem.Agent.ComponentSupervisionAgent.gConfig", new=gConfigMock), patch( "DIRAC.FrameworkSystem.Agent.ComponentSupervisionAgent.socket.gethostname", return_value=host - ), patch( - "DIRAC.FrameworkSystem.Agent.ComponentSupervisionAgent.PathFinder.getSystemInstance", - return_value="Decertification", - ), patch( - "DIRAC.FrameworkSystem.Agent.ComponentSupervisionAgent.PathFinder.getComponentSection", - side_effect=mockComponentSection, - ), patch( - "DIRAC.FrameworkSystem.Agent.ComponentSupervisionAgent.PathFinder.getSystemURLSection", - side_effect=mockURLSection, ): res = self.restartAgent.checkURLs() self.assertTrue(res["OK"]) self.restartAgent.csAPI.modifyValue.assert_has_calls( [ - call("/Systems/Sys/Production/URLs/Serv", ",".join(tempurls)), - call("/Systems/Sys/Production/URLs/Serv", ",".join(newurls)), + call("/Systems/Sys/URLs/Serv", ",".join(tempurls)), + call("/Systems/Sys/URLs/Serv", ",".join(newurls)), ], any_order=False, ) @@ -717,27 +688,14 @@ def test_checkURLs_2(self): self.restartAgent.sysAdminClient.getOverallStatus.return_value = S_OK(dict(Services={})) self.restartAgent.csAPI.commit = MagicMock(return_value=S_ERROR("Nope")) - with patch("DIRAC.FrameworkSystem.Agent.ComponentSupervisionAgent.gConfig", new=MagicMock()), patch( - "DIRAC.FrameworkSystem.Agent.ComponentSupervisionAgent.PathFinder.getSystemInstance", - return_value="Decertification", - ), patch( - "DIRAC.FrameworkSystem.Agent.ComponentSupervisionAgent.PathFinder.getComponentSection", - side_effect=mockComponentSection, - ): + with patch("DIRAC.FrameworkSystem.Agent.ComponentSupervisionAgent.gConfig", new=MagicMock()): res = self.restartAgent.checkURLs() self.assertFalse(res["OK"]) self.assertIn("Failed to commit", res["Message"]) self.assertIn("Commit to CS failed", self.restartAgent.errors[0]) self.restartAgent.csAPI.commit = MagicMock(return_value=S_OK()) - with patch("DIRAC.FrameworkSystem.Agent.ComponentSupervisionAgent.gConfig", new=MagicMock()), patch( - "DIRAC.FrameworkSystem.Agent.ComponentSupervisionAgent.PathFinder.getSystemInstance", - return_value="Decertification", - ), patch( - "DIRAC.FrameworkSystem.Agent.ComponentSupervisionAgent.PathFinder.getComponentSection", - side_effect=mockComponentSection, - ): - + with patch("DIRAC.FrameworkSystem.Agent.ComponentSupervisionAgent.gConfig", new=MagicMock()): res = self.restartAgent.checkURLs() self.assertTrue(res["OK"]) diff --git a/src/DIRAC/FrameworkSystem/Client/BundleDeliveryClient.py b/src/DIRAC/FrameworkSystem/Client/BundleDeliveryClient.py index 5680f7af713..d51ab956769 100644 --- a/src/DIRAC/FrameworkSystem/Client/BundleDeliveryClient.py +++ b/src/DIRAC/FrameworkSystem/Client/BundleDeliveryClient.py @@ -1,21 +1,43 @@ """ Client for interacting with Framework/BundleDelivery service """ -import os import getpass +import os import tarfile -from io import BytesIO from base64 import b64decode +from io import BytesIO -from DIRAC import S_OK, S_ERROR, gLogger +from DIRAC import S_ERROR, S_OK, gLogger +from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import skipCACheck from DIRAC.Core.Base.Client import Client, createClient -from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient -from DIRAC.Core.Tornado.Client.ClientSelector import TransferClientSelector as TransferClient from DIRAC.Core.Security import Locations, Utilities +from DIRAC.Core.Tornado.Client.ClientSelector import TransferClientSelector as TransferClient +from DIRAC.Core.Tornado.Client.TornadoClient import TornadoClient from DIRAC.Core.Utilities.File import mkDir -from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import skipCACheck + + +def getHash(bundleID, dirToSyncTo): + """Get hash for bundle in directory + + :param str bundleID: bundle ID + :param str dirToSyncTo: path to sync directory + + :return: str + """ + bundle_path = os.path.join(dirToSyncTo, f".dab.{bundleID}") + if not os.path.isfile(bundle_path): + return "" + try: + with open(os.path.join(dirToSyncTo, f".dab.{bundleID}"), "rb") as fd: + bdHash = fd.read().strip() + return bdHash.decode() + except OSError as e: + gLogger.exception("File can't be open/read, returning empty string", lException=e) + return "" class BundleDeliveryJSONClient(TornadoClient): + """Utility class for JSON-encoded HTTPs responses""" + def receiveFile(self, buff, fileId): retVal = self.executeRPC("streamToClient", fileId) if retVal["OK"]: @@ -26,6 +48,8 @@ def receiveFile(self, buff, fileId): @createClient("Framework/BundleDelivery") class BundleDeliveryClient(Client): + """Client for interacting with Framework/BundleDelivery service""" + def __init__(self, transferClient=False, **kwargs): super().__init__(**kwargs) self.setServer("Framework/BundleDelivery") @@ -43,21 +67,6 @@ def __getTransferClient(self): "Framework/BundleDelivery", skipCACheck=skipCACheck(), httpsClient=BundleDeliveryJSONClient ) - def __getHash(self, bundleID, dirToSyncTo): - """Get hash for bundle in directory - - :param str bundleID: bundle ID - :param str dirToSyncTo: path to sync directory - - :return: str - """ - try: - with open(os.path.join(dirToSyncTo, ".dab.%s" % bundleID), "rb") as fd: - bdHash = fd.read().strip() - return bdHash.decode() - except Exception: - return "" - def __setHash(self, bundleID, dirToSyncTo, bdHash): """Set hash for bundle in directory @@ -66,7 +75,7 @@ def __setHash(self, bundleID, dirToSyncTo, bdHash): :param str bdHash: new hash """ try: - fileName = os.path.join(dirToSyncTo, ".dab.%s" % bundleID) + fileName = os.path.join(dirToSyncTo, f".dab.{bundleID}") with open(fileName, "wb") as fd: fd.write(bdHash if isinstance(bdHash, bytes) else bdHash.encode()) except Exception as e: @@ -78,7 +87,7 @@ def syncDir(self, bundleID, dirToSyncTo): :param str bundleID: bundle ID :param str dirToSyncTo: path to sync directory - :return: S_OK(bool)/S_ERROR() + :return: S_OK()/S_ERROR() """ dirCreated = False if os.path.isdir(dirToSyncTo): @@ -87,43 +96,43 @@ def syncDir(self, bundleID, dirToSyncTo): self.log.error(f"{getpass.getuser()} does not have the permissions to update {dirToSyncTo}") return S_ERROR(f"{getpass.getuser()} does not have the permissions to update {dirToSyncTo}") else: - self.log.info("Creating dir %s" % dirToSyncTo) + self.log.info("Creating directory", dirToSyncTo) mkDir(dirToSyncTo) dirCreated = True - currentHash = self.__getHash(bundleID, dirToSyncTo) - self.log.info(f"Current hash for bundle {bundleID} in dir {dirToSyncTo} is '{currentHash}'") + currentHash = getHash(bundleID, dirToSyncTo) + self.log.info(f"Current hash for bundle {bundleID} in directory {dirToSyncTo} is '{currentHash}'") buff = BytesIO() transferClient = self.__getTransferClient() result = transferClient.receiveFile(buff, [bundleID, currentHash]) if not result["OK"]: - self.log.error("Could not sync dir", result["Message"]) + self.log.error("Could not sync directory", result["Message"]) if dirCreated: - self.log.info("Removing dir %s" % dirToSyncTo) + self.log.info("Removing directory", dirToSyncTo) os.unlink(dirToSyncTo) buff.close() return result newHash = result["Value"] if newHash == currentHash: - self.log.info("Dir %s was already in sync" % dirToSyncTo) - return S_OK(False) + self.log.info(f"Directory {dirToSyncTo} was already in sync") + return S_OK() buff.seek(0) - self.log.info("Synchronizing dir with remote bundle") + self.log.info("Synchronizing directory with remote bundle") with tarfile.open(name="dummy", mode="r:gz", fileobj=buff) as tF: for tarinfo in tF: try: tF.extract(tarinfo, dirToSyncTo) except OSError as e: - self.log.error("Could not sync dir:", str(e)) + self.log.error("Could not sync directory:", str(e)) if dirCreated: - self.log.info("Removing dir %s" % dirToSyncTo) + self.log.info("Removing directory", dirToSyncTo) os.unlink(dirToSyncTo) buff.close() - return S_ERROR("Certificates directory update failed: %s" % str(e)) + return S_ERROR(f"Certificates directory update failed: {str(e)}") buff.close() self.__setHash(bundleID, dirToSyncTo, newHash) self.log.info("Dir has been synchronized") - return S_OK(True) + return S_OK() def syncCAs(self): """Synchronize CAs @@ -134,13 +143,9 @@ def syncCAs(self): if "X509_CERT_DIR" in os.environ: X509_CERT_DIR = os.environ["X509_CERT_DIR"] del os.environ["X509_CERT_DIR"] - casLocation = Locations.getCAsLocation() - if not casLocation: - casLocation = Locations.getCAsDefaultLocation() - result = self.syncDir("CAs", casLocation) if X509_CERT_DIR: os.environ["X509_CERT_DIR"] = X509_CERT_DIR - return result + return self.syncDir("CAs", Locations.getCAsLocation()) def syncCRLs(self): """Synchronize CRLs @@ -151,10 +156,9 @@ def syncCRLs(self): if "X509_CERT_DIR" in os.environ: X509_CERT_DIR = os.environ["X509_CERT_DIR"] del os.environ["X509_CERT_DIR"] - result = self.syncDir("CRLs", Locations.getCAsLocation()) if X509_CERT_DIR: os.environ["X509_CERT_DIR"] = X509_CERT_DIR - return result + return self.syncDir("CRLs", Locations.getCAsLocation()) def getCAs(self): """This method can be used to create the CAs. If the file can not be created, @@ -163,19 +167,19 @@ def getCAs(self): :return: S_OK(str)/S_ERROR() """ retVal = Utilities.generateCAFile() - if not retVal["OK"]: - self.log.warn("Could not generate/find CA file", retVal["Message"]) - # if we can not found the file, we return the directory, where the file should be - transferClient = self.__getTransferClient() - casFile = os.path.join(os.path.dirname(retVal["Message"]), "cas.pem") - with open(casFile, "w") as fd: - result = transferClient.receiveFile(fd, "CAs") - if not result["OK"]: - return result - return S_OK(casFile) - else: + if retVal["OK"]: return retVal + self.log.warn("Could not generate/find CA file", retVal["Message"]) + # if we can not found the file, we return the directory, where the file should be + transferClient = self.__getTransferClient() + casFile = os.path.join(os.path.dirname(retVal["Message"]), "cas.pem") + with open(casFile, "w") as fd: + result = transferClient.receiveFile(fd, "CAs") + if not result["OK"]: + return result + return S_OK(casFile) + def getCLRs(self): """This method can be used to create the CRLs. If the file can not be created, it will be downloaded from the server. @@ -183,14 +187,14 @@ def getCLRs(self): :return: S_OK(str)/S_ERROR() """ retVal = Utilities.generateRevokedCertsFile() - if not retVal["OK"]: - # if we can not found the file, we return the directory, where the file should be - transferClient = self.__getTransferClient() - casFile = os.path.join(os.path.dirname(retVal["Message"]), "crls.pem") - with open(casFile, "w") as fd: - result = transferClient.receiveFile(fd, "CRLs") - if not result["OK"]: - return result - return S_OK(casFile) - else: + if retVal["OK"]: return retVal + + # if we can not found the file, we return the directory, where the file should be + transferClient = self.__getTransferClient() + casFile = os.path.join(os.path.dirname(retVal["Message"]), "crls.pem") + with open(casFile, "w") as fd: + result = transferClient.receiveFile(fd, "CRLs") + if not result["OK"]: + return result + return S_OK(casFile) diff --git a/src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py b/src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py index 7859da404e0..0e173e38190 100644 --- a/src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py +++ b/src/DIRAC/FrameworkSystem/Client/ComponentInstaller.py @@ -6,8 +6,6 @@ The Following Options are used:: - /DIRAC/Setup: Setup to be used for any operation - /LocalInstallation/InstanceName: Name of the Instance for the current Setup (default /DIRAC/Setup) /LocalInstallation/LogLevel: LogLevel set in "run" script for all components installed /LocalInstallation/Host: Used when build the URL to be published for the installed service (default: socket.getfqdn()) @@ -36,7 +34,7 @@ /LocalInstallation/PrivateConfiguration: Boolean, requires Configuration/Server to be given in the list of Services (default: no) -If a Master Configuration Server is being installed the following Options can be used:: +If a Controller Configuration Server is being installed the following Options can be used:: /LocalInstallation/ConfigurationName: Name of the Configuration (default: Setup ) /LocalInstallation/AdminUserName: Name of the Admin user (default: None ) @@ -50,66 +48,59 @@ import glob import importlib import inspect -import io -import MySQLdb import os import pkgutil import re import shutil import stat +import subprocess +import textwrap import time from collections import defaultdict import importlib_metadata as metadata import importlib_resources -import subprocess +import MySQLdb from diraccfg import CFG from prompt_toolkit import prompt import DIRAC -from DIRAC import rootPath -from DIRAC import gConfig -from DIRAC import gLogger -from DIRAC.Core.Utilities.Subprocess import systemCall -from DIRAC.Core.Utilities.ReturnValues import S_OK, S_ERROR - -from DIRAC.Core.Utilities.Version import getVersion -from DIRAC.Core.Utilities.File import mkDir, mkLink +from DIRAC import gConfig, gLogger, rootPath +from DIRAC.ConfigurationSystem.Client import PathFinder from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI from DIRAC.ConfigurationSystem.Client.Helpers import ( - cfgPath, - cfgPathToList, cfgInstallPath, cfgInstallSection, - CSGlobals, + cfgPath, + cfgPathToList, ) +from DIRAC.Core.Base.private.ModuleLoader import ModuleLoader from DIRAC.Core.Security.Properties import ( - ALARMS_MANAGEMENT, - SERVICE_ADMINISTRATOR, CS_ADMINISTRATOR, - JOB_ADMINISTRATOR, FULL_DELEGATION, - PROXY_MANAGEMENT, - OPERATOR, + JOB_ADMINISTRATOR, NORMAL_USER, + OPERATOR, + PRODUCTION_MANAGEMENT, + PROXY_MANAGEMENT, + SERVICE_ADMINISTRATOR, + SITE_MANAGER, TRUSTED_HOST, ) - -from DIRAC.ConfigurationSystem.Client import PathFinder -from DIRAC.Core.Utilities.MySQL import MySQL -from DIRAC.Core.Base.private.ModuleLoader import ModuleLoader -from DIRAC.Core.Base.AgentModule import AgentModule -from DIRAC.Core.Base.ExecutorModule import ExecutorModule -from DIRAC.Core.DISET.RequestHandler import RequestHandler -from DIRAC.Core.Utilities.PrettyPrint import printTable from DIRAC.Core.Utilities.Extensions import ( extensionsByPriority, + findAgents, findDatabases, + findExecutors, findModules, - findAgents, findServices, - findExecutors, ) +from DIRAC.Core.Utilities.File import mkDir, mkLink +from DIRAC.Core.Utilities.MySQL import MySQL +from DIRAC.Core.Utilities.PrettyPrint import printTable +from DIRAC.Core.Utilities.ReturnValues import S_ERROR, S_OK +from DIRAC.Core.Utilities.Subprocess import systemCall +from DIRAC.Core.Utilities.Version import getVersion from DIRAC.FrameworkSystem.Client.ComponentMonitoringClient import ComponentMonitoringClient @@ -150,10 +141,10 @@ def _makeComponentDict(component, setupDict, installedDict, compType, system, ru def _getSectionName(compType): """ Returns the section name for a component in the CS - For self.instance, the section for service is Services, + The section for service is Services, whereas the section for agent is Agents """ - return "%ss" % compType.title() + return f"{compType.title()}s" class ComponentInstaller: @@ -169,7 +160,7 @@ def __init__(self): gLogger.debug("DIRAC Root Path =", rootPath) self.mysqlMode = "" - self.localCfg = None + self.localCfg: CFG = None self.cfgFile = "" self.setup = "" self.instance = "" @@ -216,8 +207,6 @@ def loadDiracCfg(self): gLogger.always("Can't load ", self.cfgFile) gLogger.always("Might be OK if setting up the site") - self.setup = self.localCfg.getOption(cfgPath("DIRAC", "Setup"), "") - self.instance = self.localCfg.getOption(cfgInstallPath("InstanceName"), self.setup) self.logLevel = self.localCfg.getOption(cfgInstallPath("LogLevel"), "INFO") self.linkedRootPath = rootPath @@ -252,7 +241,7 @@ def loadDiracCfg(self): self.mysqlPassword = self.localCfg.getOption(cfgInstallPath("Database", "Password"), self.mysqlPassword) if self.mysqlPassword: - gLogger.verbose("Reading %s MySQL Password from local configuration " % self.mysqlUser) + gLogger.verbose(f"Reading {self.mysqlUser} MySQL Password from local configuration ") else: gLogger.warn("MySQL password not found") @@ -279,7 +268,7 @@ def loadDiracCfg(self): self.mysqlMode = self.localCfg.getOption(cfgInstallPath("Database", "MySQLMode"), "") if self.mysqlMode: - gLogger.verbose("Configuring MySQL server as %s" % self.mysqlMode) + gLogger.verbose(f"Configuring MySQL server as {self.mysqlMode}") # Now some noSQL defaults self.noSQLHost = self.localCfg.getOption(cfgInstallPath("NoSQLDatabase", "Host"), "") @@ -304,7 +293,7 @@ def loadDiracCfg(self): self.noSQLPassword = self.localCfg.getOption(cfgInstallPath("NoSQLDatabase", "Password"), self.noSQLPassword) if self.noSQLPassword: - gLogger.verbose("Reading %s NoSQL Password from local configuration " % self.noSQLUser) + gLogger.verbose(f"Reading {self.noSQLUser} NoSQL Password from local configuration ") else: gLogger.warn("NoSQL password not found") @@ -323,7 +312,6 @@ def getInfo(self): if not result["OK"]: return result rDict = result["Value"] - rDict["Setup"] = self.setup or "Unknown" return S_OK(rDict) def _addCfgToDiracCfg(self, cfg): @@ -351,8 +339,7 @@ def _addCfgToCS(self, cfg): result = cfgClient.mergeFromCFG(cfg) if not result["OK"]: return result - result = cfgClient.commit() - return result + return cfgClient.commit() def _addCfgToLocalCS(self, cfg): """ @@ -360,14 +347,14 @@ def _addCfgToLocalCS(self, cfg): """ csName = self.localCfg.getOption(cfgPath("DIRAC", "Configuration", "Name"), "") if not csName: - error = "Missing %s" % cfgPath("DIRAC", "Configuration", "Name") + error = f"Missing {cfgPath('DIRAC', 'Configuration', 'Name')}" if self.exitOnError: gLogger.error(error) DIRAC.exit(-1) return S_ERROR(error) csCfg = CFG() - csFile = os.path.join(rootPath, "etc", "%s.cfg" % csName) + csFile = os.path.join(rootPath, "etc", f"{csName}.cfg") if os.path.exists(csFile): csCfg.loadFromFile(csFile) newCfg = csCfg.mergeWith(cfg) if str(csCfg) else cfg @@ -403,7 +390,7 @@ def _removeSectionFromCS(self, path): def _getCentralCfg(self, installCfg): """ - Create the skeleton of central Cfg for an initial Master CS + Create the skeleton of central Cfg for an initial Controller CS """ # First copy over from installation cfg centralCfg = CFG() @@ -414,6 +401,9 @@ def _getCentralCfg(self, installCfg): if extensions: centralCfg["DIRAC"].addKey("Extensions", ",".join(extensions), "") # pylint: disable=no-member + # No Setups will be used + centralCfg["DIRAC"].addKey("NoSetup", "True", "") # pylint: disable=no-member + vo = self.localCfg.getOption(cfgInstallPath("VirtualOrganization"), "") if vo: centralCfg["DIRAC"].addKey("VirtualOrganization", vo, "") # pylint: disable=no-member @@ -431,7 +421,6 @@ def _getCentralCfg(self, installCfg): hostDN = self.localCfg.getOption(cfgInstallPath("HostDN"), "") defaultGroupName = self.localCfg.getOption(cfgInstallPath("DefaultGroupName"), "dirac_user") adminGroupProperties = [ - ALARMS_MANAGEMENT, SERVICE_ADMINISTRATOR, CS_ADMINISTRATOR, JOB_ADMINISTRATOR, @@ -443,10 +432,13 @@ def _getCentralCfg(self, installCfg): defaultHostProperties = [ TRUSTED_HOST, CS_ADMINISTRATOR, + SERVICE_ADMINISTRATOR, + SITE_MANAGER, JOB_ADMINISTRATOR, FULL_DELEGATION, PROXY_MANAGEMENT, OPERATOR, + PRODUCTION_MANAGEMENT, ] for section in ( @@ -489,23 +481,26 @@ def _getCentralCfg(self, installCfg): centralCfg["Registry"]["Groups"][group].addKey("Users", "", "") users = centralCfg["Registry"]["Groups"][group].getOption("Users", []) if adminUserName not in users: - centralCfg["Registry"]["Groups"][group].appendToOption("Users", ", %s" % adminUserName) + centralCfg["Registry"]["Groups"][group].appendToOption("Users", f", {adminUserName}") if not centralCfg["Registry"]["Groups"][group].isOption("Properties"): centralCfg["Registry"]["Groups"][group].addKey("Properties", "", "") + if vo and not centralCfg["Registry"]["Groups"][group].isOption("VO"): + centralCfg["Registry"]["Groups"][group].addKey("VO", vo, "") + properties = centralCfg["Registry"]["Groups"][adminGroupName].getOption("Properties", []) for prop in adminGroupProperties: if prop not in properties: properties.append(prop) - centralCfg["Registry"]["Groups"][adminGroupName].appendToOption("Properties", ", %s" % prop) + centralCfg["Registry"]["Groups"][adminGroupName].appendToOption("Properties", f", {prop}") properties = centralCfg["Registry"]["Groups"][defaultGroupName].getOption("Properties", []) for prop in defaultGroupProperties: if prop not in properties: properties.append(prop) - centralCfg["Registry"]["Groups"][defaultGroupName].appendToOption("Properties", ", %s" % prop) + centralCfg["Registry"]["Groups"][defaultGroupName].appendToOption("Properties", f", {prop}") - # Add the master Host description + # Add the controller Host description if hostDN: hostSection = cfgPath("Registry", "Hosts", self.host) if not centralCfg.isSection(hostSection): @@ -519,7 +514,7 @@ def _getCentralCfg(self, installCfg): for prop in defaultHostProperties: if prop not in properties: properties.append(prop) - centralCfg["Registry"]["Hosts"][self.host].appendToOption("Properties", ", %s" % prop) + centralCfg["Registry"]["Hosts"][self.host].appendToOption("Properties", f", {prop}") # Operations if adminUserEmail: @@ -573,12 +568,10 @@ def addOptionToDiracCfg(self, option, value): return S_ERROR(f"Could not merge {option}={value} with local configuration") - def removeComponentOptionsFromCS(self, system, component, mySetup=None): + def removeComponentOptionsFromCS(self, system, component): """ Remove the section with Component options from the CS, if possible """ - if mySetup is None: - mySetup = self.setup result = self.monitoringClient.getInstallations( {"UnInstallationTime": None, "Instance": component}, {"DIRACSystem": system}, {}, True @@ -587,12 +580,6 @@ def removeComponentOptionsFromCS(self, system, component, mySetup=None): return result installations = result["Value"] - instanceOption = cfgPath("DIRAC", "Setups", mySetup, system) - if gConfig: - compInstance = gConfig.getValue(instanceOption, "") - else: - compInstance = self.localCfg.getOption(instanceOption, "") - if len(installations) == 1: remove = True removeMain = False @@ -623,18 +610,16 @@ def removeComponentOptionsFromCS(self, system, component, mySetup=None): if remove: result = self._removeSectionFromCS( - cfgPath("Systems", system, compInstance, installation["Component"]["Type"].title() + "s", component) + cfgPath("Systems", system, installation["Component"]["Type"].title() + "s", component) ) if not result["OK"]: return result if not isRenamed and cType == "service": - result = self._removeOptionFromCS(cfgPath("Systems", system, compInstance, "URLs", component)) + result = self._removeOptionFromCS(cfgPath("Systems", system, "URLs", component)) if not result["OK"]: # It is maybe in the FailoverURLs ? - result = self._removeOptionFromCS( - cfgPath("Systems", system, compInstance, "FailoverURLs", component) - ) + result = self._removeOptionFromCS(cfgPath("Systems", system, "FailoverURLs", component)) if not result["OK"]: return result @@ -643,7 +628,6 @@ def removeComponentOptionsFromCS(self, system, component, mySetup=None): cfgPath( "Systems", system, - compInstance, installation["Component"]["Type"].title() + "s", installation["Component"]["Module"], ) @@ -653,14 +637,12 @@ def removeComponentOptionsFromCS(self, system, component, mySetup=None): if cType == "service": result = self._removeOptionFromCS( - cfgPath("Systems", system, compInstance, "URLs", installation["Component"]["Module"]) + cfgPath("Systems", system, "URLs", installation["Component"]["Module"]) ) if not result["OK"]: # it is maybe in the FailoverURLs ? result = self._removeOptionFromCS( - cfgPath( - "Systems", system, compInstance, "FailoverURLs", installation["Component"]["Module"] - ) + cfgPath("Systems", system, "FailoverURLs", installation["Component"]["Module"]) ) if not result["OK"]: return result @@ -675,7 +657,6 @@ def addDefaultOptionsToCS( systemName, component, extensions, - mySetup=None, specialOptions={}, overwrite=False, addDefaultOptions=True, @@ -683,26 +664,16 @@ def addDefaultOptionsToCS( """ Add the section with the component options to the CS """ - if mySetup is None: - mySetup = self.setup if gConfig_o: gConfig_o.forceRefresh() system = systemName.replace("System", "") - instanceOption = cfgPath("DIRAC", "Setups", mySetup, system) - if gConfig_o: - compInstance = gConfig_o.getValue(instanceOption, "") - else: - compInstance = self.localCfg.getOption(instanceOption, "") - if not compInstance: - return S_ERROR(f"{instanceOption} not defined in {self.cfgFile}") - sectionName = _getSectionName(componentType) # Check if the component CS options exist addOptions = True - componentSection = cfgPath("Systems", system, compInstance, sectionName, component) + componentSection = cfgPath("Systems", system, sectionName, component) if not overwrite: if gConfig_o: result = gConfig_o.getOptions(componentSection) @@ -713,26 +684,31 @@ def addDefaultOptionsToCS( return S_OK("Component options already exist") # Add the component options now - result = self.getComponentCfg( - componentType, system, component, compInstance, extensions, specialOptions, addDefaultOptions - ) + result = self.getComponentCfg(componentType, system, component, extensions, specialOptions, addDefaultOptions) if not result["OK"]: return result compCfg = result["Value"] gLogger.notice("Adding to CS", f"{componentType} {system}/{component}") resultAddToCFG = self._addCfgToCS(compCfg) + if not resultAddToCFG["OK"]: + return resultAddToCFG + resultAddToCFG = self.addTornadoOptionsToCS(gConfig_o) + if not resultAddToCFG["OK"]: + return resultAddToCFG + if componentType == "executor": # Is it a container ? - execList = compCfg.getOption("%s/Load" % componentSection, []) + execList = compCfg.getOption(f"{componentSection}/Load", []) for element in execList: result = self.addDefaultOptionsToCS( - gConfig_o, componentType, systemName, element, extensions, self.setup, {}, overwrite + gConfig_o, componentType, systemName, element, extensions, {}, overwrite ) if not result["OK"]: gLogger.warn("Can't add to default CS", result["Message"]) resultAddToCFG.setdefault("Modules", {}) resultAddToCFG["Modules"][element] = result["OK"] + return resultAddToCFG def addDefaultOptionsToComponentCfg(self, componentType, systemName, component, extensions): @@ -740,13 +716,9 @@ def addDefaultOptionsToComponentCfg(self, componentType, systemName, component, Add default component options local component cfg """ system = systemName.replace("System", "") - instanceOption = cfgPath("DIRAC", "Setups", self.setup, system) - compInstance = self.localCfg.getOption(instanceOption, "") - if not compInstance: - return S_ERROR(f"{instanceOption} not defined in {self.cfgFile}") # Add the component options now - result = self.getComponentCfg(componentType, system, component, compInstance, extensions) + result = self.getComponentCfg(componentType, system, component, extensions) if not result["OK"]: return result compCfg = result["Value"] @@ -765,28 +737,22 @@ def addCfgToComponentCfg(self, componentType, systemName, component, cfg): if not cfg: return S_OK() system = systemName.replace("System", "") - instanceOption = cfgPath("DIRAC", "Setups", self.setup, system) - compInstance = self.localCfg.getOption(instanceOption, "") - if not compInstance: - return S_ERROR(f"{instanceOption} not defined in {self.cfgFile}") compCfgFile = os.path.join(rootPath, "etc", f"{system}_{component}.cfg") compCfg = CFG() if os.path.exists(compCfgFile): compCfg.loadFromFile(compCfgFile) - sectionPath = cfgPath("Systems", system, compInstance, sectionName) + sectionPath = cfgPath("Systems", system, sectionName) newCfg = self.__getCfg(sectionPath) newCfg.createNewSection(cfgPath(sectionPath, component), "Added by ComponentInstaller", cfg) if newCfg.writeToFile(compCfgFile): return S_OK(compCfgFile) - error = "Can not write %s" % compCfgFile + error = f"Can not write {compCfgFile}" gLogger.error(error) return S_ERROR(error) - def getComponentCfg( - self, componentType, system, component, compInstance, extensions, specialOptions={}, addDefaultOptions=True - ): + def getComponentCfg(self, componentType, system, component, extensions, specialOptions={}, addDefaultOptions=True): """ Get the CFG object of the component configuration """ @@ -798,7 +764,9 @@ def getComponentCfg( for ext in extensions: cfgTemplateModule = f"{ext}.{system}System" try: - cfgTemplate = importlib_resources.read_text(cfgTemplateModule, "ConfigTemplate.cfg") + cfgTemplate = ( + importlib_resources.files(cfgTemplateModule).joinpath("ConfigTemplate.cfg").read_text() + ) except (ImportError, OSError): continue gLogger.notice("Loading configuration template from", cfgTemplateModule) @@ -808,7 +776,7 @@ def getComponentCfg( compPath = cfgPath(sectionName, componentModule) if not compCfg.isSection(compPath): - error = "Can not find %s in template" % compPath + error = f"Can not find {compPath} in template" gLogger.error(error) if self.exitOnError: DIRAC.exit(-1) @@ -819,31 +787,32 @@ def getComponentCfg( # Delete Dependencies section if any compCfg.deleteKey("Dependencies") - sectionPath = cfgPath("Systems", system, compInstance, sectionName) + sectionPath = cfgPath("Systems", system, sectionName) cfg = self.__getCfg(sectionPath) cfg.createNewSection(cfgPath(sectionPath, component), "", compCfg) for option, value in specialOptions.items(): - cfg.setOption(cfgPath(sectionPath, component, option), value) + cfg.setOption( + cfgPath(sectionPath, component, option), + value, + ) # Add the service URL if componentType == "service": port = compCfg.getOption("Port", 0) protocol = compCfg.getOption("Protocol", "dips") if (port or protocol == "https") and self.host: - urlsPath = cfgPath("Systems", system, compInstance, "URLs") + urlsPath = cfgPath("Systems", system, "URLs") cfg.createNewSection(urlsPath) - failoverUrlsPath = cfgPath("Systems", system, compInstance, "FailoverURLs") + failoverUrlsPath = cfgPath("Systems", system, "FailoverURLs") cfg.createNewSection(failoverUrlsPath) if protocol == "https": - tornadoPort = gConfig.getValue( - "/Systems/Tornado/%s/Port" % PathFinder.getSystemInstance("Tornado"), - 8443, - ) + if not port: + port = gConfig.getValue(f"/Systems/Tornado/Port", 8443) cfg.setOption( # Strip "Tornado" from the beginning of component name if present cfgPath(urlsPath, component[len("Tornado") if component.startswith("Tornado") else 0 :]), - f"https://{self.host}:{tornadoPort}/{system}/{component}", + f"https://{self.host}:{port}/{system}/{component}", ) else: cfg.setOption( @@ -852,29 +821,19 @@ def getComponentCfg( return S_OK(cfg) - def addDatabaseOptionsToCS(self, gConfig_o, systemName, dbName, mySetup=None, overwrite=False): + def addDatabaseOptionsToCS(self, gConfig_o, systemName, dbName, overwrite=False): """ Add the section with the database options to the CS """ - if mySetup is None: - mySetup = self.setup - if gConfig_o: gConfig_o.forceRefresh() system = systemName.replace("System", "") - instanceOption = cfgPath("DIRAC", "Setups", mySetup, system) - if gConfig_o: - compInstance = gConfig_o.getValue(instanceOption, "") - else: - compInstance = self.localCfg.getOption(instanceOption, "") - if not compInstance: - return S_ERROR(f"{instanceOption} not defined in {self.cfgFile}") # Check if the component CS options exist addOptions = True if not overwrite: - databasePath = cfgPath("Systems", system, compInstance, "Databases", dbName) + databasePath = cfgPath("Systems", system, "Databases", dbName) result = gConfig_o.getOptions(databasePath) if result["OK"]: addOptions = False @@ -882,20 +841,17 @@ def addDatabaseOptionsToCS(self, gConfig_o, systemName, dbName, mySetup=None, ov return S_OK("Database options already exist") # Add the component options now - result = self.getDatabaseCfg(system, dbName, compInstance) + result = self.getDatabaseCfg(system, dbName) if not result["OK"]: return result databaseCfg = result["Value"] gLogger.notice("Adding to CS", f"{system}/{dbName}") return self._addCfgToCS(databaseCfg) - def removeDatabaseOptionsFromCS(self, gConfig_o, system, dbName, mySetup=None): + def removeDatabaseOptionsFromCS(self, system, dbName): """ Remove the section with database options from the CS, if possible """ - if mySetup is None: - mySetup = self.setup - result = self.monitoringClient.installationExists( {"UnInstallationTime": None}, {"DIRACSystem": system, "Type": "DB", "DIRACModule": dbName}, {} ) @@ -903,50 +859,24 @@ def removeDatabaseOptionsFromCS(self, gConfig_o, system, dbName, mySetup=None): return result exists = result["Value"] - instanceOption = cfgPath("DIRAC", "Setups", mySetup, system) - if gConfig_o: - compInstance = gConfig_o.getValue(instanceOption, "") - else: - compInstance = self.localCfg.getOption(instanceOption, "") - if not exists: - result = self._removeSectionFromCS(cfgPath("Systems", system, compInstance, "Databases", dbName)) + result = self._removeSectionFromCS(cfgPath("Systems", system, "Databases", dbName)) if not result["OK"]: return result return S_OK("Successfully removed entries from CS") - def getDatabaseCfg(self, system, dbName, compInstance): + def getDatabaseCfg(self, system, dbName): """ Get the CFG object of the database configuration """ - databasePath = cfgPath("Systems", system, compInstance, "Databases", dbName) + databasePath = cfgPath("Systems", system, "Databases", dbName) cfg = self.__getCfg(databasePath, "DBName", dbName) cfg.setOption(cfgPath(databasePath, "Host"), self.mysqlHost) cfg.setOption(cfgPath(databasePath, "Port"), self.mysqlPort) return S_OK(cfg) - def addSystemInstance(self, systemName, compInstance, mySetup=None, myCfg=False): - """ - Add a new system self.instance to dirac.cfg and CS - """ - if mySetup is None: - mySetup = self.setup - - system = systemName.replace("System", "") - gLogger.notice( - "Adding %s system as %s self.instance for %s self.setup to dirac.cfg and CS" - % (system, compInstance, mySetup) - ) - - cfg = self.__getCfg(cfgPath("DIRAC", "Setups", mySetup), system, compInstance) - if myCfg: - if not self._addCfgToDiracCfg(cfg): - return S_ERROR("Failed to add system self.instance to dirac.cfg") - - return self._addCfgToCS(cfg) - def printStartupStatus(self, rDict): """ Print in nice format the return dictionary from self.getStartupComponentStatus @@ -959,7 +889,7 @@ def printStartupStatus(self, rDict): records.append([comp, rDict[comp]["RunitStatus"], rDict[comp]["Timeup"], str(rDict[comp]["PID"])]) printTable(fields, records) except Exception as x: - print("Exception while gathering data for printing: %s" % str(x)) + print(f"Exception while gathering data for printing: {str(x)}") return S_OK() def printOverallStatus(self, rDict): @@ -987,7 +917,7 @@ def printOverallStatus(self, rDict): records.append(record) printTable(fields, records) except Exception as x: - print("Exception while gathering data for printing: %s" % str(x)) + print(f"Exception while gathering data for printing: {str(x)}") return S_OK() @@ -1062,7 +992,7 @@ def getInstalledComponents(self): pass else: for cType in self.componentTypes: - if "dirac-%s" % cType in body or (cType.lower() == "service" and "tornado-start-all" in body): + if f"dirac-{cType}" in body or (cType.lower() == "service" and "tornado-start-all" in body): resultDict[cType][system].append(component) return S_OK({resultIndexes[cType]: dict(resultDict[cType]) for cType in self.componentTypes}) @@ -1073,7 +1003,7 @@ def getSetupComponents(self): set up for running with runsvdir in startup directory """ if not os.path.isdir(self.startDir): - return S_ERROR("Startup Directory does not exit: %s" % self.startDir) + return S_ERROR(f"Startup Directory does not exit: {self.startDir}") result = self.resultIndexes(self.componentTypes) if not result["OK"]: @@ -1090,7 +1020,7 @@ def getSetupComponents(self): pass else: for cType in self.componentTypes: - if "dirac-%s" % cType in body or (cType.lower() == "service" and "tornado-start-all" in body): + if f"dirac-{cType}" in body or (cType.lower() == "service" and "tornado-start-all" in body): system, compT = component.split("_", 1) resultDict[cType][system].append(compT) @@ -1125,7 +1055,7 @@ def getStartupComponentStatus(self, componentTupleList): if not line: continue cname, routput = line.split(":") - cname = cname.replace("%s/" % self.startDir, "") + cname = cname.replace(f"{self.startDir}/", "") run = False reResult = re.search("^ run", routput) if reResult: @@ -1183,11 +1113,7 @@ def getComponentModule(self, system, component, compType): """ Get the component software module """ - self.setup = CSGlobals.getSetup() - self.instance = gConfig.getValue(cfgPath("DIRAC", "Setups", self.setup, system), "") - if not self.instance: - return S_OK(component) - module = gConfig.getValue(cfgPath("Systems", system, self.instance, compType, component, "Module"), "") + module = gConfig.getValue(cfgPath("Systems", system, compType, component, "Module"), "") if not module: module = component return S_OK(module) @@ -1252,13 +1178,13 @@ def checkComponentModule(self, componentType, system, module): and if it inherits from the proper class """ if componentType == "agent": - loader = ModuleLoader("Agent", PathFinder.getAgentSection, AgentModule) + loader = ModuleLoader("Agent", PathFinder.getAgentSection) elif componentType == "service": - loader = ModuleLoader("Service", PathFinder.getServiceSection, RequestHandler, moduleSuffix="Handler") + loader = ModuleLoader("Service", PathFinder.getServiceSection, moduleSuffix="Handler") elif componentType == "executor": - loader = ModuleLoader("Executor", PathFinder.getExecutorSection, ExecutorModule) + loader = ModuleLoader("Executor", PathFinder.getExecutorSection) else: - return S_ERROR("Unknown component type %s" % componentType) + return S_ERROR(f"Unknown component type {componentType}") return loader.loadModule(f"{system}/{module}") @@ -1274,7 +1200,7 @@ def checkComponentSoftware(self, componentType, system, component, extensions): try: softDict = softComp[_getSectionName(componentType)] except KeyError: - return S_ERROR("Unknown component type %s" % componentType) + return S_ERROR(f"Unknown component type {componentType}") if system in softDict and component in softDict[system]: return S_OK() @@ -1286,7 +1212,7 @@ def runsvctrlComponent(self, system, component, mode): Execute runsvctrl and check status of the specified component """ if mode not in ["u", "d", "o", "p", "c", "h", "a", "i", "q", "1", "2", "t", "k", "x", "e"]: - return S_ERROR('Unknown runsvctrl mode "%s"' % mode) + return S_ERROR(f'Unknown runsvctrl mode "{mode}"') startCompDirs = glob.glob(os.path.join(self.startDir, f"{system}_{component}")) # Make sure that the Configuration server restarts first and the SystemAdmin restarts last @@ -1352,7 +1278,7 @@ def setupSite(self, scriptCfg, cfg=None): self._addCfgToDiracCfg(diracCfg) except Exception: # pylint: disable=broad-except - error = "Failed to load %s" % cfg + error = f"Failed to load {cfg}" gLogger.exception(error) if self.exitOnError: DIRAC.exit(-1) @@ -1360,14 +1286,24 @@ def setupSite(self, scriptCfg, cfg=None): # Now get the necessary info from self.localCfg setupSystems = self.localCfg.getOption(cfgInstallPath("Systems"), ["Configuration", "Framework"]) + setupDatabases = self.localCfg.getOption(cfgInstallPath("Databases"), []) - setupServices = [k.split("/") for k in self.localCfg.getOption(cfgInstallPath("Services"), [])] - setupAgents = [k.split("/") for k in self.localCfg.getOption(cfgInstallPath("Agents"), [])] - setupExecutors = [k.split("/") for k in self.localCfg.getOption(cfgInstallPath("Executors"), [])] + setupServices = [ + k.split("/") + for k in self.localCfg.getOption(cfgInstallPath("Services"), []) # pylint: disable=not-an-iterable + ] + setupAgents = [ + k.split("/") + for k in self.localCfg.getOption(cfgInstallPath("Agents"), []) # pylint: disable=not-an-iterable + ] + setupExecutors = [ + k.split("/") + for k in self.localCfg.getOption(cfgInstallPath("Executors"), []) # pylint: disable=not-an-iterable + ] setupWeb = self.localCfg.getOption(cfgInstallPath("WebPortal"), False) - setupConfigurationMaster = self.localCfg.getOption(cfgInstallPath("ConfigurationMaster"), False) + setupConfigurationController = self.localCfg.getOption(cfgInstallPath("ConfigurationMaster"), False) setupPrivateConfiguration = self.localCfg.getOption(cfgInstallPath("PrivateConfiguration"), False) - setupConfigurationName = self.localCfg.getOption(cfgInstallPath("ConfigurationName"), self.setup) + setupConfigurationName = self.localCfg.getOption(cfgInstallPath("ConfigurationName"), "DIRAC-Prod") setupAddConfiguration = self.localCfg.getOption(cfgInstallPath("AddConfiguration"), True) for serviceTuple in setupServices: @@ -1378,7 +1314,7 @@ def setupSite(self, scriptCfg, cfg=None): DIRAC.exit(-1) return S_ERROR(error) serviceSysInstance = serviceTuple[0] - if serviceSysInstance not in setupSystems: + if serviceSysInstance not in setupSystems: # pylint: disable=unsupported-membership-test setupSystems.append(serviceSysInstance) for agentTuple in setupAgents: @@ -1389,7 +1325,7 @@ def setupSite(self, scriptCfg, cfg=None): DIRAC.exit(-1) return S_ERROR(error) agentSysInstance = agentTuple[0] - if agentSysInstance not in setupSystems: + if agentSysInstance not in setupSystems: # pylint: disable=unsupported-membership-test setupSystems.append(agentSysInstance) for executorTuple in setupExecutors: @@ -1400,7 +1336,7 @@ def setupSite(self, scriptCfg, cfg=None): DIRAC.exit(-1) return S_ERROR(error) executorSysInstance = executorTuple[0] - if executorSysInstance not in setupSystems: + if executorSysInstance not in setupSystems: # pylint: disable=unsupported-membership-test setupSystems.append(executorSysInstance) # And to find out the available extensions @@ -1422,7 +1358,7 @@ def setupSite(self, scriptCfg, cfg=None): DIRAC.exit(-1) return S_ERROR(error) - # if any server or agent needs to be install we need the startup directory and runsvdir running + # if any server or agent needs to be installed we need the startup directory and runsvdir running if setupServices or setupAgents or setupExecutors or setupWeb: if not os.path.exists(self.startDir): mkDir(self.startDir) @@ -1447,30 +1383,32 @@ def setupSite(self, scriptCfg, cfg=None): universal_newlines=True, ) - if ["Configuration", "Server"] in setupServices and setupConfigurationMaster: - # This server hosts the Master of the CS + if ["Configuration", "Server"] in setupServices and setupConfigurationController: + # This server hosts the Controller of the CS from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData - gLogger.notice("Installing Master Configuration Server") + gLogger.notice("Installing Controller Configuration Server (Tornado-based)") - cfg = self.__getCfg(cfgPath("DIRAC", "Setups", self.setup), "Configuration", self.instance) + # Add some needed bootstrapping configuration + cfg = self.__getCfg( + cfgPath("Systems", "Configuration", "Services", "Server"), + "HandlerPath", + "DIRAC/ConfigurationSystem/Service/TornadoConfigurationHandler.py", + ) + self._addCfgToDiracCfg(cfg) + cfg = self.__getCfg(cfgPath("Systems", "Configuration", "Services", "Server"), "Port", "9135") self._addCfgToDiracCfg(cfg) cfg = self.__getCfg(cfgPath("DIRAC", "Configuration"), "Master", "yes") cfg.setOption(cfgPath("DIRAC", "Configuration", "Name"), setupConfigurationName) serversCfgPath = cfgPath("DIRAC", "Configuration", "Servers") if not self.localCfg.getOption(serversCfgPath, []): - serverUrl = "dips://%s:9135/Configuration/Server" % self.host + serverUrl = f"https://{self.host}:9135/Configuration/Server" cfg.setOption(serversCfgPath, serverUrl) gConfigurationData.setOptionInCFG(serversCfgPath, serverUrl) - instanceOptionPath = cfgPath("DIRAC", "Setups", self.setup) - instanceCfg = self.__getCfg(instanceOptionPath, "Configuration", self.instance) - cfg = cfg.mergeWith(instanceCfg) self._addCfgToDiracCfg(cfg) - result = self.getComponentCfg( - "service", "Configuration", "Server", self.instance, extensions, addDefaultOptions=True - ) + result = self.getComponentCfg("service", "Configuration", "Server", extensions, addDefaultOptions=True) if not result["OK"]: if self.exitOnError: DIRAC.exit(-1) @@ -1505,20 +1443,17 @@ def setupSite(self, scriptCfg, cfg=None): DIRAC.exit(-1) return S_ERROR(error) - # We need to make sure components are connecting to the Master CS, that is the only one being update + # We need to make sure components are connecting to the Controller CS, that is the only one being update localServers = self.localCfg.getOption(cfgPath("DIRAC", "Configuration", "Servers")) - masterServer = gConfig.getValue(cfgPath("DIRAC", "Configuration", "MasterServer"), "") + controllerServer = gConfig.getValue(cfgPath("DIRAC", "Configuration", "MasterServer"), "") initialCfg = self.__getCfg(cfgPath("DIRAC", "Configuration"), "Servers", localServers) - masterCfg = self.__getCfg(cfgPath("DIRAC", "Configuration"), "Servers", masterServer) - self._addCfgToDiracCfg(masterCfg) + controllerCfg = self.__getCfg(cfgPath("DIRAC", "Configuration"), "Servers", controllerServer) + self._addCfgToDiracCfg(controllerCfg) # 1.- Setup the instances in the CS - # If the Configuration Server used is not the Master, it can take some time for this + # If the Configuration Server used is not the Controller, it can take some time for this # info to be propagated, this may cause the later self.setup to fail if setupAddConfiguration: - gLogger.notice("Registering System instances") - for system in setupSystems: - self.addSystemInstance(system, self.instance, self.setup, True) for system, service in setupServices: if not self.addDefaultOptionsToCS(None, "service", system, service, extensions, overwrite=True)["OK"]: # If we are not allowed to write to the central CS, add the configuration to the local file @@ -1529,6 +1464,17 @@ def setupSite(self, scriptCfg, cfg=None): res = self.addDefaultOptionsToComponentCfg("service", system, service, extensions) if not res["OK"]: gLogger.warn("Can't write to the specific component CFG") + + if service.startswith("Tornado"): + gLogger.notice("Installing Tornado") + if not (res := self.installTornado())["OK"]: + return res + + if not (res := self.setupTornadoService(system, service))["OK"]: + return res + + self.runsvctrlComponent("Tornado", "Tornado", "t") + for system, agent in setupAgents: if not self.addDefaultOptionsToCS(None, "agent", system, agent, extensions, overwrite=True)["OK"]: # If we are not allowed to write to the central CS, add the configuration to the local file @@ -1566,7 +1512,7 @@ def setupSite(self, scriptCfg, cfg=None): return result dbDict = result["Value"] - for dbName in setupDatabases: + for dbName in setupDatabases: # pylint: disable=not-an-iterable gLogger.verbose("Setting up database", dbName) if dbName not in installedDatabases: result = self.installDatabase(dbName) @@ -1576,12 +1522,12 @@ def setupSite(self, scriptCfg, cfg=None): extension, system = result["Value"] gLogger.notice(f"Database {dbName} from {extension}/{system} installed") else: - gLogger.notice("Database %s already installed" % dbName) + gLogger.notice(f"Database {dbName} already installed") dbSystem = dbDict[dbName]["System"] result = self.addDatabaseOptionsToCS(None, dbSystem, dbName, overwrite=True) if not result["OK"]: - gLogger.error("Database {} CS registration failed: {}".format(dbName, result["Message"])) + gLogger.error(f"Database {dbName} CS registration failed: {result['Message']}") if self.mysqlPassword and not self._addMySQLToDiracCfg(): error = "Failed to add MySQL user/password to local configuration" @@ -1599,22 +1545,20 @@ def setupSite(self, scriptCfg, cfg=None): # 3.- Then installed requested services for system, service in setupServices: - result = self.setupComponent("service", system, service, extensions) - if not result["OK"]: - gLogger.error(result["Message"]) - continue + if not service.startswith("Tornado"): + if not (result := self.setupComponent("service", system, service, extensions))["OK"]: + gLogger.error(result["Message"]) + continue # 4.- Now the agents for system, agent in setupAgents: - result = self.setupComponent("agent", system, agent, extensions) - if not result["OK"]: + if not (result := self.setupComponent("agent", system, agent, extensions))["OK"]: gLogger.error(result["Message"]) continue # 5.- Now the executors for system, executor in setupExecutors: - result = self.setupComponent("executor", system, executor, extensions) - if not result["OK"]: + if not (result := self.setupComponent("executor", system, executor, extensions))["OK"]: gLogger.error(result["Message"]) continue @@ -1622,7 +1566,7 @@ def setupSite(self, scriptCfg, cfg=None): if setupWeb: self.setupPortal() - if localServers != masterServer: + if localServers != controllerServer: self._addCfgToDiracCfg(initialCfg) for system, service in setupServices: self.runsvctrlComponent(system, service, "t") @@ -1643,29 +1587,33 @@ def _createRunitLog(self, runitCompDir): logConfigFile = os.path.join(logDir, "config") with open(logConfigFile, "w") as fd: fd.write( - """s10000000 - n20 - """ + textwrap.dedent( + """ + s10000000 + n20 + """ + ) ) logRunFile = os.path.join(logDir, "run") with open(logRunFile, "w") as fd: fd.write( - """#!/bin/bash - -rcfile=%(bashrc)s -[[ -e $rcfile ]] && source ${rcfile} -# -exec svlogd . - """ - % {"bashrc": os.path.join(self.instancePath, "bashrc")} + textwrap.dedent( + f"""#!/bin/bash + + rcfile={os.path.join(self.instancePath, "bashrc")} + [[ -e ${{rcfile}} ]] && source ${{rcfile}} + # + exec svlogd . + """ + ) ) os.chmod(logRunFile, self.gDefaultPerms) def installComponent(self, componentType, system, component, extensions, componentModule="", checkModule=True): """ - Install runit directory for the specified component + DIPS services: install runit directory for the specified component """ # Check if the component is already installed runitCompDir = os.path.join(self.runitDir, system, component) @@ -1693,18 +1641,10 @@ def installComponent(self, componentType, system, component, extensions, compone gLogger.notice(f"Installing {componentType} {system}/{component}") - # Retrieve bash variables to be set - result = gConfig.getOption(f"DIRAC/Setups/{CSGlobals.getSetup()}/{system}") - if not result["OK"]: - return result - self.instance = result["Value"] - specialOptions = {} if componentModule: specialOptions["Module"] = componentModule - result = self.getComponentCfg( - componentType, system, component, self.instance, extensions, specialOptions=specialOptions - ) + result = self.getComponentCfg(componentType, system, component, extensions, specialOptions=specialOptions) if not result["OK"]: return result compCfg = result["Value"] @@ -1712,13 +1652,13 @@ def installComponent(self, componentType, system, component, extensions, compone section = _getSectionName(componentType) bashVars = "" - if compCfg.isSection(f"Systems/{system}/{self.instance}/{section}/{component}/Environment"): + if compCfg.isSection(f"Systems/{system}/{section}/{component}/Environment"): dictionary = compCfg.getAsDict() - bashSection = dictionary["Systems"][system][self.instance][section][component]["BashVariables"] + bashSection = dictionary["Systems"][system][section][component]["BashVariables"] for var in bashSection: bashVars = f"{bashVars}\nexport {var}={bashSection[var]}" - # Now do the actual installation + # Now do the actual installation (for DIPS) try: componentCfg = os.path.join(self.linkedRootPath, "etc", f"{system}_{component}.cfg") if not os.path.exists(componentCfg): @@ -1728,28 +1668,42 @@ def installComponent(self, componentType, system, component, extensions, compone runFile = os.path.join(runitCompDir, "run") with open(runFile, "w") as fd: - fd.write( - """#!/bin/bash - -rcfile=%(bashrc)s -[[ -e $rcfile ]] && source ${rcfile} -# -exec 2>&1 -# -[[ "%(componentType)s" = "agent" ]] && renice 20 -p $$ -#%(bashVariables)s -# -exec dirac-%(componentType)s %(system)s/%(component)s --cfg %(componentCfg)s < /dev/null - """ - % { - "bashrc": os.path.join(self.instancePath, "bashrc"), - "bashVariables": bashVars, - "componentType": componentType.replace("-", "_"), - "system": system, - "component": component, - "componentCfg": componentCfg, - } - ) + # Special case for tornado-based master CS + if system == "Configuration" and component == "Server": + fd.write( + textwrap.dedent( + f"""#!/bin/bash + + rcfile={os.path.join(self.instancePath, 'bashrc')} + [[ -e ${{rcfile}} ]] && source ${{rcfile}} + # + export DIRAC_USE_TORNADO_IOLOOP=Yes + exec 2>&1 + # + [ "service" = "agent" ] && renice 20 -p $$ + # + # + exec tornado-start-CS -ddd + """ + ) + ) + else: + fd.write( + textwrap.dedent( + f"""#!/bin/bash + + rcfile={os.path.join(self.instancePath, "bashrc")} + [[ -e ${{rcfile}} ]] && source ${{rcfile}} + # + exec 2>&1 + # + [[ "{componentType.replace("-", "_")}" = "agent" ]] && renice 20 -p $$ + #{bashVars} + # + exec dirac-{componentType.replace("-", "_")} {system}/{component} --cfg {componentCfg} < /dev/null + """ + ) + ) os.chmod(runFile, self.gDefaultPerms) @@ -1761,18 +1715,19 @@ def installComponent(self, componentType, system, component, extensions, compone controlDir = self.runitDir.replace("runit", "control") with open(stopFile, "w") as fd: fd.write( - """#!/bin/bash + textwrap.dedent( + f"""#!/bin/bash -echo %(controlDir)s/%(system)s/%(component)s/stop_%(type)s -touch %(controlDir)s/%(system)s/%(component)s/stop_%(type)s -""" - % {"controlDir": controlDir, "system": system, "component": component, "type": cTypeLower} + echo {controlDir}/{system}/{component}/stop_{cTypeLower} + touch {controlDir}/{system}/{component}/stop_{cTypeLower} + """ + ) ) os.chmod(stopFile, self.gDefaultPerms) except Exception: - error = f"Failed to prepare self.setup for {componentType} {system}/{component}" + error = f"Failed to prepare setup for {componentType} {system}/{component}" gLogger.exception(error) if self.exitOnError: DIRAC.exit(-1) @@ -1884,7 +1839,7 @@ def setupPortal(self): result = self.getStartupComponentStatus([("Web", "WebApp")]) if not result["OK"]: return S_ERROR("Failed to start the Portal") - if result["Value"] and result["Value"]["{}_{}".format("Web", "WebApp")]["RunitStatus"] == "Run": + if result["Value"] and result["Value"]["Web_WebApp"]["RunitStatus"] == "Run": break time.sleep(1) @@ -1926,21 +1881,22 @@ def installPortal(self): runFile = os.path.join(runitWebAppDir, "run") with open(runFile, "w") as fd: fd.write( - """#!/bin/bash - -rcfile=%(bashrc)s -[[ -e $rcfile ]] && source $rcfile -# -exec 2>&1 -# -exec dirac-webapp-run -p < /dev/null - """ - % {"bashrc": os.path.join(self.instancePath, "bashrc"), "DIRAC": self.linkedRootPath} + textwrap.dedent( + f"""#!/bin/bash + + rcfile={os.path.join(self.instancePath, 'bashrc')} + [[ -e $rcfile ]] && source $rcfile + # + exec 2>&1 + # + exec dirac-webapp-run -p < /dev/null + """ + ) ) os.chmod(runFile, self.gDefaultPerms) except Exception: - error = "Failed to prepare self.setup for Web Portal" + error = "Failed to prepare setup for Web Portal" gLogger.exception(error) if self.exitOnError: DIRAC.exit(-1) @@ -2052,7 +2008,7 @@ def getAvailableESDatabases(self, extensions): Result should be something like:: {'MonitoringDB': {'Type': 'ES', 'System': 'Monitoring', 'Extension': ''}, - 'ElasticJobParametersDB': {'Type': 'ES', 'System': 'WorkloadManagement', 'Extension': ''}} + 'JobParametersDB': {'Type': 'ES', 'System': 'WorkloadManagement', 'Extension': ''}} :param list extensions: list of DIRAC extensions :return: dict of ES DBs @@ -2121,7 +2077,7 @@ def installDatabase(self, dbName): if filename in databases: break else: - error = "Database %s not found" % dbName + error = f"Database {dbName} not found" gLogger.error(error) if self.exitOnError: DIRAC.exit(-1) @@ -2129,7 +2085,7 @@ def installDatabase(self, dbName): systemName = databases[filename] moduleName = ".".join([extension, systemName, "DB"]) gLogger.debug(f"Installing {filename} from {moduleName}") - dbSql = importlib_resources.read_text(moduleName, filename) + dbSql = importlib_resources.files(moduleName).joinpath(filename).read_text() # just check result = self.execMySQL("SHOW STATUS") @@ -2142,7 +2098,7 @@ def installDatabase(self, dbName): gLogger.debug("SHOW STATUS : OK") # now creating the Database - result = self.execMySQL("CREATE DATABASE `%s`" % dbName) + result = self.execMySQL(f"CREATE DATABASE `{dbName}`") if not result["OK"] and "database exists" not in result["Message"]: gLogger.error("Failed to create databases", result["Message"]) if self.exitOnError: @@ -2157,12 +2113,12 @@ def installDatabase(self, dbName): cmd = f"GRANT {perms} ON `{dbName}`.* TO '{self.mysqlUser}'@'%'" result = self.execMySQL(cmd) if not result["OK"]: - error = "Error executing '%s'" % cmd + error = f"Error executing '{cmd}'" gLogger.error(error, result["Message"]) if self.exitOnError: DIRAC.exit(-1) return S_ERROR(error) - gLogger.debug("%s : OK" % cmd, result["Value"]) + gLogger.debug(f"{cmd} : OK", result["Value"]) result = self.execMySQL("FLUSH PRIVILEGES") if not result["OK"]: gLogger.error("Failed to flush privileges", result["Message"]) @@ -2210,7 +2166,7 @@ def uninstallDatabase(self, gConfig_o, dbName): dbSystem = result["Value"][dbName]["System"] - result = self.removeDatabaseOptionsFromCS(gConfig_o, dbSystem, dbName) + result = self.removeDatabaseOptionsFromCS(dbSystem, dbName) if not result["OK"]: return result @@ -2227,9 +2183,9 @@ def _createMySQLCMDLines(self, dbSql): # Should we first source an SQL file (is this sql file an extension)? if line.lower().startswith("source"): sourcedDBbFileName = line.split(" ")[1].replace("\n", "") - gLogger.info("Found file to source: %s" % sourcedDBbFileName) + gLogger.info(f"Found file to source: {sourcedDBbFileName}") module, filename = sourcedDBbFileName.rsplit("/", 1) - dbSourced = importlib_resources.read_text(module.replace("/", "."), filename) + dbSourced = importlib_resources.files(module.replace("/", ".")).joinpath(filename).read_text() for lineSourced in dbSourced.split("\n"): if lineSourced.strip(): cmdLines.append(lineSourced.strip()) @@ -2265,7 +2221,7 @@ def _addMySQLToDiracCfg(self): Add the database access info to the local configuration """ if not self.mysqlPassword: - return S_ERROR("Missing {} in {}".format(cfgInstallPath("Database", "Password"), self.cfgFile)) + return S_ERROR(f"Missing {cfgInstallPath('Database', 'Password')} in {self.cfgFile}") sectionPath = cfgPath("Systems", "Databases") cfg = self.__getCfg(sectionPath, "User", self.mysqlUser) @@ -2300,7 +2256,7 @@ def execCommand(self, timeout, cmd): if not result["OK"]: if timeout and result["Message"].startswith("Timeout"): return result - gLogger.error("Failed to execute", "{}: {}".format(cmd[0], result["Message"])) + gLogger.error("Failed to execute", f"{cmd[0]}: {result['Message']}") if self.exitOnError: DIRAC.exit(-1) return result @@ -2308,7 +2264,7 @@ def execCommand(self, timeout, cmd): if result["Value"][0]: error = "Failed to execute" gLogger.error(error, cmd[0]) - gLogger.error("Exit code:", ("%s\n" % result["Value"][0]) + "\n".join(result["Value"][1:])) + gLogger.error("Exit code:", (f"{result['Value'][0]}\n") + "\n".join(result["Value"][1:])) if self.exitOnError: DIRAC.exit(-1) error = S_ERROR(error) @@ -2326,41 +2282,35 @@ def installTornado(self): # Check if the Tornado itself is already installed runitCompDir = os.path.join(self.runitDir, "Tornado", "Tornado") if os.path.exists(runitCompDir): - msg = "Tornado_Tornado already installed" - gLogger.notice(msg) + gLogger.notice("Tornado_Tornado already installed") return S_OK(runitCompDir) - # Check the setup for the given system - result = gConfig.getOption("DIRAC/Setups/%s/Tornado" % (CSGlobals.getSetup())) - if not result["OK"]: - return result - self.instance = result["Value"] - # Now do the actual installation try: - self._createRunitLog(runitCompDir) runFile = os.path.join(runitCompDir, "run") - with open(runFile, "wt") as fd: + with open(runFile, "w") as fd: fd.write( - """#!/bin/bash -rcfile=%(bashrc)s -[ -e $rcfile ] && source $rcfile -# -export DIRAC_USE_TORNADO_IOLOOP=Yes -exec 2>&1 -# -# -exec tornado-start-all -""" - % {"bashrc": os.path.join(self.instancePath, "bashrc")} + textwrap.dedent( + f"""#!/bin/bash + + rcfile={os.path.join(self.instancePath, 'bashrc')} + [ -e $rcfile ] && source $rcfile + # + export DIRAC_USE_TORNADO_IOLOOP=Yes + exec 2>&1 + # + # + exec tornado-start-all + """ + ) ) os.chmod(runFile, self.gDefaultPerms) except Exception: - error = "Failed to prepare self.setup forTornado" + error = "Failed to prepare setup for Tornado" gLogger.exception(error) if self.exitOnError: DIRAC.exit(-1) @@ -2379,21 +2329,18 @@ def addTornadoOptionsToCS(self, gConfig_o): if gConfig_o: gConfig_o.forceRefresh() - instanceOption = cfgPath("DIRAC", "Setups", self.setup, "Tornado") + tornadoSection = cfgPath("Systems", "Tornado") if gConfig_o: - compInstance = gConfig_o.getValue(instanceOption, "") + tornadoPort = gConfig_o.getValue(cfgPath(tornadoSection, "Port"), "8443") else: - compInstance = self.localCfg.getOption(instanceOption, "") - if not compInstance: - return S_ERROR(f"{instanceOption} not defined in {self.cfgFile}") - tornadoSection = cfgPath("Systems", "Tornado", compInstance) + tornadoPort = self.localCfg.getOption(cfgPath(tornadoSection, "Port"), "8443") + + cfg = self.__getCfg(tornadoSection, "Port", tornadoPort) - cfg = self.__getCfg(tornadoSection, "Port", 8443) - # cfg.setOption(cfgPath(tornadoSection, 'Password'), self.mysqlPassword) return self._addCfgToCS(cfg) - def setupTornadoService(self, system, component, extensions, componentModule="", checkModule=True): + def setupTornadoService(self, system, component): """ Install and create link in startup """ @@ -2437,14 +2384,5 @@ def setupTornadoService(self, system, component, extensions, componentModule="", return S_OK(resDict) - # port = compCfg.getOption('Port', 0) - # if port and self.host: - # urlsPath = cfgPath('Systems', system, compInstance, 'URLs') - # cfg.createNewSection(urlsPath) - # failoverUrlsPath = cfgPath('Systems', system, compInstance, 'FailoverURLs') - # cfg.createNewSection(failoverUrlsPath) - # cfg.setOption(cfgPath(urlsPath, component), - # 'dips://%s:%d/%s/%s' % (self.host, port, system, component)) - gComponentInstaller = ComponentInstaller() diff --git a/src/DIRAC/FrameworkSystem/Client/Logger.py b/src/DIRAC/FrameworkSystem/Client/Logger.py index 864883ac866..7b48d9b49ee 100755 --- a/src/DIRAC/FrameworkSystem/Client/Logger.py +++ b/src/DIRAC/FrameworkSystem/Client/Logger.py @@ -1,3 +1,4 @@ +from DIRAC.FrameworkSystem.private.standardLogging.LoggingContext import contextLogger, setContextLogger from DIRAC.FrameworkSystem.private.standardLogging.LoggingRoot import LoggingRoot gLogger = LoggingRoot() @@ -5,3 +6,6 @@ def getLogger(): return gLogger + + +__all__ = ["contextLogger", "setContextLogger", "getLogger"] diff --git a/src/DIRAC/FrameworkSystem/Client/NotificationClient.py b/src/DIRAC/FrameworkSystem/Client/NotificationClient.py index a396ddf7264..f72a7facc41 100644 --- a/src/DIRAC/FrameworkSystem/Client/NotificationClient.py +++ b/src/DIRAC/FrameworkSystem/Client/NotificationClient.py @@ -1,7 +1,7 @@ """ DIRAC Notification Client class encapsulates the methods exposed by the Notification service. """ -from DIRAC import gLogger, S_ERROR +from DIRAC import S_ERROR, gLogger from DIRAC.Core.Base.Client import Client, createClient from DIRAC.Core.Utilities.Mail import Mail @@ -27,7 +27,6 @@ def sendMail(self, addresses, subject, body, fromAddress=None, localAttempt=True addresses = [addresses] if isinstance(addresses, str) else list(addresses) for address in addresses: - if localAttempt: try: m = Mail() @@ -39,7 +38,7 @@ def sendMail(self, addresses, subject, body, fromAddress=None, localAttempt=True m._fromAddress = fromAddress result = m._send() except Exception as x: - self.log.warn("Sending mail failed with exception:\n%s" % (str(x))) + self.log.warn(f"Sending mail failed with exception:\n{str(x)}") if result["OK"]: self.log.verbose(f"Mail sent successfully from local host to {address} with subject {subject}") @@ -59,59 +58,3 @@ def sendMail(self, addresses, subject, body, fromAddress=None, localAttempt=True self.log.verbose(result["Value"]) return result - - def sendSMS(self, userName, body, fromAddress=None): - """Send an SMS with body to the specified DIRAC user name.""" - if not fromAddress: - fromAddress = "" - - self.log.verbose(f"Received signal to send the following SMS to {userName}:\n{body}") - result = self._getRPC().sendSMS(userName, body, fromAddress) - if not result["OK"]: - self.log.error("Could not send SMS via central Notification service", result["Message"]) - else: - self.log.verbose(result["Value"]) - - return result - - ########################################################################### - # ALARMS - ########################################################################### - - def newAlarm(self, subject, status, notifications, assignee, body, priority, alarmKey=""): - if not isinstance(notifications, (list, tuple)): - return S_ERROR( - "Notifications parameter has to be a list or a tuple with a combination of [ 'Web', 'Mail', 'SMS' ]" - ) - alarmDef = { - "subject": subject, - "status": status, - "notifications": notifications, - "assignee": assignee, - "priority": priority, - "body": body, - } - if alarmKey: - alarmDef["alarmKey"] = alarmKey - return self._getRPC().newAlarm(alarmDef) - - def updateAlarm(self, id=-1, alarmKey="", comment=False, modDict={}): - if id == -1 and not alarmKey: - return S_ERROR("Need either alarm id or key to update an alarm!") - updateReq = {"comment": comment, "modifications": modDict} - if id != -1: - updateReq["id"] = id - if alarmKey: - updateReq["alarmKey"] = alarmKey - return self._getRPC().updateAlarm(updateReq) - - ########################################################################### - # MANAGE NOTIFICATIONS - ########################################################################### - - def addNotificationForUser(self, user, message, lifetime=604800, deferToMail=True): - try: - lifetime = int(lifetime) - except Exception: - return S_ERROR("Message lifetime has to be a non decimal number") - return self._getRPC().addNotificationForUser(user, message, lifetime, deferToMail) diff --git a/src/DIRAC/FrameworkSystem/Client/ProxyGeneration.py b/src/DIRAC/FrameworkSystem/Client/ProxyGeneration.py index 6334fe5f330..3329e90718e 100644 --- a/src/DIRAC/FrameworkSystem/Client/ProxyGeneration.py +++ b/src/DIRAC/FrameworkSystem/Client/ProxyGeneration.py @@ -6,12 +6,10 @@ from prompt_toolkit import prompt from DIRAC import S_OK, S_ERROR, gLogger from DIRAC.Core.Base.Script import Script -from DIRAC.Core.Utilities.NTP import getClockDeviation from DIRAC.Core.Security.m2crypto import DEFAULT_PROXY_STRENGTH class CLIParams: - proxyLifeTime = 86400 diracGroup = False proxyStrength = DEFAULT_PROXY_STRENGTH @@ -24,7 +22,6 @@ class CLIParams: checkWithCS = True stdinPasswd = False userPasswd = "" - checkClock = True embedDefaultGroup = True def setProxyLifeTime(self, arg): @@ -93,7 +90,7 @@ def setProxyStrength(self, arg): try: self.proxyStrength = int(arg) except Exception: - gLogger.error("Can't parse bits! Is it a number?", "%s" % arg) + gLogger.error("Can't parse bits! Is it a number?", f"{arg}") return S_ERROR("Can't parse strength argument") return S_OK() @@ -178,16 +175,6 @@ def setStrict(self, _arg): self.strict = True return S_OK() - def disableClockCheck(self, _arg): - """Disable clock check - - :param _arg: unuse - - :return: S_OK() - """ - self.checkClock = False - return S_OK() - def registerCLISwitches(self): """Register CLI switches""" Script.registerSwitch( @@ -203,7 +190,6 @@ def registerCLISwitches(self): Script.registerSwitch("u:", "out=", "File to write as proxy", self.setProxyLocation) Script.registerSwitch("x", "nocs", "Disable CS check", self.setDisableCSCheck) Script.registerSwitch("p", "pwstdin", "Get passwd from stdin", self.setStdinPasswd) - Script.registerSwitch("j", "noclockcheck", "Disable checking if time is ok", self.disableClockCheck) from DIRAC.Core.Security.X509Chain import X509Chain # pylint: disable=import-error @@ -218,21 +204,6 @@ def generateProxy(params): :return: S_OK()/S_ERROR() """ - if params.checkClock: - result = getClockDeviation() - if result["OK"]: - deviation = result["Value"] - if deviation > 600: - gLogger.error("Your host clock seems to be off by more than TEN MINUTES! Thats really bad.") - gLogger.error("We're cowardly refusing to generate a proxy. Please fix your system time") - sys.exit(1) - elif deviation > 180: - gLogger.error("Your host clock seems to be off by more than THREE minutes! Thats bad.") - gLogger.notice("We'll generate the proxy but please fix your system time") - elif deviation > 60: - gLogger.error("Your host clock seems to be off by more than a minute! Thats not good.") - gLogger.notice("We'll generate the proxy but please fix your system time") - certLoc = params.certLoc keyLoc = params.keyLoc if not certLoc or not keyLoc: @@ -250,7 +221,7 @@ def generateProxy(params): testChain = X509Chain() retVal = testChain.loadChainFromFile(params.certLoc) if not retVal["OK"]: - return S_ERROR("Cannot load certificate {}: {}".format(params.certLoc, retVal["Message"])) + return S_ERROR(f"Cannot load certificate {params.certLoc}: {retVal['Message']}") timeLeft = int(testChain.getRemainingSecs()["Value"] / 86400) if timeLeft < 30: gLogger.notice("\nYour certificate will expire in %d days. Please renew it!\n" % timeLeft) @@ -277,13 +248,13 @@ def generateProxy(params): retVal = chain.loadChainFromFile(certLoc) if not retVal["OK"]: gLogger.warn(retVal["Message"]) - return S_ERROR("Can't load %s" % certLoc) + return S_ERROR(f"Can't load {certLoc}") retVal = chain.loadKeyFromFile(keyLoc, password=params.userPasswd) if not retVal["OK"]: gLogger.warn(retVal["Message"]) if "bad decrypt" in retVal["Message"] or "bad pass phrase" in retVal["Message"]: return S_ERROR("Bad passphrase") - return S_ERROR("Can't load %s" % keyLoc) + return S_ERROR(f"Can't load {keyLoc}") if params.checkWithCS: retVal = chain.generateProxyToFile( @@ -297,29 +268,29 @@ def generateProxy(params): if "Unauthorized query" in retVal["Message"]: # add hint for users return S_ERROR( - "Can't contact DIRAC CS: %s (User possibly not registered with dirac server) " % retVal["Message"] + f"Can't contact DIRAC CS: {retVal['Message']} (User possibly not registered with dirac server) " ) - return S_ERROR("Can't contact DIRAC CS: %s" % retVal["Message"]) + return S_ERROR(f"Can't contact DIRAC CS: {retVal['Message']}") userDN = chain.getCertInChain(-1)["Value"].getSubjectDN()["Value"] if not params.diracGroup: result = Registry.findDefaultGroupForDN(userDN) if not result["OK"]: - gLogger.warn("Could not get a default group for DN {}: {}".format(userDN, result["Message"])) + gLogger.warn(f"Could not get a default group for DN {userDN}: {result['Message']}") else: params.diracGroup = result["Value"] - gLogger.info("Default discovered group is %s" % params.diracGroup) - gLogger.info("Checking DN %s" % userDN) + gLogger.info(f"Default discovered group is {params.diracGroup}") + gLogger.info(f"Checking DN {userDN}") retVal = Registry.getUsernameForDN(userDN) if not retVal["OK"]: gLogger.warn(retVal["Message"]) - return S_ERROR("DN %s is not registered" % userDN) + return S_ERROR(f"DN {userDN} is not registered") username = retVal["Value"] - gLogger.info("Username is %s" % username) + gLogger.info(f"Username is {username}") retVal = Registry.getGroupsForUser(username) if not retVal["OK"]: gLogger.warn(retVal["Message"]) - return S_ERROR("User %s has no groups defined" % username) + return S_ERROR(f"User {username} has no groups defined") groups = retVal["Value"] if params.diracGroup not in groups: return S_ERROR(f"Requested group {params.diracGroup} is not valid for DN {userDN}") @@ -328,14 +299,14 @@ def generateProxy(params): h = int(params.proxyLifeTime / 3600) m = int(params.proxyLifeTime / 60) - h * 60 gLogger.notice("Proxy lifetime will be %02d:%02d" % (h, m)) - gLogger.notice("User cert is %s" % certLoc) - gLogger.notice("User key is %s" % keyLoc) - gLogger.notice("Proxy will be written to %s" % proxyLoc) + gLogger.notice(f"User cert is {certLoc}") + gLogger.notice(f"User key is {keyLoc}") + gLogger.notice(f"Proxy will be written to {proxyLoc}") if params.diracGroup: - gLogger.notice("DIRAC Group will be set to %s" % params.diracGroup) + gLogger.notice(f"DIRAC Group will be set to {params.diracGroup}") else: gLogger.notice("No DIRAC Group will be set") - gLogger.notice("Proxy strength will be %s" % params.proxyStrength) + gLogger.notice(f"Proxy strength will be {params.proxyStrength}") if params.limitedProxy: gLogger.notice("Proxy will be limited") retVal = chain.generateProxyToFile( @@ -347,5 +318,5 @@ def generateProxy(params): ) if not retVal["OK"]: gLogger.warn(retVal["Message"]) - return S_ERROR("Couldn't generate proxy: %s" % retVal["Message"]) + return S_ERROR(f"Couldn't generate proxy: {retVal['Message']}") return S_OK(proxyLoc) diff --git a/src/DIRAC/FrameworkSystem/Client/ProxyManagerClient.py b/src/DIRAC/FrameworkSystem/Client/ProxyManagerClient.py index bd8ca84df71..cc435685ef8 100755 --- a/src/DIRAC/FrameworkSystem/Client/ProxyManagerClient.py +++ b/src/DIRAC/FrameworkSystem/Client/ProxyManagerClient.py @@ -3,19 +3,21 @@ This inherits the DIRAC base Client for direct execution of server functionality. Client also contain caching of the requested proxy information. """ -import os + import datetime +import os -from DIRAC import S_OK, S_ERROR, gLogger +from DIRAC import S_ERROR, S_OK, gLogger from DIRAC.ConfigurationSystem.Client.Helpers import Registry -from DIRAC.Core.Utilities import ThreadSafe, DIRACSingleton -from DIRAC.Core.Utilities.DictCache import DictCache -from DIRAC.Core.Security.ProxyFile import multiProxyArgument, deleteMultiProxy +from DIRAC.Core.Base.Client import Client +from DIRAC.Core.Security import Locations +from DIRAC.Core.Security.DiracX import addTokenToPEM +from DIRAC.Core.Security.ProxyFile import deleteMultiProxy, multiProxyArgument +from DIRAC.Core.Security.VOMS import VOMS from DIRAC.Core.Security.X509Chain import X509Chain # pylint: disable=import-error from DIRAC.Core.Security.X509Request import X509Request # pylint: disable=import-error -from DIRAC.Core.Security.VOMS import VOMS -from DIRAC.Core.Security import Locations -from DIRAC.Core.Base.Client import Client +from DIRAC.Core.Utilities import DIRACSingleton, ThreadSafe +from DIRAC.Core.Utilities.DictCache import DictCache gUsersSync = ThreadSafe.Synchronizer() gProxiesSync = ThreadSafe.Synchronizer() @@ -30,7 +32,8 @@ def __init__(self): self.__pilotProxiesCache = DictCache() self.__filesCache = DictCache(self.__deleteTemporalFile) - def __deleteTemporalFile(self, filename): + @staticmethod + def __deleteTemporalFile(filename): """Delete temporal file :param str filename: path to file @@ -75,7 +78,7 @@ def __refreshUserCache(self, validSeconds=0): data = retVal["Value"] # Update the cache for record in data: - cacheKey = (record["DN"], record["group"]) + cacheKey = record["DN"] self.__usersCache.add(cacheKey, self.__getSecondsLeftToExpiration(record["expirationtime"]), record) return S_OK() @@ -91,14 +94,9 @@ def userHasProxy(self, userDN, userGroup, validSeconds=0): :return: S_OK()/S_ERROR() """ - # For backward compatibility reasons with versions prior to v7r1 - # we need to check for proxy with a group - # AND for groupless proxy even if not specified - - cacheKeys = ((userDN, userGroup), (userDN, "")) - for cacheKey in cacheKeys: - if self.__usersCache.exists(cacheKey, validSeconds): - return S_OK(True) + cacheKeys = (userDN, "") + if self.__usersCache.exists(cacheKeys, validSeconds): + return S_OK(True) # Get list of users from the DB with proxys at least 300 seconds gLogger.verbose("Updating list of users in proxy management") @@ -112,57 +110,6 @@ def userHasProxy(self, userDN, userGroup, validSeconds=0): return S_OK(False) - @gUsersSync - def getUserPersistence(self, userDN, userGroup, validSeconds=0): - """Check if a user(DN-group) has a proxy in the proxy management - Updates internal cache if needed to minimize queries to the service - - :param str userDN: user DN - :param str userGroup: user group - :param int validSeconds: proxy valid time in a seconds - - :return: S_OK()/S_ERROR() - """ - cacheKey = (userDN, userGroup) - userData = self.__usersCache.get(cacheKey, validSeconds) - if userData: - if userData["persistent"]: - return S_OK(True) - # Get list of users from the DB with proxys at least 300 seconds - gLogger.verbose("Updating list of users in proxy management") - retVal = self.__refreshUserCache(validSeconds) - if not retVal["OK"]: - return retVal - userData = self.__usersCache.get(cacheKey, validSeconds) - if userData: - return S_OK(userData["persistent"]) - return S_OK(False) - - def setPersistency(self, userDN, userGroup, persistent): - """Set the persistency for user/group - - :param str userDN: user DN - :param str userGroup: user group - :param boolean persistent: presistent flag - - :return: S_OK()/S_ERROR() - """ - # Hack to ensure bool in the rpc call - persistentFlag = True - if not persistent: - persistentFlag = False - rpcClient = Client(url="Framework/ProxyManager", timeout=120) - retVal = rpcClient.setPersistency(userDN, userGroup, persistentFlag) - if not retVal["OK"]: - return retVal - # Update internal persistency cache - cacheKey = (userDN, userGroup) - record = self.__usersCache.get(cacheKey, 0) - if record: - record["persistent"] = persistentFlag - self.__usersCache.add(cacheKey, self.__getSecondsLeftToExpiration(record["expirationtime"]), record) - return retVal - def uploadProxy(self, proxy=None, restrictLifeTime: int = 0, rfcIfPossible=None): """Upload a proxy to the proxy management service using delegation @@ -217,7 +164,7 @@ def uploadProxy(self, proxy=None, restrictLifeTime: int = 0, rfcIfPossible=None) @gProxiesSync def downloadProxy( - self, userDN, userGroup, limited=False, requiredTimeLeft=1200, cacheTime=14400, proxyToConnect=None, token=None + self, userDN, userGroup, limited=False, requiredTimeLeft=1200, cacheTime=14400, proxyToConnect=None ): """Get a proxy Chain from the proxy management @@ -227,7 +174,6 @@ def downloadProxy( :param int requiredTimeLeft: required proxy live time in a seconds :param int cacheTime: store in a cache time in a seconds :param X509Chain proxyToConnect: proxy as a chain - :param str token: valid token to get a proxy :return: S_OK(X509Chain)/S_ERROR() """ @@ -253,14 +199,7 @@ def downloadProxy( req = X509Request() req.generateProxyRequest(**generateProxyArgs) - if token: - retVal = rpcClient.getProxyWithToken( - userDN, userGroup, req.dumpRequest()["Value"], int(cacheTime + requiredTimeLeft), token - ) - else: - retVal = rpcClient.getProxy( - userDN, userGroup, req.dumpRequest()["Value"], int(cacheTime + requiredTimeLeft) - ) + retVal = rpcClient.getProxy(userDN, userGroup, req.dumpRequest()["Value"], int(cacheTime + requiredTimeLeft)) if not retVal["OK"]: return retVal chain = X509Chain(keyObj=req.getPKey()) @@ -279,7 +218,6 @@ def downloadProxyToFile( cacheTime=14400, filePath=None, proxyToConnect=None, - token=None, ): """Get a proxy Chain from the proxy management and write it to file @@ -290,11 +228,10 @@ def downloadProxyToFile( :param int cacheTime: store in a cache time in a seconds :param str filePath: path to save proxy :param X509Chain proxyToConnect: proxy as a chain - :param str token: valid token to get a proxy :return: S_OK(X509Chain)/S_ERROR() """ - retVal = self.downloadProxy(userDN, userGroup, limited, requiredTimeLeft, cacheTime, proxyToConnect, token) + retVal = self.downloadProxy(userDN, userGroup, limited, requiredTimeLeft, cacheTime, proxyToConnect) if not retVal["OK"]: return retVal chain = retVal["Value"] @@ -314,7 +251,6 @@ def downloadVOMSProxy( cacheTime=14400, requiredVOMSAttribute=None, proxyToConnect=None, - token=None, ): """Download a proxy if needed and transform it into a VOMS one @@ -325,7 +261,6 @@ def downloadVOMSProxy( :param int cacheTime: store in a cache time in a seconds :param str requiredVOMSAttribute: VOMS attr to add to the proxy :param X509Chain proxyToConnect: proxy as a chain - :param str token: valid token to get a proxy :return: S_OK(X509Chain)/S_ERROR() """ @@ -350,20 +285,9 @@ def downloadVOMSProxy( req = X509Request() req.generateProxyRequest(**generateProxyArgs) - if token: - retVal = rpcClient.getVOMSProxyWithToken( - userDN, - userGroup, - req.dumpRequest()["Value"], - int(cacheTime + requiredTimeLeft), - token, - requiredVOMSAttribute, - ) - - else: - retVal = rpcClient.getVOMSProxy( - userDN, userGroup, req.dumpRequest()["Value"], int(cacheTime + requiredTimeLeft), requiredVOMSAttribute - ) + retVal = rpcClient.getVOMSProxy( + userDN, userGroup, req.dumpRequest()["Value"], int(cacheTime + requiredTimeLeft), requiredVOMSAttribute + ) if not retVal["OK"]: return retVal chain = X509Chain(keyObj=req.getPKey()) @@ -383,7 +307,6 @@ def downloadVOMSProxyToFile( requiredVOMSAttribute=None, filePath=None, proxyToConnect=None, - token=None, ): """Download a proxy if needed, transform it into a VOMS one and write it to file @@ -395,12 +318,11 @@ def downloadVOMSProxyToFile( :param str requiredVOMSAttribute: VOMS attr to add to the proxy :param str filePath: path to save proxy :param X509Chain proxyToConnect: proxy as a chain - :param str token: valid token to get a proxy :return: S_OK(X509Chain)/S_ERROR() """ retVal = self.downloadVOMSProxy( - userDN, userGroup, limited, requiredTimeLeft, cacheTime, requiredVOMSAttribute, proxyToConnect, token + userDN, userGroup, limited, requiredTimeLeft, cacheTime, requiredVOMSAttribute, proxyToConnect ) if not retVal["OK"]: return retVal @@ -424,7 +346,7 @@ def getPilotProxyFromDIRACGroup(self, userDN, userGroup, requiredTimeLeft=43200, # Assign VOMS attribute vomsAttr = Registry.getVOMSAttributeForGroup(userGroup) if not vomsAttr: - gLogger.warn("No voms attribute assigned to group %s when requested pilot proxy" % userGroup) + gLogger.warn(f"No voms attribute assigned to group {userGroup} when requested pilot proxy") return self.downloadProxy( userDN, userGroup, limited=False, requiredTimeLeft=requiredTimeLeft, proxyToConnect=proxyToConnect ) @@ -450,7 +372,7 @@ def getPilotProxyFromVOMSGroup(self, userDN, vomsAttr, requiredTimeLeft=43200, p """ groups = Registry.getGroupsWithVOMSAttribute(vomsAttr) if not groups: - return S_ERROR("No group found that has %s as voms attrs" % vomsAttr) + return S_ERROR(f"No group found that has {vomsAttr} as voms attrs") for userGroup in groups: result = self.downloadVOMSProxy( @@ -465,13 +387,12 @@ def getPilotProxyFromVOMSGroup(self, userDN, vomsAttr, requiredTimeLeft=43200, p return result return result - def getPayloadProxyFromDIRACGroup(self, userDN, userGroup, requiredTimeLeft, token=None, proxyToConnect=None): + def getPayloadProxyFromDIRACGroup(self, userDN, userGroup, requiredTimeLeft, proxyToConnect=None): """Download a payload proxy with VOMS extensions depending on the group :param str userDN: user DN :param str userGroup: user group :param int requiredTimeLeft: required proxy live time in a seconds - :param str token: valid token to get a proxy :param X509Chain proxyToConnect: proxy as a chain :return: S_OK(X509Chain)/S_ERROR() @@ -479,14 +400,13 @@ def getPayloadProxyFromDIRACGroup(self, userDN, userGroup, requiredTimeLeft, tok # Assign VOMS attribute vomsAttr = Registry.getVOMSAttributeForGroup(userGroup) if not vomsAttr: - gLogger.verbose("No voms attribute assigned to group %s when requested payload proxy" % userGroup) + gLogger.verbose(f"No voms attribute assigned to group {userGroup} when requested payload proxy") return self.downloadProxy( userDN, userGroup, limited=True, requiredTimeLeft=requiredTimeLeft, proxyToConnect=proxyToConnect, - token=token, ) else: return self.downloadVOMSProxy( @@ -496,36 +416,9 @@ def getPayloadProxyFromDIRACGroup(self, userDN, userGroup, requiredTimeLeft, tok requiredTimeLeft=requiredTimeLeft, requiredVOMSAttribute=vomsAttr, proxyToConnect=proxyToConnect, - token=token, ) - def getPayloadProxyFromVOMSGroup(self, userDN, vomsAttr, token, requiredTimeLeft, proxyToConnect=None): - """Download a payload proxy with VOMS extensions depending on the VOMS attr - - :param str userDN: user DN - :param str vomsAttr: VOMS attribute - :param str token: valid token to get a proxy - :param int requiredTimeLeft: required proxy live time in a seconds - :param X509Chain proxyToConnect: proxy as a chain - - :return: S_OK(X509Chain)/S_ERROR() - """ - groups = Registry.getGroupsWithVOMSAttribute(vomsAttr) - if not groups: - return S_ERROR("No group found that has %s as voms attrs" % vomsAttr) - userGroup = groups[0] - - return self.downloadVOMSProxy( - userDN, - userGroup, - limited=True, - requiredTimeLeft=requiredTimeLeft, - requiredVOMSAttribute=vomsAttr, - proxyToConnect=proxyToConnect, - token=token, - ) - - def dumpProxyToFile(self, chain, destinationFile=None, requiredTimeLeft=600): + def dumpProxyToFile(self, chain, destinationFile=None, requiredTimeLeft=600, includeToken=True): """Dump a proxy to a file. It's cached so multiple calls won't generate extra files :param X509Chain chain: proxy as a chain @@ -547,6 +440,13 @@ def dumpProxyToFile(self, chain, destinationFile=None, requiredTimeLeft=600): if not retVal["OK"]: return retVal filename = retVal["Value"] + if not (result := chain.getDIRACGroup())["OK"]: + return result + if ( + includeToken + and not (result := addTokenToPEM(filename, result["Value"]))["OK"] # pylint: disable=unsubscriptable-object + ): + return result self.__filesCache.add(cHash, chain.getRemainingSecs()["Value"], filename) return S_OK(filename) @@ -570,19 +470,6 @@ def deleteProxyBundle(self, idList): rpcClient = Client(url="Framework/ProxyManager", timeout=120) return rpcClient.deleteProxyBundle(idList) - def requestToken(self, requesterDN, requesterGroup, numUses=1): - """Request a number of tokens. usesList must be a list of integers and each integer is the number of uses a token - must have - - :param str requesterDN: user DN - :param str requesterGroup: user group - :param int numUses: number of uses - - :return: S_OK(tuple)/S_ERROR() -- tuple contain token, number uses - """ - rpcClient = Client(url="Framework/ProxyManager", timeout=120) - return rpcClient.generateToken(requesterDN, requesterGroup, numUses) - def renewProxy(self, proxyToBeRenewed=None, minLifeTime=3600, newProxyLifeTime=43200, proxyToConnect=None): """Renew a proxy using the ProxyManager @@ -655,7 +542,14 @@ def renewProxy(self, proxyToBeRenewed=None, minLifeTime=3600, newProxyLifeTime=4 chain = retVal["Value"] if not proxyToRenewDict["tempFile"]: - return chain.dumpAllToFile(proxyToRenewDict["file"]) + filename = proxyToRenewDict["file"] + if not (result := chain.dumpAllToFile(filename))["OK"]: + return result + if not (result := chain.getDIRACGroup())["OK"]: + return result + if not (result := addTokenToPEM(filename, result["Value"]))["OK"]: # pylint: disable=unsubscriptable-object + return result + return S_OK(filename) return S_OK(chain) @@ -678,17 +572,14 @@ def getVOMSAttributes(self, chain): """ return VOMS().getVOMSAttributes(chain) - def getUploadedProxyLifeTime(self, DN, group=None): + def getUploadedProxyLifeTime(self, DN): """Get the remaining seconds for an uploaded proxy :param str DN: user DN - :param str group: group :return: S_OK(int)/S_ERROR() """ parameters = dict(UserDN=[DN]) - if group: - parameters["UserGroup"] = [group] result = self.getDBContents(parameters) if not result["OK"]: return result @@ -697,10 +588,9 @@ def getUploadedProxyLifeTime(self, DN, group=None): return S_OK(0) pNames = list(data["ParameterNames"]) dnPos = pNames.index("UserDN") - groupPos = pNames.index("UserGroup") expiryPos = pNames.index("ExpirationTime") for row in data["Records"]: - if DN == row[dnPos] and group == row[groupPos]: + if DN == row[dnPos]: td = row[expiryPos] - datetime.datetime.utcnow() secondsLeft = td.days * 86400 + td.seconds return S_OK(max(0, secondsLeft)) diff --git a/src/DIRAC/FrameworkSystem/Client/ProxyUpload.py b/src/DIRAC/FrameworkSystem/Client/ProxyUpload.py index f0528033a08..834ef9ff571 100644 --- a/src/DIRAC/FrameworkSystem/Client/ProxyUpload.py +++ b/src/DIRAC/FrameworkSystem/Client/ProxyUpload.py @@ -7,7 +7,6 @@ class CLIParams: - proxyLifeTime = 2592000 certLoc = False keyLoc = False @@ -23,7 +22,7 @@ def __str__(self): data.append("userPasswd = *****") else: data.append(f"{k}={getattr(self, k)}") - msg = "" % " ".join(data) + msg = f"" return msg def setProxyLifeTime(self, arg): @@ -31,7 +30,7 @@ def setProxyLifeTime(self, arg): fields = [f.strip() for f in arg.split(":")] self.proxyLifeTime = int(fields[0]) * 3600 + int(fields[1]) * 60 except ValueError: - gLogger.notice("Can't parse %s time! Is it a HH:MM?" % arg) + gLogger.notice(f"Can't parse {arg} time! Is it a HH:MM?") return DIRAC.S_ERROR("Can't parse time argument") return DIRAC.S_OK() @@ -106,8 +105,8 @@ def uploadProxy(params): if not keyLoc: keyLoc = cakLoc[1] - DIRAC.gLogger.info("Cert file %s" % certLoc) - DIRAC.gLogger.info("Key file %s" % keyLoc) + DIRAC.gLogger.info(f"Cert file {certLoc}") + DIRAC.gLogger.info(f"Key file {keyLoc}") testChain = X509Chain() retVal = testChain.loadKeyFromFile(keyLoc, password=params.userPasswd) @@ -126,10 +125,10 @@ def uploadProxy(params): # Load user cert and key retVal = chain.loadChainFromFile(certLoc) if not retVal["OK"]: - return S_ERROR("Can't load %s" % certLoc) + return S_ERROR(f"Can't load {certLoc}") retVal = chain.loadKeyFromFile(keyLoc, password=params.userPasswd) if not retVal["OK"]: - return S_ERROR("Can't load %s" % keyLoc) + return S_ERROR(f"Can't load {keyLoc}") DIRAC.gLogger.info("User credentials loaded") restrictLifeTime = params.proxyLifeTime @@ -137,7 +136,7 @@ def uploadProxy(params): proxyChain = X509Chain() retVal = proxyChain.loadProxyFromFile(proxyLoc) if not retVal["OK"]: - return S_ERROR("Can't load proxy file {}: {}".format(params.proxyLoc, retVal["Message"])) + return S_ERROR(f"Can't load proxy file {params.proxyLoc}: {retVal['Message']}") chain = proxyChain restrictLifeTime = 0 diff --git a/src/DIRAC/FrameworkSystem/Client/SecurityLogClient.py b/src/DIRAC/FrameworkSystem/Client/SecurityLogClient.py index 388b78a68fb..d4231831cda 100644 --- a/src/DIRAC/FrameworkSystem/Client/SecurityLogClient.py +++ b/src/DIRAC/FrameworkSystem/Client/SecurityLogClient.py @@ -9,7 +9,6 @@ class SecurityLogClient: - __securityLogStore = [] def __init__(self): diff --git a/src/DIRAC/FrameworkSystem/Client/SystemAdministratorClientCLI.py b/src/DIRAC/FrameworkSystem/Client/SystemAdministratorClientCLI.py index 1c8fb304352..9b65a237518 100644 --- a/src/DIRAC/FrameworkSystem/Client/SystemAdministratorClientCLI.py +++ b/src/DIRAC/FrameworkSystem/Client/SystemAdministratorClientCLI.py @@ -2,40 +2,45 @@ ######################################################################## """ System Administrator Client Command Line Interface """ -import sys -import pprint -import os import atexit -import readline import datetime +import os +import pprint +import readline +import sys import time from DIRAC import gConfig, gLogger from DIRAC.Core.Base.CLI import CLI, colorize -from DIRAC.FrameworkSystem.Client.SystemAdministratorClient import SystemAdministratorClient -from DIRAC.FrameworkSystem.Client.SystemAdministratorIntegrator import SystemAdministratorIntegrator -from DIRAC.FrameworkSystem.Client.ComponentMonitoringClient import ComponentMonitoringClient -from DIRAC.FrameworkSystem.Utilities import MonitoringUtilities -from DIRAC.MonitoringSystem.Client.MonitoringClient import MonitoringClient -from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller -from DIRAC.Core.Utilities.Extensions import extensionsByPriority +from DIRAC.Core.Security.ProxyInfo import getProxyInfo from DIRAC.Core.Utilities import List -from DIRAC.Core.Utilities.PromptUser import promptUser -from DIRAC.Core.Utilities.PrettyPrint import printTable +from DIRAC.Core.Utilities.Extensions import extensionsByPriority from DIRAC.Core.Utilities.File import mkDir -from DIRAC.Core.Security.ProxyInfo import getProxyInfo +from DIRAC.Core.Utilities.PrettyPrint import printTable +from DIRAC.Core.Utilities.PromptUser import promptUser +from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller +from DIRAC.FrameworkSystem.Client.ComponentMonitoringClient import ( + ComponentMonitoringClient, +) +from DIRAC.FrameworkSystem.Client.SystemAdministratorClient import ( + SystemAdministratorClient, +) +from DIRAC.FrameworkSystem.Client.SystemAdministratorIntegrator import ( + SystemAdministratorIntegrator, +) +from DIRAC.FrameworkSystem.Utilities import MonitoringUtilities +from DIRAC.MonitoringSystem.Client.MonitoringClient import MonitoringClient class SystemAdministratorClientCLI(CLI): """Line oriented command interpreter for administering DIRAC components""" def __init__(self, host=None): - CLI.__init__(self) # Check if Port is given self.host = None self.port = None - self.prompt = "[%s]> " % colorize("no host", "yellow") + self.prompt = f"[{colorize('no host', 'yellow')}]> " if host: self.__setHost(host) self.cwd = "" @@ -45,7 +50,7 @@ def __init__(self, host=None): # store history histfilename = os.path.basename(sys.argv[0]) - historyFile = os.path.expanduser("~/.dirac/%s.history" % histfilename[0:-3]) + historyFile = os.path.expanduser(f"~/.dirac/{histfilename[0:-3]}.history") mkDir(os.path.dirname(historyFile)) if os.path.isfile(historyFile): readline.read_history_file(historyFile) @@ -59,14 +64,14 @@ def __setHost(self, host): self.port = hostList[1] else: self.port = None - gLogger.notice("Pinging %s..." % self.host) + gLogger.notice(f"Pinging {self.host}...") result = self.__getClient().ping() if result["OK"]: colorHost = colorize(host, "green") else: - self._errMsg("Could not connect to {}: {}".format(self.host, result["Message"])) + self._errMsg(f"Could not connect to {self.host}: {result['Message']}") colorHost = colorize(host, "red") - self.prompt = "[%s]> " % colorHost + self.prompt = f"[{colorHost}]> " def __getClient(self): return SystemAdministratorClient(self.host, self.port) @@ -110,9 +115,9 @@ def __do_set_project(self, args): project = args[0] result = self.__getClient().setProject(project) if not result["OK"]: - self._errMsg("Cannot set project: %s" % result["Message"]) + self._errMsg(f"Cannot set project: {result['Message']}") else: - gLogger.notice("Project set to %s" % project) + gLogger.notice(f"Project set to {project}") def do_show(self, args): """ @@ -183,7 +188,7 @@ def do_show(self, args): if not result["OK"]: self._errMsg(result["Message"]) else: - gLogger.notice("Current project is %s" % result["Value"]) + gLogger.notice(f"Current project is {result['Value']}") elif option == "status": client = SystemAdministratorClient(self.host, self.port) result = client.getOverallStatus() @@ -214,7 +219,7 @@ def do_show(self, args): record += [str(rDict[compType][system][component]["PID"])] records.append(record) printTable(fields, records) - elif option == "database" or option == "databases": + elif option in ("database", "databases"): client = SystemAdministratorClient(self.host, self.port) if not gComponentInstaller.mysqlPassword: gComponentInstaller.mysqlPassword = "LocalConfig" @@ -260,7 +265,7 @@ def do_show(self, args): gLogger.notice("") gLogger.notice("Setup:", result["Value"]["Setup"]) for e, v in result["Value"]["Extensions"].items(): - gLogger.notice("%s version" % e, v) + gLogger.notice(f"{e} version", v) gLogger.notice("") elif option == "host": client = SystemAdministratorClient(self.host, self.port) @@ -279,7 +284,7 @@ def do_show(self, args): extensions = parameter[1].split(",") for extension in extensions: extensionName, extensionVersion = extension.split(":") - records.append(["%sVersion" % extensionName, str(extensionVersion)]) + records.append([f"{extensionName}Version", str(extensionVersion)]) else: records.append([parameter[0], str(parameter[1])]) @@ -288,7 +293,7 @@ def do_show(self, args): client = ComponentMonitoringClient() result = client.getHosts({}, False, False) if not result["OK"]: - self._errMsg("Error retrieving the list of hosts: %s" % (result["Message"])) + self._errMsg(f"Error retrieving the list of hosts: {result['Message']}") else: hostList = result["Value"] gLogger.notice("") @@ -458,9 +463,9 @@ def getInstallations(self, argss): elif arg == "-t": key = "Component.Type" elif arg == "-m": - key = "Component.Module" + key = "Component.DIRACModule" elif arg == "-s": - key = "Component.System" + key = "Component.DIRACSystem" elif arg == "-h": key = "Host.HostName" elif arg == "-n": @@ -487,7 +492,7 @@ def getInstallations(self, argss): client = ComponentMonitoringClient() result = client.getInstallations(installationFilter, componentFilter, hostFilter, True) if not result["OK"]: - self._errMsg("Could not retrieve the installations: %s" % (result["Message"])) + self._errMsg(f"Could not retrieve the installations: {result['Message']}") installations = None else: installations = result["Value"] @@ -617,7 +622,13 @@ def do_install(self, args): install agent [-m ] [-p