merge develop into main for release v0.7.0

.github/workflows/tests_scripts.yml

@@ -0,0 +1,39 @@
+# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
+name: Tests for scripts
+on:
+  push:
+    paths:
+      - scripts/sign_verify_file_ssh.sh
+  pull_request:
+    paths:
+      - scripts/sign_verify_file_ssh.sh
+permissions:
+  contents: read # to fetch code (actions/checkout)
+jobs:
+  build:
+    runs-on: ubuntu-24.04
+    steps:
+    - name: checkout
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+    - name: test sign_verify_file_ssh.sh script
+      run: |
+          # Create a PEM format ssh identity
+          ssh-keygen -t rsa -b 4096 -m PEM -f id_rsa.pem -N ""
+          # Create a file to sign
+          echo "Very important stuff" > out.txt
+          export FILE_TO_SIGN="out.txt"
+          # Sign the file
+          ./scripts/sign_verify_file_ssh.sh sign id_rsa.pem "$FILE_TO_SIGN"
+          # Create an allowed_signers file based on the public key
+          echo -n "allowed_identity " > allowed_signers
+          cat id_rsa.pem.pub >> allowed_signers
+          # Verify the signature
+          ./scripts/sign_verify_file_ssh.sh verify allowed_signers "$FILE_TO_SIGN"
+          # Make a new signature that does not appear in the allowed signers file
+          ssh-keygen -t rsa -b 4096 -m PEM -f id_rsa.alt.pem -N ""
+          # Replace the allowed signers file
+          echo -n "disallowed_identity " > allowed_signers
+          cat id_rsa.alt.pem.pub >> allowed_signers
+          # Make sure signature checking fails in this case
+          ./scripts/sign_verify_file_ssh.sh verify allowed_signers "$FILE_TO_SIGN" && exit 1 || echo "Expected failure for unknown identity"

RELEASE_NOTES

@@ -1,6 +1,37 @@
 This file contains a description of the major changes to the EESSI
 build-and-deploy bot. For more detailed information, please see the git log.
 
+v0.7.0 (13 March 2025)
+--------------------------
+
+This is a minor release of the EESSI build-and-deploy bot.
+
+Bug fixes:
+* bot only reports moving to trash_bin when relevant (#292)
+
+Improvements:
+* add support for specifying that build job script is located in another repository (#283)
+* implement exportvariable filter (#288, #291)
+  * see related configuration setting `allowed_exportvars`
+* add alternative method to submit job (using `--begin=now+SOME_DELAY`) (#297)
+  * also see the new related configuration settings `job_handover_protocol` and `job_delay_begin_factor`
+* set the local_tmp that is configured for a site as tmpdir in bot build job script (#299)
+* add setting for a script to customize build environment (#302)
+* add support for signing tarball and metadata file and uploading signatures to S3 bucket (#303)
+* add SSH signing script `sign_verify_file_ssh.sh` (#304)
+* updates of the docs (#293, #298)
+
+Changes to 'app.cfg' settings (see README.md and app.cfg.example for details):
+* NEW (optional) 'allowed_exportvars' in section '[buildenv]'
+* NEW (required) 'awaits_release_hold_release_msg' in section '[submitted_job_comments]'
+* NEW (required) 'awaits_release_hold_release_msg' in section '[submitted_job_comments]'
+* DEPRECATED (optional) 'awaits_release' in section '[submitted_job_comments]'
+* NEW (optional) 'job_delay_begin_factor' in section '[buildenv]'
+* NEW (required) 'job_handover_protocol' in section '[buildenv]'
+* NEW (optional) 'signing' in section '[deploycfg]'
+* NEW (optional) 'site_config_script' in section '[buildenv]'
+
+
 v0.6.0 (18 September 2024)
 --------------------------
 

app.cfg.example

@@ -88,6 +88,25 @@ container_cachedir = PATH_TO_SHARED_DIRECTORY
 # http_proxy = http://PROXY_DNS:3128/
 # https_proxy = http://PROXY_DNS:3128/
 
+# The job_delay_begin_factor setting defines how many times the poll_interval a
+# job's begin (EligibleTime) from now should be delayed if the handover protocol
+# is set to `delayed_begin` (see setting `job_handover_protocol`). That is, if
+# the job_delay_begin_factor is set to five (5) the delay time is calculated as
+# 5 * poll_interval. The event manager would use 2 as the default factor when
+# submitting jobs.
+job_delay_begin_factor = 2
+
+# The job_handover_protocol setting defines which method is used to handover a
+# job from the event handler to the job manager. Values are
+#  - hold_release (job is submitted with '--hold', job manager removes the hold
+#    with 'scontrol release')
+#  - delayed_begin (job is submitted with '--begin=now+(5 * poll_interval)' and
+#    any '--hold' is removed from the submission parameters); this is useful if the
+#    bot account cannot run 'scontrol release' to remove the hold of the job;
+#    also, the status update in the PR comment of the job is extended by noting
+#    the 'EligibleTime'
+job_handover_protocol = hold_release
+
 # Used to give all jobs of a bot instance the same name. Can be used to allow
 # multiple bot instances running on the same Slurm cluster.
 job_name = prod
@@ -110,6 +129,15 @@ load_modules =
 #     variables that are only set inside a Slurm job
 local_tmp = /tmp/$USER/EESSI
 
+# PATH to a script that - if it exists - is sourced in the build job
+#     before any 'bot/*' script is run. This allows to customize the
+#     build environment due to specifics of the build site/cluster.
+#     Note, such customizations could also be performed by putting them
+#     into a module file and using the setting 'load_modules' (see above).
+#     However, the setting 'site_config_script' provides a low threshold
+#     for achieving this, too.
+site_config_script = /path/to/script/if/any
+
 # parameters to be added to all job submissions
 # NOTE do not quote parameter string. Quotes are retained when reading in config and
 #      then the whole 'string' is recognised as a single parameter.
@@ -132,6 +160,19 @@ no_build_permission_comment = Label `bot:build` has been set by user `{build_lab
 # whether or not to allow updating the submit options via custom module det_submit_opts
 allow_update_submit_opts = false
 
+# defines which name-value pairs (environment variables) are allowed to be
+# exported into the build environment via 'exportvariable' filters
+# The bot build script makes use of the variable 'SKIP_TESTS' to determine if
+# ReFrame tests shall be skipped or not. Default value is 'no'. If the value is
+# 'yes' and the exportvariable filter is added to a bot build command
+# ('export:SKIP_TESTS=yes'), ReFrame tests are skipped.
+# NOTE, the setting is optional and commented by default. If you want to enable
+# this feature ('exportvariable' filters), uncomment the line below and define
+# meaningful key-value pair(s). For example, to enable the use of
+# 'exportvariable:SKIP_TESTS=yes' as a filter, the key-value pair would be
+# "SKIP_TESTS=yes".
+# allowed_exportvars = ["NAME1=value_1a", "NAME1=value_1b", "NAME2=value_2"]
+
 
 [deploycfg]
 # script for uploading built software packages
@@ -153,6 +194,31 @@ endpoint_url = URL_TO_S3_SERVER
 # like: bucket_name = {"eessi-pilot-2023.06": "eessi-staging-pilot-2023.06", "eessi.io-2023.06": "software.eessi.io-2023.06"}
 bucket_name = eessi-staging
 
+# settings for signing artefacts with JSON-like format
+#   REPO_ID: { "script": PATH_TO_SIGN_SCRIPT, "key": PATH_TO_KEY_FILE, "container_runtime": PATH_TO_CONTAINER_RUNTIME }
+#  If PATH_TO_SIGN_SCRIPT is a relative path, the script must reside in the
+#  checked out pull request of the target repository (e.g.,
+#  EESSI/software-layer).
+#  The bot calls the script with the two arguments:
+#   1. private key (as provided by the attribute 'key')
+#   2. path to the file to be signed (the upload script will determine that)
+# NOTE (on "container_runtime"), signing requires a recent installation of OpenSSH
+#   (8.2 or newer). If the frontend where the event handler runs does not have that
+#   version installed, you can specify a container runtime via the 'container_runtime'
+#   attribute below. Currently, only Singularity or Apptainer are supported.
+# NOTE (on the key), make sure the file permissions are restricted to `0600` (only
+#   readable+writable by the file owner, or the signing will likely fail.
+# Note (on json format), make sure no trailing commas are used after any elements
+#   or parsing/loading the json will likely fail. Also, the whole value should start
+#   at a new line and be indented as shown below.
+signing =
+    {
+        "eessi.io-2023.06-software: {
+            "script": PATH_TO_SIGN_SCRIPT,
+            "key": PATH_TO_EESSI_BOT/config/user-site-system.key,
+            "container_runtime": PATH_TO_CONTAINER_RUNTIME
+        }
+    }
 # upload policy: defines what policy is used for uploading built artefacts
 #                to an S3 bucket
 # 'all' ..: upload all artefacts (mulitple uploads of the same artefact possible)
@@ -244,12 +310,14 @@ scontrol_command = /usr/bin/scontrol
 #         information.
 [submitted_job_comments]
 awaits_release = job id `{job_id}` awaits release by job manager
+awaits_release_delayed_begin_msg = job id `{job_id}` will be eligible to start in about {delay_seconds} seconds
+awaits_release_hold_release_msg = job id `{job_id}` awaits release by job manager
 initial_comment = New job on instance `{app_name}` for CPU micro-architecture `{arch_name}`{accelerator_spec} for repository `{repo_id}` in job dir `{symlink}`
 with_accelerator =  and accelerator `{accelerator}`
 
 
 [new_job_comments]
-awaits_launch = job awaits launch by Slurm scheduler 
+awaits_launch = job awaits launch by Slurm scheduler{extra_info} 
 
 [running_job_comments]
 running_job = job `{job_id}` is running

eessi_bot_event_handler.py

@@ -56,6 +56,8 @@
         # config.BUILDENV_SETTING_CVMFS_CUSTOMIZATIONS,              # optional
         # config.BUILDENV_SETTING_HTTPS_PROXY,                       # optional
         # config.BUILDENV_SETTING_HTTP_PROXY,                        # optional
+        # config.BUILDENV_SETTING_JOB_DELAY_BEGIN_FACTOR,            # optional (default: 2)
+        config.BUILDENV_SETTING_JOB_HANDOVER_PROTOCOL,             # required
         config.BUILDENV_SETTING_JOB_NAME,                          # required
         config.BUILDENV_SETTING_JOBS_BASE_DIR,                     # required
         # config.BUILDENV_SETTING_LOAD_MODULES,                      # optional
@@ -75,6 +77,7 @@
         # config.DEPLOYCFG_SETTING_ENDPOINT_URL,                     # optional
         config.DEPLOYCFG_SETTING_METADATA_PREFIX,                  # (required)
         config.DEPLOYCFG_SETTING_NO_DEPLOY_PERMISSION_COMMENT,     # required
+        # config.DEPLOYCFG_SETTING_SIGNING,                          # optional
         config.DEPLOYCFG_SETTING_UPLOAD_POLICY],                   # required
     config.SECTION_DOWNLOAD_PR_COMMENTS: [
         config.DOWNLOAD_PR_COMMENTS_SETTING_CURL_FAILURE,          # required
@@ -92,12 +95,18 @@
         config.GITHUB_SETTING_APP_NAME,                            # required
         config.GITHUB_SETTING_INSTALLATION_ID,                     # required
         config.GITHUB_SETTING_PRIVATE_KEY],                        # required
+    # the poll interval setting is required for the alternative job handover
+    # protocol (delayed_begin)
+    config.SECTION_JOB_MANAGER: [
+        config.JOB_MANAGER_SETTING_POLL_INTERVAL],                 # required
     config.SECTION_REPO_TARGETS: [
         config.REPO_TARGETS_SETTING_REPO_TARGET_MAP,               # required
         config.REPO_TARGETS_SETTING_REPOS_CFG_DIR],                # required
     config.SECTION_SUBMITTED_JOB_COMMENTS: [
         config.SUBMITTED_JOB_COMMENTS_SETTING_INITIAL_COMMENT,     # required
-        config.SUBMITTED_JOB_COMMENTS_SETTING_AWAITS_RELEASE,      # required
+        # config.SUBMITTED_JOB_COMMENTS_SETTING_AWAITS_RELEASE,      # optional
+        config.SUBMITTED_JOB_COMMENTS_SETTING_AWAITS_RELEASE_DELAYED_BEGIN_MSG,  # required
+        config.SUBMITTED_JOB_COMMENTS_SETTING_AWAITS_RELEASE_HOLD_RELEASE_MSG,   # required
         config.SUBMITTED_JOB_COMMENTS_SETTING_WITH_ACCELERATOR],   # required
     }
 
@@ -636,33 +645,37 @@ def handle_pull_request_closed_event(self, event_info, pr):
         self.log(f"PR {pr.number}: determining directories to be moved to trash bin")
         job_dirs = determine_job_dirs(pr.number)
 
-        # 2) Get trash_bin_dir from configs
-        trash_bin_root_dir = self.cfg[config.SECTION_CLEAN_UP][config.CLEAN_UP_SETTING_TRASH_BIN_ROOT_DIR]
-
-        repo_name = request_body['repository']['full_name']
-        dt_start = datetime.now(timezone.utc)
-        trash_bin_dir = "/".join([trash_bin_root_dir, repo_name, dt_start.strftime('%Y.%m.%d')])
-
-        # Subdirectory with date of move. Also with repository name. Handle symbolic links (later?)
-        # cron job deletes symlinks?
-
-        # 3) move the directories to the trash_bin
-        self.log(f"PR {pr.number}: moving directories to trash bin {trash_bin_dir}")
-        move_to_trash_bin(trash_bin_dir, job_dirs)
-        dt_end = datetime.now(timezone.utc)
-        dt_delta = dt_end - dt_start
-        seconds_elapsed = dt_delta.days * 24 * 3600 + dt_delta.seconds
-        self.log(f"PR {pr.number}: moved directories to trash bin {trash_bin_dir} (took {seconds_elapsed} seconds)")
+        if job_dirs == []:
+            self.log(f"PR {pr.number}: No job directories found; nothing to move.")
+        else:
+            # 2) Get trash_bin_dir from configs
+            trash_bin_root_dir = self.cfg[config.SECTION_CLEAN_UP][config.CLEAN_UP_SETTING_TRASH_BIN_ROOT_DIR]
 
-        # 4) report move to pull request
-        repo_name = pr.base.repo.full_name
-        gh = github.get_instance()
-        repo = gh.get_repo(repo_name)
-        pull_request = repo.get_pull(pr.number)
-        clean_up_comment = self.cfg[config.SECTION_CLEAN_UP][config.CLEAN_UP_SETTING_MOVED_JOB_DIRS_COMMENT]
-        moved_comment = clean_up_comment.format(job_dirs=job_dirs, trash_bin_dir=trash_bin_dir)
-        issue_comment = pull_request.create_issue_comment(moved_comment)
-        return issue_comment
+            repo_name = request_body['repository']['full_name']
+            dt_start = datetime.now(timezone.utc)
+            trash_bin_dir = "/".join([trash_bin_root_dir, repo_name, dt_start.strftime('%Y.%m.%d')])
+
+            # Subdirectory with date of move. Also with repository name. Handle symbolic links (later?)
+            # cron job deletes symlinks?
+
+            # 3) move the directories to the trash_bin
+            self.log(f"PR {pr.number}: moving directories to trash bin {trash_bin_dir}")
+            move_to_trash_bin(trash_bin_dir, job_dirs)
+            dt_end = datetime.now(timezone.utc)
+            dt_delta = dt_end - dt_start
+            seconds_elapsed = dt_delta.days * 24 * 3600 + dt_delta.seconds
+            self.log(f"PR {pr.number}: moved directories to trash bin {trash_bin_dir} (took {seconds_elapsed} seconds)")
+
+            # 4) report move to pull request
+
+            repo_name = pr.base.repo.full_name
+            gh = github.get_instance()
+            repo = gh.get_repo(repo_name)
+            pull_request = repo.get_pull(pr.number)
+            clean_up_comment = self.cfg[config.SECTION_CLEAN_UP][config.CLEAN_UP_SETTING_MOVED_JOB_DIRS_COMMENT]
+            moved_comment = clean_up_comment.format(job_dirs=job_dirs, trash_bin_dir=trash_bin_dir)
+            issue_comment = pull_request.create_issue_comment(moved_comment)
+            return issue_comment
 
 
 def main():