Switch to cgroup for job control and associated settings (#205)

* Change default ProctrackType, SelectTypeParameters,TaskPlugin

- switching ProctrackType to proctrack/cgroup, which is the recommended setting
  (was disabled for CI)
- Enabling TaskPlugins task/cgroup,task/affinity to bind jobs to
  allocated cpuset
- SelectTypeParameters=CR_Core_Memory to also limit memory

* test cgroup enabled

* Enable the cpuset controller in the VM

enable cpuset for whole user.slice hierarchy

* Running podman pull in RL8 tests to create the missing session.slice

in cgroup. So that we can enable the cpuset controller for it.

* Switch to jobacct_gather/cgroup

* [skipci] molecule update readme

* Go back to SelectTypeParameters=CR_Core

* Wait for job to be running before checking cgroup properties

* molecule: add name to a task

Co-authored-by: Steve Brasier <33413598+sjpb@users.noreply.github.com>

* molecule: remove cgroup memory limit test

---------

Co-authored-by: Steve Brasier <33413598+sjpb@users.noreply.github.com>

Author: elelaysh

.github/workflows/ci.yml

@@ -79,6 +79,10 @@ jobs:
         working-directory: molecule/images
         if: matrix.image == 'localhost/rocky9systemd'
 
+      - name: Load rocky8 container image
+        run: podman pull ${{ matrix.image }}
+        if: matrix.image != 'localhost/rocky9systemd'
+
       - name: Set up Python 3.
         uses: actions/setup-python@v4
         with:
@@ -99,6 +103,22 @@ jobs:
       - name: Create ansible.cfg with correct roles_path
         run:  printf '[defaults]\nroles_path=../' >ansible.cfg
 
+      - name: Enable the cpuset controller
+        run: |
+          echo "======== BEFORE ========"
+          find /sys/fs/cgroup/ -name cgroup.subtree_control -exec grep -H '' '{}' ';'
+          echo "==== CHANGING CPUSET ==="
+          set -x
+          echo +cpuset | sudo tee  /sys/fs/cgroup/user.slice/cgroup.subtree_control
+          echo +cpuset | sudo tee  /sys/fs/cgroup/user.slice/user-1001.slice/cgroup.subtree_control
+          echo +cpuset | sudo tee  /sys/fs/cgroup/user.slice/user-1001.slice/user@1001.service/cgroup.subtree_control
+          echo +cpuset | sudo tee  /sys/fs/cgroup/user.slice/user-1001.slice/user@1001.service/session.slice/cgroup.subtree_control
+          echo +cpuset | sudo tee  /sys/fs/cgroup/user.slice/user-1001.slice/user@1001.service/user.slice/cgroup.subtree_control
+          echo +cpuset | sudo tee  /sys/fs/cgroup/user.slice/user-1001.slice/user@1001.service/app.slice/cgroup.subtree_control
+          set +x
+          echo "======= CHECKING ======="
+          find /sys/fs/cgroup/ -name cgroup.subtree_control -exec grep -H '' '{}' ';'
+
       - name: Run Molecule tests.
         run: molecule test -s ${{ matrix.scenario }}
         env:

defaults/main.yml

@@ -16,7 +16,8 @@ openhpc_gres_autodetect: 'off'
 openhpc_default_config:
   # This only defines values which are not Slurm defaults
   SlurmctldHost: "{{ openhpc_slurm_control_host }}{% if openhpc_slurm_control_host_address is defined %}({{ openhpc_slurm_control_host_address }}){% endif %}"
-  ProctrackType: proctrack/linuxproc # TODO: really want cgroup but needs cgroup.conf and workaround for CI
+  ProctrackType: proctrack/cgroup
+  TaskPlugin: task/cgroup,task/affinity
   SlurmdSpoolDir: /var/spool/slurm # NB: not OpenHPC default!
   SlurmUser: slurm
   StateSaveLocation: "{{ openhpc_state_save_location }}"
@@ -80,7 +81,7 @@ openhpc_slurm_accounting_storage_user: slurm
 #openhpc_slurm_accounting_storage_pass:
 
 # Job accounting
-openhpc_slurm_job_acct_gather_type: jobacct_gather/linux
+openhpc_slurm_job_acct_gather_type: jobacct_gather/cgroup
 openhpc_slurm_job_acct_gather_frequency: 30
 openhpc_slurm_job_comp_type: jobcomp/none
 openhpc_slurm_job_comp_loc: /var/log/slurm_jobacct.log

molecule/README.md

@@ -47,9 +47,11 @@ Build a Rocky Linux 9 image with systemd included:
 Run tests, e.g.:
 
     cd ansible-role-openhpc/
-    MOLECULE_NO_LOG="false" MOLECULE_IMAGE=rockylinux:8 molecule test --all
+    MOLECULE_NO_LOG="false" MOLECULE_IMAGE=rockylinux/rockylinux:8 molecule test --all
 
-where the image may be `rockylinux:8` or `localhost/rocky9systemd`.
+where the image may be `rockylinux/rockylinux:8` or `localhost/rocky9systemd`.
+
+Tested with version 8.7.0 of `ansible`, 2.15.13 of `ansible-core` (installed when python version is 3.9).
 
 Other useful options during development:
 - Prevent destroying instances by using `molecule test --destroy never`

molecule/test1/verify.yml

@@ -6,7 +6,28 @@
   - name: Get slurm partition info
     command: sinfo --noheader --format="%P,%a,%l,%D,%t,%N" # using --format ensures we control whitespace
     register: sinfo
-  - name: 
+  - name: Check nodes are up/idle
     assert:                        # PARTITION AVAIL  TIMELIMIT  NODES  STATE NODELIST
       that: "sinfo.stdout_lines == ['compute*,up,60-00:00:00,2,idle,testohpc-compute-[0-1]']"
       fail_msg: "FAILED - actual value: {{ sinfo.stdout_lines }}"
+
+- name: Run limited job and check limits
+  hosts: testohpc-compute-0
+  tasks:
+  - name: Assert expected SelectTypeParameters
+    ansible.builtin.assert:
+      that: ansible_local.slurm.SelectTypeParameters in ('CR_CORE', 'CR_CORE_MEMORY')
+  - ansible.builtin.shell: |
+      printf '#!bin/bash\nsleep 500' | sbatch --cpus-per-task=1 --ntasks=1 --mem=200m --nodelist=testohpc-compute-0
+      retry=0
+      while [[ "$retry" -lt 10 ]] && ! squeue -l | grep -q RUNNING; do
+        sleep 1
+        retry=$(( retry + 1))
+      done
+  - name: Get cpuset cgroup limit
+    shell: cat /sys/fs/cgroup/system.slice/testohpc-compute-0_slurmstepd.scope/job_*/cpuset.cpus
+    register: job_cpuset
+  - name: Assert cpuset cgroup presence
+    assert:
+      that: "job_cpuset.stdout_lines[0] in ('0', '0-1')"  # depending on the VM's state
+      fail_msg: "FAILED - actual value: {{ job_cpuset.stdout_lines }}"