Compute AccountingStorageTRES

Author: elelaysh

defaults/main.yml

@@ -33,6 +33,7 @@ openhpc_default_config:
   AccountingStoragePort: "{{ openhpc_slurm_accounting_storage_port }}"
   AccountingStorageType: "{{ openhpc_slurm_accounting_storage_type | default('omit') }}"
   AccountingStorageUser: "{{ openhpc_slurm_accounting_storage_user }}"
+  AccountingStorageTRES: "{{ ohpc_accounting_storage_tres | trim | default('omit', true) }}"
   JobCompLoc: "{{ openhpc_slurm_job_comp_loc }}"
   JobCompType: "{{ openhpc_slurm_job_comp_type }}"
   JobAcctGatherFrequency: "{{ openhpc_slurm_job_acct_gather_frequency }}"

templates/slurm.conf.j2

@@ -36,10 +36,8 @@ NodeName={{ hostlists | join(',') }} {{ '' -}}
     CoresPerSocket={{ first_host_hv['ansible_processor_cores'] }} {{ '' -}}
     ThreadsPerCore={{ first_host_hv['ansible_processor_threads_per_core'] }} {{ '' -}}
     {{ nodegroup.node_params | default({}) | dict2parameters }} {{ '' -}}
-    {% if 'gres' in nodegroup -%}
-    Gres={{ ','.join(nodegroup.gres | map(attribute='conf')) -}}
-    {% elif nodegroup.gres_autodetect | default(openhpc_gres_autodetect) == 'nvml' and first_host_hv['ohpc_node_gpu_gres'] != '' -%}
-    Gres={{ first_host_hv['ohpc_node_gpu_gres'] -}}
+    {% if 'gres' in nodegroup_computed -%}
+    Gres={{ nodegroup_computed.gres -}}
     {% endif %}
 
 {%  endif %}{# 1 or more hosts in inventory #}

vars/main.yml

@@ -35,9 +35,27 @@ ohpc_nodegroups_computed: >
           "first_host": {{ first_host | to_json }},
           "ram_mb": {{ ram_mb }},
           "def_mem_per_cpu": {{ (ram_mb / first_host_hv['ansible_processor_vcpus']) | int }},
+  {%         if 'gres' in nodegroup -%}
+          "gres": {{ ','.join(nodegroup.gres | map(attribute='conf')) | to_json }},
+  {%         elif nodegroup.gres_autodetect | default(openhpc_gres_autodetect) == 'nvml' and first_host_hv['ohpc_node_gpu_gres'] != '' -%}
+          "gres": {{ first_host_hv['ohpc_node_gpu_gres'] | to_json }},
+          {% endif %}
       },
   {%      endif %}
   {% endfor %}
   }
 
+ohpc_accounting_storage_tres: >-
+  {%- set tres =
+      ohpc_nodegroups_computed.values()
+      | selectattr('gres', 'defined') | map(attribute='gres')
+      | map('regex_replace', ':[0-9]+(,|$)', '\\1')
+      | map('split', ',')
+      | flatten | sort | unique
+      | map('regex_replace', '^', 'gres/')
+      | join(',')
+  -%}
+  {# add the generic gres/gpu otherwise a job requesting `--gpus 1` or `--gres=gpu:1` will be assigned to a "random" gpu model #}
+  {{ ('gres/gpu:' in tres and not tres is search('gres/gpu(,|$)')) | ternary('gres/gpu,' ~ tres, tres) }}
+
 ...