Merge pull request #209 from stackhpc/feat/cr_core_memory

SelectTypeParameters=CR_Core_Memory; DefMemPerCPU

Author: elelaysh

defaults/main.yml

@@ -24,7 +24,7 @@ openhpc_default_config:
   SlurmctldTimeout: 300
   SchedulerType: sched/backfill
   SelectType: select/cons_tres
-  SelectTypeParameters: CR_Core
+  SelectTypeParameters: CR_Core_Memory
   PriorityWeightPartition: 1000
   PreemptType: preempt/partition_prio
   PreemptMode: SUSPEND,GANG
@@ -43,6 +43,7 @@ openhpc_default_config:
   Epilog: /etc/slurm/slurm.epilog.clean
   ReturnToService: 2
   GresTypes: "{{ ohpc_gres_types if ohpc_gres_types != '' else 'omit' }}"
+  DefMemPerCPU: "{{ ohpc_nodegroups_computed.values() | map(attribute='def_mem_per_cpu') | default([100], true) | min }}"
 openhpc_cgroup_default_config:
   ConstrainCores: "yes"
   ConstrainDevices: "yes"

templates/slurm.conf.j2

@@ -22,18 +22,16 @@ NodeName={{ node }}
 # COMPUTE NODES
 {% for nodegroup in openhpc_nodegroups %}
 # nodegroup: {{ nodegroup.name }}
-{%  set inventory_group_name = openhpc_cluster_name ~ '_' ~ nodegroup.name %}
-{%  set inventory_group_hosts = groups.get(inventory_group_name, []) %}
-{%      if inventory_group_hosts | length > 0 %}
-{%          set play_group_hosts = inventory_group_hosts | intersect (play_hosts) %}
-{%          set first_host = play_group_hosts | first | mandatory('Inventory group "' ~ inventory_group_name ~ '" contains no hosts in this play - was --limit used?') %}
-{%          set first_host_hv = hostvars[first_host] %}
-{%          set ram_mb = (first_host_hv['ansible_memory_mb']['real']['total'] * (nodegroup.ram_multiplier | default(openhpc_ram_multiplier))) | int %}
-{%          set hostlists = (inventory_group_hosts | hostlist_expression) %}{# hosts in inventory group aren't necessarily a single hostlist expression #}
+{%  set nodegroup_computed = ohpc_nodegroups_computed.get(nodegroup.name) %}
+{#  see vars/main.yml: if nodegroup_computed, nodegroup has at least 1 host in the inventory #}
+{%  if nodegroup_computed %}
+{%    set inventory_group_hosts = groups[nodegroup_computed.inventory_group_name] %}
+{%    set first_host_hv = hostvars[nodegroup_computed.first_host] %}
+{%    set hostlists = (inventory_group_hosts | hostlist_expression) %}{# hosts in inventory group aren't necessarily a single hostlist expression #}
 NodeName={{ hostlists | join(',') }} {{ '' -}}
     Features={{ (['nodegroup_' ~ nodegroup.name] + nodegroup.features | default([]) ) | join(',') }} {{ '' -}}
     State=UNKNOWN {{ '' -}}
-    RealMemory={{ nodegroup.ram_mb | default(ram_mb) }} {{ '' -}}
+    RealMemory={{ nodegroup.ram_mb | default(nodegroup_computed.ram_mb) }} {{ '' -}}
     Sockets={{ first_host_hv['ansible_processor_count'] }} {{ '' -}}
     CoresPerSocket={{ first_host_hv['ansible_processor_cores'] }} {{ '' -}}
     ThreadsPerCore={{ first_host_hv['ansible_processor_threads_per_core'] }} {{ '' -}}
@@ -44,7 +42,7 @@ NodeName={{ hostlists | join(',') }} {{ '' -}}
     Gres={{ first_host_hv['ohpc_node_gpu_gres'] -}}
     {% endif %}
 
-{%      endif %}{# 1 or more hosts in inventory #}
+{%  endif %}{# 1 or more hosts in inventory #}
 NodeSet=nodegroup_{{ nodegroup.name }} Feature=nodegroup_{{ nodegroup.name }}
 
 {% endfor %}

vars/main.yml

@@ -19,4 +19,25 @@ ohpc_slurm_packages:
     - "slurm-slurmdbd-ohpc"
 
 openhpc_merged_config: "{{ openhpc_default_config | combine(openhpc_config) }}"
+
+ohpc_nodegroups_computed: >
+  {
+  {% for nodegroup in openhpc_nodegroups %}
+  {%  set inventory_group_name = openhpc_cluster_name ~ '_' ~ nodegroup.name %}
+  {%  set inventory_group_hosts = groups.get(inventory_group_name, []) %}
+  {%      if inventory_group_hosts | length > 0 %}
+  {%          set play_group_hosts = inventory_group_hosts | intersect (play_hosts) %}
+  {%          set first_host = play_group_hosts | first | mandatory('Inventory group "' ~ inventory_group_name ~ '" contains no hosts in this play - was --limit used?') %}
+  {%          set first_host_hv = hostvars[first_host] %}
+  {%          set ram_mb = (first_host_hv['ansible_memory_mb']['real']['total'] * (nodegroup.ram_multiplier | default(openhpc_ram_multiplier))) | int %}
+      {{ nodegroup.name | to_json }}: {
+          "inventory_group_name": {{ inventory_group_name | to_json }},
+          "first_host": {{ first_host | to_json }},
+          "ram_mb": {{ ram_mb }},
+          "def_mem_per_cpu": {{ (ram_mb / first_host_hv['ansible_processor_vcpus']) | int }},
+      },
+  {%      endif %}
+  {% endfor %}
+  }
+
 ...