EH: CS-612 allow to overwrite the PE allocation rule on the submission command line (#77)

* EH: CS-612 allow to overwrite the PE allocation rule on the submission command line
// added command line switches and qstat output
// scheduler code still missing

* scheduler changes to overwrite the allocation rule by the global -par switch

* implemented handling of master and slave allocation rules in scheduler

* reject requesting allocation rule $pe_slots for master or slave scope

* - fixes for disjoint queues, fixed allocation rule and -scope master -par 1
- workaround for CS-1898 / CS-1899

* fixed category string for -scope slave -par ...

* - fixed issue with binding
- optimization (do not touch master host in second round of round_robin if it has been filled with a fixed alloc rule)
- added -par switch to man page

Author: jgabler-hpc

doc/markdown/man/man1/submit.include.md

@@ -1259,6 +1259,24 @@ If this option is specified and the priority is not 0 then this value will be pa
 instances as parameter with the name *p*. (see `-jsv` option above or find more information concerning JSV in 
 xxqs_name_sxx_jsv(1))
 
+## -par *allocation_rule*
+
+Available for `qsub`, `qsh`, `qrsh`, `qlogin` and `qalter` only.
+
+Allows overwriting the allocation rule defined in the parallel environment, see xxqs_name_sxx_pe(1), attribute `allocation_rule`.
+
+The allocation rule can be overwritten globally, as well as for master and/or slave scope, when combined with the `-scope` option.
+
+Examples:
+
+`qsub -pe <pe_name> 4 -par $round_robin` will overwrite the allocation rule defined in the PE globally.
+
+`qsub -pe <pe_name> 4 -scope master -par 1` will overwrite the allocation rule defined in the PE for the master scope.
+
+`qsub -pe <pe_name> 5 -scope master -par 1 -q master.q -scope slave -par 4 -q slave.q` will overwrite the allocation rule defined in the PE
+with `1` for the master scope, and `4` for the slave scope. This is a typical use case for jobs which requires
+the master task to run on a separate host / in a different queue instance than the slave tasks.
+
 ## -pe *parallel_environment* n\[-\[m\]\]\|\[-\]m,...  
 
 Available for `qsub`, `qsh`, `qrsh`, `qlogin` and `qalter` only.

source/clients/common/ocs_client_job.cc

@@ -337,6 +337,16 @@ void cull_show_job(const lListElem *job, int flags, bool show_binding) {
             printf("%-33s", str_attrib);
             uni_print_list(stdout, nullptr, 0, lp, fields, delis, FLG_NO_DELIS_STRINGS);
          }
+
+         const char *allocation_rule = lGetString(jrs, JRS_allocation_rule);
+         if (allocation_rule != nullptr) {
+            if (str_scope == nullptr) {
+               str_attrib = sge_dstring_sprintf(&dstr_attrib, "allocation_rule:");
+            } else {
+               str_attrib = sge_dstring_sprintf(&dstr_attrib, "%s_allocation_rule:", str_scope);
+            }
+            printf("%-33s%s\n", str_attrib, allocation_rule);
+         }
       }
    }
 

source/clients/qalter/ocs_qalter.cc

@@ -27,7 +27,7 @@
  * 
  *   All Rights Reserved.
  * 
- *  Portions of this software are Copyright (c) 2023-2025 HPC-Gridware GmbH
+ *  Portions of this software are Copyright (c) 2023-2026 HPC-Gridware GmbH
  *
  ************************************************************************/
 /*___INFO__MARK_END__*/
@@ -701,6 +701,17 @@ static lList *qalter_parse_job_parameter(u_long32 me_who, lList *cmdline, lList
          nm_set(job_field, JB_project);
       }
 
+      while ((ep = lGetElemStrRW(cmdline, SPA_switch_val, "-par"))) {
+         const char *allocation_rule = lGetString(ep, SPA_switch_arg);
+         u_long32 scope = lGetChar(ep, SPA_argval_lCharT);
+         lListElem *jrs = job_get_or_create_request_setRW(job, scope);
+         if (jrs != nullptr) {
+            lSetString(jrs, JRS_allocation_rule, allocation_rule);
+         }
+         lRemoveElem(cmdline, &ep);
+         nm_set(job_field, JB_request_set_list);
+      }
+
       while ((ep = lGetElemStrRW(cmdline, SPA_switch_val, "-pe"))) {
          lSetString(job, JB_pe, lGetString(ep, SPA_argval_lStringT));
          /* put sublist from parsing into job */
@@ -808,7 +819,6 @@ static lList *qalter_parse_job_parameter(u_long32 me_who, lList *cmdline, lList
             nm_set(job_field, JB_job_args);
          }
          lSetList(job, JB_job_args, lp);
-
       }
 
       /* context switches are sensitive to order */
@@ -829,8 +839,9 @@ static lList *qalter_parse_job_parameter(u_long32 me_who, lList *cmdline, lList
             lRemoveElem(cmdline, &ep);
             ep = temp;
             nm_set(job_field, JB_context);
-         } else
+         } else {
             ep = lNextRW(ep);
+         }
    }
 
    /* complain about unused options */
@@ -862,8 +873,10 @@ static lList *qalter_parse_job_parameter(u_long32 me_who, lList *cmdline, lList
       DRETURN(answer);
    }
 
-/* printf("=============== lWriteElemTo(job, stdout); ==================\n"); */
-/* lWriteElemTo(job, stdout); */
+#if 0
+   printf("=============== lWriteElemTo(job, stdout); ==================\n");
+   lWriteElemTo(job, stdout);
+#endif
 
 
    /* 

source/clients/qsh/ocs_qsh.cc

@@ -29,7 +29,7 @@
  *
  *  Portions of this code are Copyright 2011 Univa Inc.
  *
- *  Portions of this software are Copyright (c) 2024-2025 HPC-Gridware GmbH
+ *  Portions of this software are Copyright (c) 2023-2026 HPC-Gridware GmbH
  *
  ************************************************************************/
 /*___INFO__MARK_END__*/
@@ -2373,8 +2373,8 @@ static void remove_unknown_opts(lList *lp, u_long32 jb_now, int tightly_integrat
             strcmp(cp, "-hold_jid") && strcmp(cp, "-hold_jid_ad") && strcmp(cp, "-h") &&
             strcmp(cp, "-l") && strcmp(cp, "-m") && strcmp(cp, "-masterq") &&
             strcmp(cp, "-N") && strcmp(cp, "-noshell") && strcmp(cp, "-now") &&
-            strcmp(cp, "-notify") && strcmp(cp, "-P") &&
-            strcmp(cp, "-p") && strcmp(cp, "-pe") && strcmp(cp, "-q") && strcmp(cp, "-v") &&
+            strcmp(cp, "-notify") && strcmp(cp, "-P") && strcmp(cp, "-p") && strcmp(cp, "-par") &&
+            strcmp(cp, "-pe") && strcmp(cp, "-q") && strcmp(cp, "-v") &&
             strcmp(cp, "-V") && strcmp(cp, "-display") && strcmp(cp, "-verify") &&
             strcmp(cp, "-soft") && strcmp(cp, "-M") && strcmp(cp, "-verbose") &&
             strcmp(cp, "-ac") && strcmp(cp, "-dc") && strcmp(cp, "-sc") && strcmp(cp, "-scope") &&

source/clients/qsh/ocs_qsh_parse.cc

@@ -27,7 +27,7 @@
  * 
  *   All Rights Reserved.
  * 
- *  Portions of this software are Copyright (c) 2023-2025 HPC-Gridware GmbH
+ *  Portions of this software are Copyright (c) 2023-2026 HPC-Gridware GmbH
  *
  ************************************************************************/
 /*___INFO__MARK_END__*/
@@ -493,6 +493,16 @@ lList *cull_parse_qsh_parameter(u_long32 prog_number, u_long32 uid, const char *
       lRemoveElem(cmdline, &ep);
    }
 
+   while ((ep = lGetElemStrRW(cmdline, SPA_switch_val, "-par"))) {
+      const char *allocation_rule = lGetString(ep, SPA_switch_arg);
+      u_long32 scope = lGetChar(ep, SPA_argval_lCharT);
+      lListElem *jrs = job_get_or_create_request_setRW(*pjob, scope);
+      if (jrs != nullptr) {
+         lSetString(jrs, JRS_allocation_rule, allocation_rule);
+      }
+      lRemoveElem(cmdline, &ep);
+   }
+
    while ((ep = lGetElemStrRW(cmdline, SPA_switch_val, "-js"))) {
       lSetUlong(*pjob, JB_jobshare, lGetUlong(ep, SPA_argval_lUlongT));
       lRemoveElem(cmdline, &ep);

source/common/msg_common.h

@@ -30,7 +30,7 @@
  *
  *  Portions of this code are Copyright 2011 Univa Inc.
  *
- *  Portions of this software are Copyright (c) 2023-2025 HPC-Gridware GmbH
+ *  Portions of this software are Copyright (c) 2023-2026 HPC-Gridware GmbH
  *
  ************************************************************************/
 /*___INFO__MARK_END__*/
@@ -50,9 +50,10 @@
 #define MSG_TABLE_EV_POOL                          "POOL"
 #define MSG_TABLE_SIZE                             "SIZE"
 
-#define MSG_GDI_ARGUMENTSYNTAX_QA_CATEGORY_ID          "cat_id                  category ID"
+#define MSG_GDI_ARGUMENTSYNTAX_OA_ALLOCATION_RULE      "allocation_rule         <int> | $pe_slots | $fill_up | $round_robin"
 #define MSG_GDI_ARGUMENTSYNTAX_OA_ACCOUNT_STRING       "account_string          account_name"
-#define MSG_GDI_ARGUMENTSYNTAX_QA_BINDING_FILTER_STR   "topology_string         topology string where lower case letters show masked units"
+#define MSG_GDI_ARGUMENTSYNTAX_OA_BINDING_FILTER_STR   "topology_string         topology string where lower case letters show masked units"
+#define MSG_GDI_ARGUMENTSYNTAX_OA_CATEGORY_ID          "cat_id                  category ID"
 #define MSG_GDI_ARGUMENTSYNTAX_OA_COMPLEX_LIST         "complex_list            complex[,complex,...]"
 #define MSG_GDI_ARGUMENTSYNTAX_OA_CONTEXT_LIST         "context_list            variable[=value][,variable[=value],...]"
 #define MSG_GDI_ARGUMENTSYNTAX_OA_CKPT_SEL             "ckpt_selector           `n' `s' `m' `x' <interval> "
@@ -965,7 +966,7 @@
 #define MSG_SEC_DELCREDNOBIN_US           _MESSAGE(60413, _("could not delete credentials for job " sge_u32 " - " SFN " binary does not exist"))
 #define MSG_SEC_PUTCREDSTDERR_S           _MESSAGE(60414, _("put_cred stderr: " SFN))
 #define MSG_SEC_NOSTARTCMD4GETCRED_SU     _MESSAGE(60415, _("can't start command " SFQ " for job " sge_u32 " to get credentials"))
-#define MSG_PE_ALLOCRULE_S                _MESSAGE(60416, _("incorrect allocation_rule " SFQ ))
+#define MSG_PE_ALLOCRULE_S                _MESSAGE(60416, _("invalid allocation_rule " SFQ ))
 #define MSG_GDI_OUTOFMEMORY               _MESSAGE(60418, _("out of memory"))
 #define MSG_COM_UNPACKINT_I               _MESSAGE(60419, _("unpacking integer %d failed"))
 #define MSG_COM_UNPACKOBJ_S               _MESSAGE(60420, _("unpacking a " SFN " object failed"))
@@ -1110,7 +1111,10 @@
 
 #define MSG_PARSE_BSORT_CONTRA_S          _MESSAGE(60744, _("contradicting ascending and descending sort order in binding sort " SFQ))
 
-#define MSG_GDI_USAGE_when_OPT              "[-when now|on_reschedule]"
-#define MSG_GDI_UTEXT_when_OPT              _MESSAGE(60745, _("apply modification immediately or on job rescheduling"))
+#define MSG_GDI_USAGE_when_OPT            "[-when now|on_reschedule]"
+#define MSG_GDI_UTEXT_when_OPT            _MESSAGE(60745, _("apply modification immediately or on job rescheduling"))
+
+#define MSG_GDI_USAGE_PAR_OPT             "[-par allocation_rule]"
+#define MSG_GDI_UTEXT_PAR_OPT             _MESSAGE(60746, _("set the parallel job allocation rule"))
 
 // clang-format on

source/common/parse_job_cull.cc

@@ -27,7 +27,7 @@
  * 
  *   All Rights Reserved.
  * 
- *  Portions of this software are Copyright (c) 2023-2025 HPC-Gridware GmbH
+ *  Portions of this software are Copyright (c) 2023-2026 HPC-Gridware GmbH
  *
  ************************************************************************/
 /*___INFO__MARK_END__*/
@@ -621,6 +621,16 @@ lList *cull_parse_job_parameter(u_long32 uid, const char *username, const char *
       lRemoveElem(cmdline, &ep);
    }
 
+   while ((ep = lGetElemStrRW(cmdline, SPA_switch_val, "-par"))) {
+      const char *allocation_rule = lGetString(ep, SPA_switch_arg);
+      u_long32 scope = lGetChar(ep, SPA_argval_lCharT);
+      lListElem *jrs = job_get_or_create_request_setRW(*pjob, scope);
+      if (jrs != nullptr) {
+         lSetString(jrs, JRS_allocation_rule, allocation_rule);
+      }
+      lRemoveElem(cmdline, &ep);
+   }
+
    while ((ep = lGetElemStrRW(cmdline, SPA_switch_val, "-pe"))) {
       lSetString(*pjob, JB_pe, lGetString(ep, SPA_argval_lStringT));
       lSwapList(*pjob, JB_pe_range, ep, SPA_argval_lListT);

source/common/parse_qsub.cc

@@ -37,6 +37,7 @@
 #include <cstdlib>
 #include <unistd.h>
 
+#include "uti/config_file.h"
 #include "uti/sge_dstring.h"
 #include "uti/sge_log.h"
 #include "uti/sge_parse_num_par.h"
@@ -1540,7 +1541,35 @@ lList *cull_parse_cmdline(
          continue;
       }
 
-/*----------------------------------------------------------------------------*/
+      /*-----------------------------------------------------------------------------*/
+      /* "-par" */
+
+      if (strcmp("-par", *sp) == 0) {
+
+         DPRINTF("\"%s\"\n", *sp);
+
+         sp++;
+         if (*sp == nullptr) {
+            answer_list_add_sprintf(&answer, STATUS_ESEMANTIC, ANSWER_QUALITY_ERROR,
+                                    MSG_PARSE_XOPTIONMUSTHAVEARGUMENT_S, "-par");
+            DRETURN(answer);
+         }
+
+         // check if the given allocation rule (e.g., $pe_slots) is correct
+         if (replace_params(*sp, nullptr, 0, pe_alloc_rule_variables) != 0) {
+            answer_list_add_sprintf(&answer, STATUS_ESEMANTIC, ANSWER_QUALITY_ERROR,
+                                    MSG_PARSE_INVALIDOPTIONARGUMENT_SS, "-par", *sp);
+            DRETURN(answer);
+         }
+
+         ep_opt = sge_add_arg(pcmdline, par_OPT, lStringT, *(sp - 1), *sp);
+         lSetChar(ep_opt, SPA_argval_lCharT, scope_flag);
+
+         sp++;
+         continue;
+      }
+
+      /*----------------------------------------------------------------------------*/
       if (!strcmp("-pe", *sp)) {
          lList *pe_range = nullptr;
          dstring d_arg = DSTRING_INIT;

source/common/sge_options.cc

@@ -29,7 +29,7 @@
  *
  *  Portions of this code are Copyright 2011 Univa Inc.
  *
- *  Portions of this software are Copyright (c) 2023-2025 HPC-Gridware GmbH
+ *  Portions of this software are Copyright (c) 2023-2026 HPC-Gridware GmbH
  *
  ************************************************************************/
 /*___INFO__MARK_END__*/
@@ -533,6 +533,8 @@ unsigned short sge_options[][ALL_OPT + 1] =
  {0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1},
 /* when_OPT */
  {0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1},
+/* par_OPT */
+ {0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1},
 /*
   n  q  q  q  q  q  q  q  q  q  q  q  q  q  q  e  q  q  q  q  n  A
   o  a  c  d  h  m  m  r  r  s  s  r  l  s  s  x  e  r  r  r  o  L

source/common/sge_options.h

@@ -30,7 +30,7 @@
  *
  *  Portions of this code are Copyright 2011 Univa Inc.
  *
- *  Portions of this software are Copyright (c) 2023-2025 HPC-Gridware GmbH
+ *  Portions of this software are Copyright (c) 2023-2026 HPC-Gridware GmbH
  *
  ************************************************************************/
 /*___INFO__MARK_END__*/
@@ -309,6 +309,7 @@ enum {
    binstance_OPT, //< binding instance
 
    when_OPT, //< for qalter set if changes shall be applied on a running job or only after rescheduling
+   par_OPT,  //< submit / qalter parameter to set the PE allocation rule per scope
 };
 
 /* macros used in parsing */