BF: CS-1922 shepherd_cmd does not work with qrsh jobs

Author: jgabler-hpc

doc/markdown/man/man5/sge_conf.md

@@ -613,6 +613,19 @@ Changing *shepherd_cmd* will take immediate effect. The default for *shepherd_cm
 
 The global configuration entry for this value may be overwritten by the execution host local configuration.
 
+In a shepherd wrapper make sure to call the actual shepherd binary with the same arguments as the wrapper, e.g.,
+
+```bash
+#!/bin/sh
+
+ARCH=`$SGE_ROOT/util/arch`
+SHEPHERD="$SGE_ROOT/bin/$ARCH/sge_shepherd"
+
+# Do the shepherd wrapper specific actions here.
+
+exec "$SHEPHERD" "$@"
+```
+
 ## gid_range
 
 The *gid_range* is a comma separated list of range expressions of the form n-m (n as well as m are integer numbers 

source/daemons/common/err_trace.cc

@@ -27,7 +27,7 @@
  * 
  *   All Rights Reserved.
  * 
- *  Portions of this software are Copyright (c) 2023-2025 HPC-Gridware GmbH
+ *  Portions of this software are Copyright (c) 2026 HPC-Gridware GmbH
  *
  ************************************************************************/
 /*___INFO__MARK_END__*/
@@ -92,7 +92,7 @@ static pthread_mutex_t g_trace_mutex;
 extern pid_t coshepherd_pid;
 extern int   shepherd_state;  /* holds exit status for shepherd_error() */
 extern bool  g_new_interactive_job_support;
-int foreground = 1;           /* usability of stderr/out */
+bool foreground = true; // make shepherd_trace() write to both trace file and stdout for debugging
 
 /* Forward declaration of static functions */
 
@@ -329,8 +329,7 @@ int shepherd_trace(const char *format, ...)
          sge_dstring_vsprintf(&message, format, ap);
          va_end(ap);
 
-   	 ret = sh_str2file(header_str, sge_dstring_get_string(&message), 
-                           shepherd_trace_fp);
+      	ret = sh_str2file(header_str, sge_dstring_get_string(&message), shepherd_trace_fp);
 
          if (foreground) {
             printf("%s%s\n", header_str, sge_dstring_get_string(&message));
@@ -549,6 +548,10 @@ int is_shepherd_trace_fd(int fd)
    }
 }
 
+int get_shepherd_trace_fp() {
+	return fileno(shepherd_trace_fp);
+}
+
 /****** count_exit_status *****************************************************
 *  NAME
 *     count_exit_status() -- Return the number of lines in the exit status file
@@ -816,7 +819,7 @@ static FILE* shepherd_trace_init_intern(st_shepherd_file_t shepherd_file)
 		return nullptr;
 	}
 
-	/* To avoid to block stdin, stdout or stderr, dup the fd until it is >= 3 */
+	/* To avoid blocking stdin, stdout or stderr, dup the fd until it is >= 3 */
 	if (fd<3) {
 		dup_fd(&fd);
 	}

source/daemons/common/err_trace.h

@@ -28,14 +28,14 @@
  * 
  *   All Rights Reserved.
  * 
- *  Portions of this software are Copyright (c) 2023-2024 HPC-Gridware GmbH
+ *  Portions of this software are Copyright (c) 2026 HPC-Gridware GmbH
  *
  ************************************************************************/
 /*___INFO__MARK_END__*/
 
 typedef int (*tShepherd_trace)(const char *format, ...); 
 
-extern int foreground;      /* != 0 if we can write to stderr/out     */
+extern bool foreground; // make shepherd_trace() write to both trace file and stdout for debugging
 
 void shepherd_trace_init();
 void shepherd_trace_exit();
@@ -50,5 +50,6 @@ void shepherd_error(int do_exit, const char *format, ...);
 void shepherd_error_ptr(const char *text);
 void shepherd_write_exit_status( const char *exit_status );
 
-int  is_shepherd_trace_fd( int fd );
-int  count_exit_status();
+int get_shepherd_trace_fp();
+int is_shepherd_trace_fd(int fd);
+int count_exit_status();

source/daemons/execd/exec_job.cc

@@ -27,7 +27,7 @@
  *
  *   All Rights Reserved.
  *
- *  Portions of this software are Copyright (c) 2023-2026 HPC-Gridware GmbH
+ *  Portions of this software are Copyright (c) 2026 HPC-Gridware GmbH
  *
  ************************************************************************/
 /*___INFO__MARK_END__*/
@@ -1996,7 +1996,11 @@ int sge_exec_job(lListElem *jep, lListElem *jatep, lListElem *petep, char *err_s
          DPRINTF("CHILD - About to exec shepherd wrapper job ->%s< under queue -<%s<\n",
                  lGetString(jep, JB_job_name),
                  lGetString(master_q, QU_full_name));
-         execlp(shepherd_cmd, ps_name, nullptr);
+         if (ISTRACE) {
+            execlp(shepherd_cmd, ps_name, nullptr);
+         } else {
+            execlp(shepherd_cmd, ps_name, "-bg", nullptr);
+         }
       } else if (mconf_get_do_credentials() && ocs::Bootstrap::has_security_mode(ocs::Bootstrap::BS_SEC_MODE_DCE)) {
          DPRINTF("CHILD - About to exec DCE shepherd wrapper job ->%s< under queue -<%s<\n",
                  lGetString(jep, JB_job_name),
@@ -2008,10 +2012,11 @@ int sge_exec_job(lListElem *jep, lListElem *jatep, lListElem *petep, char *err_s
                  lGetString(jep, JB_job_name),
                  lGetString(master_q, QU_full_name));
 
-         if (ISTRACE)
+         if (ISTRACE) {
             execlp(shepherd_path, ps_name, nullptr);
-         else
+         } else {
             execlp(shepherd_path, ps_name, "-bg", nullptr);
+         }
       } else {
          char commandline[2048];
 

source/daemons/shepherd/builtin_starter.cc

@@ -29,7 +29,7 @@
  *
  *  Portions of this software are Copyright (c) 2011-2012 Univa Corporation
  *
- *  Portions of this software are Copyright (c) 2023-2025 HPC-Gridware GmbH
+ *  Portions of this software are Copyright (c) 2026 HPC-Gridware GmbH
  *
  ************************************************************************/
 /*___INFO__MARK_END__*/
@@ -175,7 +175,7 @@ void son(const char *childname, char *script_file, int truncate_stderr_out, bool
    int pty;
    bool is_the_job = strcmp(childname, "job") == 0;
 
-   foreground = 0; /* VX sends SIGTTOU if trace messages go to foreground */
+   foreground = false; // VX sends SIGTTOU if trace messages go to foreground
 
    /* From here only the son --------------------------------------*/
    if (script_file == nullptr) {
@@ -427,7 +427,7 @@ void son(const char *childname, char *script_file, int truncate_stderr_out, bool
          }
       }
    }
-   foreground = 0;
+   foreground = false;
 
    /* We have different possibilities to start the job script:
     * - We can start it as login shell or not

source/daemons/shepherd/ocs_shepherd_pty.cc

@@ -458,40 +458,35 @@ pid_t fork_no_pty(int *fd_pipe_in, int *fd_pipe_out,
 
    ret = pipe(fd_pipe_in);
    if (ret == -1) {
-      sge_dstring_sprintf(err_msg, "can't create pipe for stdin: %d: %s",
-                          errno, strerror(errno));
+      sge_dstring_sprintf(err_msg, "can't create pipe for stdin: %d: %s", errno, strerror(errno));
       return -1;
    }
 
    ret = pipe(fd_pipe_out);
    if (ret == -1) {
-      sge_dstring_sprintf(err_msg, "can't create pipe for stdout: %d: %s",
-                          errno, strerror(errno));
+      sge_dstring_sprintf(err_msg, "can't create pipe for stdout: %d: %s", errno, strerror(errno));
       return -1;
    }
 
    ret = pipe(fd_pipe_err);
    if (ret == -1) {
-      sge_dstring_sprintf(err_msg, "can't create pipe for stderr: %d: %s",
-                          errno, strerror(errno));
+      sge_dstring_sprintf(err_msg, "can't create pipe for stderr: %d: %s", errno, strerror(errno));
       return -1;
    }
 
    if ((pid = fork()) < 0) {
       return -1;
    } else if (pid == 0) {     /* child */
       if (setsid() < 0) {
-         sge_dstring_sprintf(err_msg, "setsid() error: %d, %s",
-                             errno, strerror(errno));
+         sge_dstring_sprintf(err_msg, "setsid() error: %d, %s", errno, strerror(errno));
          return -1;
       }
 
       /* attach pipes to stdin/stdout/stderr of child */
       close(fd_pipe_in[1]);
       fd_pipe_in[1] = -1;
       if ((dup2(fd_pipe_in[0], STDIN_FILENO)) != STDIN_FILENO) {
-         sge_dstring_sprintf(err_msg, "dup2 to stdin error: %d, %s",
-                             errno, strerror(errno));
+         sge_dstring_sprintf(err_msg, "dup2 to stdin error: %d, %s", errno, strerror(errno));
          return -1;
       }
       close(fd_pipe_in[0]);
@@ -500,8 +495,7 @@ pid_t fork_no_pty(int *fd_pipe_in, int *fd_pipe_out,
       close(fd_pipe_out[0]);
       fd_pipe_out[0] = -1;
       if ((dup2(fd_pipe_out[1], STDOUT_FILENO)) != STDOUT_FILENO) {
-         sge_dstring_sprintf(err_msg, "dup2 to stdout error: %d, %s",
-                             errno, strerror(errno));
+         sge_dstring_sprintf(err_msg, "dup2 to stdout error: %d, %s", errno, strerror(errno));
          return -1;
       }
       close(fd_pipe_out[1]);
@@ -510,8 +504,7 @@ pid_t fork_no_pty(int *fd_pipe_in, int *fd_pipe_out,
       close(fd_pipe_err[0]);
       fd_pipe_out[0] = -1;
       if ((dup2(fd_pipe_err[1], STDERR_FILENO)) != STDERR_FILENO) {
-         sge_dstring_sprintf(err_msg, "dup2 to stderr error: %d, %s",
-                             errno, strerror(errno));
+         sge_dstring_sprintf(err_msg, "dup2 to stderr error: %d, %s", errno, strerror(errno));
          return -1;
       }
       close(fd_pipe_err[1]);
@@ -526,5 +519,3 @@ pid_t fork_no_pty(int *fd_pipe_in, int *fd_pipe_out,
    }
    DRETURN(pid);
 }
-
-

source/daemons/shepherd/shepherd.cc

@@ -29,7 +29,7 @@
  *
  *  Portions of this software are Copyright (c) 2011 Univa Corporation
  *
- *  Portions of this software are Copyright (c) 2023-2026 HPC-Gridware GmbH
+ *  Portions of this software are Copyright (c) 2026 HPC-Gridware GmbH
  *
  ************************************************************************/
 /*___INFO__MARK_END__*/
@@ -301,19 +301,19 @@ static int wait_until_parent_has_registered_to_server(int fd_pipe_to_child[])
    SIGIGNORE(SIGWINCH);
 
    /* close parents end of our copy of the pipe */
-   shepherd_trace("child: closing parents end of the pipe_to_child");
+   shepherd_trace("child: closing parent side ends of pipes: %d", fd_pipe_to_child[1]);
    close(fd_pipe_to_child[1]);
    fd_pipe_to_child[1] = -1;
 
    /* wait until parent has registered at the server */
-   shepherd_trace("child: trying to read from parent through the pipe_to_child");
+   shepherd_trace("child: trying to read from parent through the pipe_to_child[0] = %d", fd_pipe_to_child[0]);
    ret = read(fd_pipe_to_child[0], tmpbuf, 11);
    if (ret <= 0) {
-      shepherd_trace("child: error communicating with parent: %d, %s",
-                     errno, strerror(errno));
+      shepherd_trace("child: error communicating with parent: %d, %s", errno, strerror(errno));
       ret = -1;
    } else {
       /* close other side of our copy of the pipe */
+      shepherd_trace("child: closing child side ends of pipes: %d", fd_pipe_to_child[0]);
       close(fd_pipe_to_child[0]);
       fd_pipe_to_child[0] = -1;
       shepherd_trace("child: parent sent us '%s'", tmpbuf);
@@ -699,8 +699,8 @@ int main(int argc, char **argv)
       shepherd_error(1, "can't read cwd - getcwd failed: %s", strerror(errno));
    }
 
-   if (argc >= 2 && !strcmp("-bg", argv[1])) {
-      foreground = 0;   /* no output to stderr */
+   if (argc >= 2 && strcmp("-bg", argv[1]) == 0) {
+      foreground = false;   // no shepherd_trace() output to stdout
    }
 
    set_shepherd_signal_mask();
@@ -1128,8 +1128,14 @@ static int start_child(
          }
       }
 
-      if (pid==0) { /* child */
+      if (pid == 0) { // child
          if (g_new_interactive_job_support && is_interactive) {
+            // When we are running a builtin interactive job, in foreground mode shepherd_trace() would
+            // print the messages to stdout which is redirected to a pipe to the shepherd parent.
+            // The output would then be forwarded to the qrsh client.
+            // Disable foreground mode.
+            foreground = false;
+
             // Why do we wait until the connection to qrsh is up?
             // To avoid starting the job when the qrsh client has terminated in the meantime?
             ret = wait_until_parent_has_registered_to_server(fd_pipe_to_child);
@@ -1383,9 +1389,7 @@ static int start_child(
       }
 
       /******* write usage to file "usage" ************/
-      shepherd_write_usage_file(wait_status, exit_status,
-                                child_signal, start_time,
-                                end_time, &rusage);
+      shepherd_write_usage_file(wait_status, exit_status, child_signal, start_time, end_time, &rusage);
 
       /* this is SEMPA stuff */
       notify_tasker(exit_status);
@@ -1528,9 +1532,6 @@ dstring       *dstr_error       /* OUT: error message - if any */
    int     remote_port  = 0;
    int     exit_status  = -1;
 
-   /* close child's end of the pipe */
-   shepherd_trace("parent: closing child's end of the pipe");
-
    /* read destination host and port from config */
    ret = get_remote_host_and_port_from_config(&remote_host, &remote_port, dstr_error);
    if (ret != 0 || remote_host == nullptr || remote_port == 0) {
@@ -2764,6 +2765,8 @@ static int start_async_command(const char *descr, char *cmd)
    if ((pid = fork()) == -1) {
       shepherd_trace("can't fork for starting %s command", descr);
    } else if (pid == 0) {
+      foreground = false;
+
       int use_qsub_gid;
       gid_t gid;
       char *tmp_str;
@@ -2808,8 +2811,6 @@ static int start_async_command(const char *descr, char *cmd)
          exit(1);
       }
 
-      foreground = 0;
-
       cwd = get_conf_val("cwd");
       if (sge_chdir(cwd)) {
          shepherd_trace("%s: can't chdir to %s", descr, cwd);