backport supervisor/worker reaping (#452)

* backport supervisor/worker reaping
* add more context to test_coprocess failures
* make sure reaper can't block SIGHUP/SIGTERM handler

Author: tgross

integration_tests/tests/test_coprocess/run.sh

@@ -21,29 +21,33 @@ if [ ! $? -eq 0 ] ; then exit 1 ; fi
 ID=$(docker ps -l -f "ancestor=cpfix2_app" --format="{{.ID}}")
 
 # verify the coprocess is running
-docker exec -it "$ID" ps -ef | grep coprocess
+docker exec -it "$ID" ps -ef | grep coprocess \
+    || ( echo 'coprocess not running' && exit 1 )
 
 # kill the coprocess and verify it restarts
 docker exec -it "$ID" pkill coprocess
 sleep 1
-docker exec -it "$ID" ps -ef | grep coprocess
+docker exec -it "$ID" ps -ef | grep coprocess \
+    || ( echo 'coprocess not restarted' && exit 1 )
 
 # kill the coprocess and verify it doesn't restart again
 docker exec -it "$ID" pkill coprocess
 sleep 1
-
-set +e
-docker exec -it "$ID" ps -ef | grep coprocess && exit 1
-set +e
+docker exec -it "$ID" ps -ef | grep coprocess \
+    && ( echo 'coprocess should be stopped' && exit 1 )
 
 # update the ContainerPilot config and verify the coprocess is running
 # with the new flags (this resets the restart limit)
 docker exec -it "$ID" sed -i 's/arg1/arg2/' /app-with-coprocess.json
 docker exec -it "$ID" kill -SIGHUP 1
 sleep 1
-docker exec -it "$ID" ps -ef | grep coprocess | grep arg2
+docker exec -it "$ID" ps -ef | grep coprocess | grep arg2 \
+    || ( echo 'coprocess should use new args, got:' \
+              && docker exec -it "$ID" ps -ef \
+              && exit 1 )
 
 # kill the coprocess and verify it restarts
 docker exec -it "$ID" pkill coprocess
 sleep 1
-docker exec -it "$ID" ps -ef | grep coprocess
+docker exec -it "$ID" ps -ef | grep coprocess \
+    || ( echo 'coprocess not restarted w/ new args' && exit 1 )

main.go

@@ -1,9 +1,11 @@
 package main // import "github.com/joyent/containerpilot"
 
 import (
+	"os"
 	"runtime"
 
 	"github.com/joyent/containerpilot/core"
+	"github.com/joyent/containerpilot/sup"
 
 	log "github.com/sirupsen/logrus"
 
@@ -18,6 +20,15 @@ func main() {
 	// contention on the main application
 	runtime.GOMAXPROCS(1)
 
+	// If we're running as PID1, we fork and run as a supervisor
+	// so that we can cleanly handle reaping of child processes.
+	// We fork before doing *anything* else so we don't have to
+	// worry about where any new threads spawned by the runtime.
+	if os.Getpid() == 1 {
+		sup.Run() // blocks forever
+		return
+	}
+
 	app, configErr := core.LoadApp()
 	if configErr != nil {
 		log.Fatal(configErr)

sup/sup.go

@@ -0,0 +1,71 @@
+package sup
+
+import (
+	"log"
+	"os"
+	"os/exec"
+	"os/signal"
+	"syscall"
+)
+
+// Run forks the ContainerPilot process and then starts signal handlers
+// that will reap child processes and pass-thru SIGINT and SIGKILL to
+// the ContainerPilot worker process.
+func Run() {
+	self, err := exec.LookPath(os.Args[0])
+	if err != nil {
+		log.Fatal("failed to find ContainerPilot binary: ", err)
+	}
+	proc, err := os.StartProcess(self, os.Args, &os.ProcAttr{Dir: "", Env: nil,
+		Files: []*os.File{os.Stdin, os.Stdout, os.Stderr}, Sys: nil})
+	if err != nil {
+		log.Fatal("failed to start ContainerPilot worker process:", err)
+	}
+	handleSignals(proc.Pid)
+	proc.Wait()
+}
+
+// handleSignals listens for signals used to gracefully shutdown and
+// passes them thru to the ContainerPilot worker process.
+func handleSignals(pid int) {
+	sig := make(chan os.Signal, 1)
+	signal.Notify(sig, syscall.SIGTERM, syscall.SIGHUP, syscall.SIGINT, syscall.SIGCHLD, syscall.SIGUSR1)
+	go func() {
+		for signal := range sig {
+			switch signal {
+			case syscall.SIGINT:
+				syscall.Kill(pid, syscall.SIGINT)
+			case syscall.SIGTERM:
+				syscall.Kill(pid, syscall.SIGTERM)
+			case syscall.SIGHUP:
+				syscall.Kill(pid, syscall.SIGHUP)
+			case syscall.SIGUSR1:
+				syscall.Kill(pid, syscall.SIGUSR1)
+			case syscall.SIGCHLD:
+				go reap()
+			}
+		}
+	}()
+}
+
+// reaps child processes that have been reparented to PID1
+func reap() {
+	for {
+	POLL:
+		var wstatus syscall.WaitStatus
+		pid, err := syscall.Wait4(-1, &wstatus, 0, nil)
+		switch err {
+		case nil:
+			if pid > 0 {
+				goto POLL
+			}
+			return
+		case syscall.ECHILD:
+			return // no more children, we're done till next signal
+		case syscall.EINTR:
+			goto POLL
+		default:
+			return
+		}
+	}
+}