fix gateway_wait script for hermes (#1931)

Signed-off-by: Peter Jausovec <peter.jausovec@solo.io>

Author: peterj

go/core/pkg/sandboxbackend/openshell/agentharness_openshell_client.go

@@ -177,11 +177,23 @@ func (c *AgentHarnessOpenShellClient) ExecSandboxID(ctx context.Context, sandbox
 	return name, nil
 }
 
+type ExecSandboxResult struct {
+	ExitCode int32
+	Stdout   string
+	Stderr   string
+}
+
 // ExecSandbox runs a command inside the sandbox via OpenShell ExecSandbox streaming RPC.
 func (c *AgentHarnessOpenShellClient) ExecSandbox(ctx context.Context, sandboxID string, command []string, stdin []byte, env map[string]string, timeoutSec uint32) (int32, string, error) {
+	res, err := c.ExecSandboxOutput(ctx, sandboxID, command, stdin, env, timeoutSec)
+	return res.ExitCode, res.Stderr, err
+}
+
+// ExecSandboxOutput runs a command inside the sandbox and captures stdout, stderr, and the exit code.
+func (c *AgentHarnessOpenShellClient) ExecSandboxOutput(ctx context.Context, sandboxID string, command []string, stdin []byte, env map[string]string, timeoutSec uint32) (ExecSandboxResult, error) {
 	osCli := c.openShell()
 	if osCli == nil {
-		return -1, "", fmt.Errorf("openshell client is nil")
+		return ExecSandboxResult{ExitCode: -1}, fmt.Errorf("openshell client is nil")
 	}
 	req := &openshellv1.ExecSandboxRequest{
 		SandboxId:      sandboxID,
@@ -194,8 +206,9 @@ func (c *AgentHarnessOpenShellClient) ExecSandbox(ctx context.Context, sandboxID
 	}
 	stream, err := osCli.ExecSandbox(ctx, req)
 	if err != nil {
-		return -1, "", err
+		return ExecSandboxResult{ExitCode: -1}, err
 	}
+	var stdout strings.Builder
 	var stderr strings.Builder
 	var exitCode int32 = -1
 	for {
@@ -204,10 +217,13 @@ func (c *AgentHarnessOpenShellClient) ExecSandbox(ctx context.Context, sandboxID
 			if errors.Is(err, io.EOF) {
 				break
 			}
-			return exitCode, stderr.String(), err
+			return ExecSandboxResult{ExitCode: exitCode, Stdout: stdout.String(), Stderr: stderr.String()}, err
 		}
 		switch p := ev.GetPayload().(type) {
 		case *openshellv1.ExecSandboxEvent_Stdout:
+			if p.Stdout != nil {
+				stdout.Write(p.Stdout.GetData())
+			}
 		case *openshellv1.ExecSandboxEvent_Stderr:
 			if p.Stderr != nil {
 				stderr.Write(p.Stderr.GetData())
@@ -219,9 +235,9 @@ func (c *AgentHarnessOpenShellClient) ExecSandbox(ctx context.Context, sandboxID
 		}
 	}
 	if exitCode == -1 {
-		return exitCode, stderr.String(), fmt.Errorf("ExecSandbox finished without exit status")
+		return ExecSandboxResult{ExitCode: exitCode, Stdout: stdout.String(), Stderr: stderr.String()}, fmt.Errorf("ExecSandbox finished without exit status")
 	}
-	return exitCode, stderr.String(), nil
+	return ExecSandboxResult{ExitCode: exitCode, Stdout: stdout.String(), Stderr: stderr.String()}, nil
 }
 
 // ErrEmptyResponse is returned when OpenShell returns success with an empty Sandbox payload.

go/core/pkg/sandboxbackend/openshell/hermes.go

@@ -144,14 +144,9 @@ func (b *HermesBackend) OnAgentHarnessReady(ctx context.Context, ah *v1alpha2.Ag
 		return fmt.Errorf("start hermes gateway: exit %d: %s", code, strings.TrimSpace(stderr))
 	}
 
-	waitGateway := hermes.GatewayListenWaitScript(hermes.HermesInternalGatewayPort)
-	code, stderr, err = b.ExecSandbox(withAuth(gwCtx, token), execID, []string{"sh", "-c", waitGateway}, nil, execEnv, 45)
-	if err != nil {
+	if err := b.waitHermesGatewayListen(withAuth(gwCtx, token), execID, hermes.HermesInternalGatewayPort, execEnv); err != nil {
 		return fmt.Errorf("wait for hermes gateway listen: %w", err)
 	}
-	if code != 0 {
-		return fmt.Errorf("wait for hermes gateway listen: exit %d: %s", code, strings.TrimSpace(stderr))
-	}
 
 	socatStart := fmt.Sprintf(
 		`command -v socat >/dev/null 2>&1 && nohup socat TCP-LISTEN:%d,bind=0.0.0.0,fork,reuseaddr TCP:127.0.0.1:%d >>/tmp/socat.log 2>&1 &`,
@@ -170,6 +165,37 @@ func (b *HermesBackend) OnAgentHarnessReady(ctx context.Context, ah *v1alpha2.Ag
 	return nil
 }
 
+func (b *HermesBackend) waitHermesGatewayListen(ctx context.Context, execID string, port int, execEnv map[string]string) error {
+	listenAddr := fmt.Sprintf("127.0.0.1:%d", port)
+	var lastResult ExecSandboxResult
+	for range 30 {
+		result, err := b.ExecSandboxOutput(ctx, execID, []string{"ss", "-tln"}, nil, execEnv, 5)
+		if err != nil {
+			return err
+		}
+		lastResult = result
+		if result.ExitCode != 0 {
+			return fmt.Errorf("ss -tln exit %d: %s", result.ExitCode, strings.TrimSpace(result.Stderr))
+		}
+		if strings.Contains(result.Stdout, listenAddr) {
+			return nil
+		}
+		timer := time.NewTimer(time.Second)
+		select {
+		case <-ctx.Done():
+			timer.Stop()
+			return ctx.Err()
+		case <-timer.C:
+		}
+	}
+	return fmt.Errorf(
+		"timed out after 30s waiting for %s; last ss output: %s; stderr: %s",
+		listenAddr,
+		strings.TrimSpace(lastResult.Stdout),
+		strings.TrimSpace(lastResult.Stderr),
+	)
+}
+
 func buildHermesCreateRequest(ah *v1alpha2.AgentHarness, messagingProviders []string) (*openshellv1.CreateSandboxRequest, []string) {
 	req, unsupported := buildAgentHarnessOpenshellCreateRequest(ah)
 	if req.GetSpec().GetTemplate() == nil {