Move waiting logic for host orchestrator settlement to backend

While interacting with host orchestrator right after creating host API is returned, there is a possibility of getting 503 service temporarily unavailable error because host is created and booted but host orchestrator is not executed / ready yet.

Handling logic for waiting host orchestrator readiness is currently in the client logic. Moving this logic into cloud orchestrator API endpoint (specifically inside WaitOperation) makes the client simpler and reduces redundant API endpoints.

Context: b/501288123

TAG=agy
CONV=f5f788b6-7aa2-4614-b441-87f3ad9fed4e

Author: k311093

pkg/app/instances/docker.go

@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"net/http"
 	"net/url"
 	"strings"
 	"sync"
@@ -188,6 +189,13 @@ func (m *DockerInstanceManager) waitCreateHostOperation(host string) (*apiv1.Hos
 				return nil, fmt.Errorf("failed to inspect docker container: %w", err)
 			}
 			if res.State.Running {
+				client, err := m.GetHostClient("local", host)
+				if err != nil {
+					return nil, err
+				}
+				if err := waitForHostReady(client); err != nil {
+					return nil, err
+				}
 				return &apiv1.HostInstance{
 					Name: host,
 				}, nil
@@ -500,3 +508,18 @@ func (m *DockerInstanceManager) getRWMutex(user accounts.User) *sync.RWMutex {
 	mu, _ := m.mutexes.LoadOrStore(user.Username(), &sync.RWMutex{})
 	return mu.(*sync.RWMutex)
 }
+
+func waitForHostReady(client HostClient) error {
+	maxWait := 2 * time.Minute
+	retryDelay := 5 * time.Second
+	deadline := time.Now().Add(maxWait)
+
+	for time.Now().Before(deadline) {
+		status, err := client.Get("/", "", nil)
+		if err == nil && status != http.StatusBadGateway {
+			return nil
+		}
+		time.Sleep(retryDelay)
+	}
+	return errors.NewServiceUnavailableError("wait for host orchestrator timed out", nil)
+}

pkg/app/instances/docker_test.go

@@ -15,6 +15,8 @@
 package instances
 
 import (
+	"net/http"
+	"net/http/httputil"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
@@ -52,3 +54,53 @@ func TestDecodeOperationFailsMissingUnderscore(t *testing.T) {
 		t.Errorf("expected error")
 	}
 }
+
+type mockHostClient struct {
+	getFunc func(string, string, *HostResponse) (int, error)
+}
+
+func (c *mockHostClient) Get(path, query string, res *HostResponse) (int, error) {
+	return c.getFunc(path, query, res)
+}
+
+func (c *mockHostClient) Post(path, query string, body any, res *HostResponse) (int, error) {
+	return 0, nil
+}
+
+func (c *mockHostClient) GetReverseProxy() *httputil.ReverseProxy {
+	return nil
+}
+
+func TestWaitForHostReadySucceeds(t *testing.T) {
+	client := &mockHostClient{
+		getFunc: func(path, query string, res *HostResponse) (int, error) {
+			return http.StatusOK, nil
+		},
+	}
+
+	err := waitForHostReady(client)
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+}
+
+func TestWaitForHostReadyRetriesOnBadGateway(t *testing.T) {
+	calls := 0
+	client := &mockHostClient{
+		getFunc: func(path, query string, res *HostResponse) (int, error) {
+			calls++
+			if calls == 1 {
+				return http.StatusBadGateway, nil
+			}
+			return http.StatusOK, nil
+		},
+	}
+
+	err := waitForHostReady(client)
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	if calls != 2 {
+		t.Errorf("expected 2 calls, got %d", calls)
+	}
+}

pkg/app/instances/gce.go

@@ -18,9 +18,11 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"net/http"
 	"net/url"
 	"path"
 	"regexp"
+	"time"
 
 	apiv1 "github.com/google/cloud-android-orchestration/api/v1"
 	"github.com/google/cloud-android-orchestration/pkg/app/accounts"
@@ -41,6 +43,7 @@ type GCPIMConfig struct {
 	UseExternalIP        bool
 	// If true, instances created should be compatible with `acloud CLI`.
 	AcloudCompatible bool
+	HostReadyTimeout time.Duration
 }
 
 const (
@@ -78,28 +81,40 @@ func (m *GCEInstanceManager) ListZones() (*apiv1.ListZonesResponse, error) {
 	}, nil
 }
 
-func (m *GCEInstanceManager) GetHostAddr(zone string, host string) (string, error) {
-	instance, err := m.getHostInstance(zone, host)
-	if err != nil {
-		return "", err
-	}
-	ilen := len(instance.NetworkInterfaces)
+func getHostAddrWithIns(ins *compute.Instance) (string, error) {
+	ilen := len(ins.NetworkInterfaces)
 	if ilen == 0 {
-		log.Printf("host instance %s in zone %s is missing a network interface", host, zone)
+		log.Printf("host instance %s in zone %s is missing a network interface", ins.Name, ins.Zone)
 		return "", errors.NewInternalError("host instance missing a network interface", nil)
 	}
 	if ilen > 1 {
-		log.Printf("host instance %s in zone %s has %d network interfaces", host, zone, ilen)
+		log.Printf("host instance %s in zone %s has %d network interfaces", ins.Name, ins.Zone, ilen)
+	}
+	return ins.NetworkInterfaces[0].NetworkIP, nil
+}
+
+func (m *GCEInstanceManager) GetHostAddr(zone string, host string) (string, error) {
+	ins, err := m.getHostInstance(zone, host)
+	if err != nil {
+		return "", err
+	}
+	return getHostAddrWithIns(ins)
+}
+
+func getHostURLWithIns(ins *compute.Instance, config *Config) (*url.URL, error) {
+	addr, err := getHostAddrWithIns(ins)
+	if err != nil {
+		return nil, err
 	}
-	return instance.NetworkInterfaces[0].NetworkIP, nil
+	return url.Parse(fmt.Sprintf("%s://%s:%d", config.HostOrchestratorProtocol, addr, config.GCP.HostOrchestratorPort))
 }
 
 func (m *GCEInstanceManager) GetHostURL(zone string, host string) (*url.URL, error) {
-	addr, err := m.GetHostAddr(zone, host)
+	ins, err := m.getHostInstance(zone, host)
 	if err != nil {
 		return nil, err
 	}
-	return url.Parse(fmt.Sprintf("%s://%s:%d", m.Config.HostOrchestratorProtocol, addr, m.Config.GCP.HostOrchestratorPort))
+	return getHostURLWithIns(ins, &m.Config)
 }
 
 const operationStatusDone = "DONE"
@@ -253,16 +268,81 @@ func (m *GCEInstanceManager) WaitOperation(zone string, user accounts.User, name
 	if op.Status != operationStatusDone {
 		return nil, errors.NewServiceUnavailableError("Wait for operation timed out", nil)
 	}
-	getter := opResultGetter{Service: m.Service, Op: op}
-	return getter.Get()
+	getter := opResultGetter{
+		Service: m.Service,
+		Op:      op,
+		Config:  &m.Config,
+	}
+	res, err := getter.Get()
+	if err != nil {
+		return nil, err
+	}
+	if hostInst, ok := res.(*apiv1.HostInstance); ok && op.OperationType == "insert" {
+		return m.WaitHostAvailability(zone, user, hostInst.Name)
+	}
+	return res, nil
+}
+
+func (m *GCEInstanceManager) WaitHostAvailability(zone string, user accounts.User, host string) (*apiv1.HostInstance, error) {
+	ins, err := m.getHostInstance(zone, host)
+	if err != nil {
+		return nil, err
+	}
+	hostInstance, err := BuildHostInstance(ins)
+	if err != nil {
+		return nil, err
+	}
+	client, err := getHostClientWithIns(ins, &m.Config)
+	if err != nil {
+		return nil, err
+	}
+	if err := m.waitForOrchestrator(zone, hostInstance, client); err != nil {
+		return nil, err
+	}
+	return hostInstance, nil
+}
+
+func (m *GCEInstanceManager) waitForOrchestrator(zone string, host *apiv1.HostInstance, client HostClient) error {
+	timeout := m.Config.GCP.HostReadyTimeout
+	if timeout == 0 {
+		timeout = 5 * time.Minute
+	}
+	retryDelay := 5 * time.Second
+	deadline := time.Now().Add(timeout)
+
+	for time.Now().Before(deadline) {
+		_, err := m.Service.Instances.Get(m.Config.GCP.ProjectID, zone, host.Name).Context(context.TODO()).Do()
+		if err != nil {
+			if apiErr, ok := err.(*googleapi.Error); ok && apiErr.Code == http.StatusNotFound {
+				return errors.NewNotFoundError("Host was deleted concurrently", err)
+			}
+			return fmt.Errorf("failed to check host existence: %w", err)
+		}
+
+		status, err := client.Get("/", "", nil)
+		if err == nil && status != http.StatusBadGateway {
+			return nil
+		}
+
+		time.Sleep(retryDelay)
+	}
+	return errors.NewServiceUnavailableError("Wait for host orchestrator timed out", nil)
+}
+
+func getHostClientWithIns(ins *compute.Instance, config *Config) (HostClient, error) {
+	url, err := getHostURLWithIns(ins, config)
+	if err != nil {
+		return nil, err
+	}
+	return NewNetHostClient(url, config.AllowSelfSignedHostSSLCertificate), nil
 }
 
 func (m *GCEInstanceManager) GetHostClient(zone string, host string) (HostClient, error) {
-	url, err := m.GetHostURL(zone, host)
+	ins, err := m.getHostInstance(zone, host)
 	if err != nil {
 		return nil, err
 	}
-	return NewNetHostClient(url, m.Config.AllowSelfSignedHostSSLCertificate), nil
+	return getHostClientWithIns(ins, &m.Config)
 }
 
 func (m *GCEInstanceManager) getHostInstance(zone string, host string) (*compute.Instance, error) {
@@ -326,6 +406,7 @@ var (
 type opResultGetter struct {
 	Service *compute.Service
 	Op      *compute.Operation
+	Config  *Config
 }
 
 func (g *opResultGetter) Get() (any, error) {