Move waiting logic for host orchestrator settlement to backend

While interacting with host orchestrator right after creating host
API is returned, there is a possibility of getting 503 service
temporarily unavailable error because host it created and booted but
host orchestrator is not executed / ready yet.

Handling logic for waiting host orchestrator readiness is currently
in the client logic. Moving this logic into cloud orchestrator API
endpoint would make the client that uses host chestrator API simpler.

Context: b/501288123

Author: k311093

pkg/app/app_test.go

@@ -111,6 +111,10 @@ func (hc *testHostClient) GetReverseProxy() *httputil.ReverseProxy {
 	return httputil.NewSingleHostReverseProxy(hc.url)
 }
 
+func (hc *testHostClient) WaitForHostReady() error {
+	return nil
+}
+
 func TestListZonesSucceeds(t *testing.T) {
 	controller := NewApp(&testInstanceManager{}, &testAccountManager{}, nil, nil, nil, "", nil, apiv1.InfraConfig{}, &config.Config{})
 	ts := httptest.NewServer(controller.Handler())

pkg/app/instances/docker.go

@@ -188,6 +188,13 @@ func (m *DockerInstanceManager) waitCreateHostOperation(host string) (*apiv1.Hos
 				return nil, fmt.Errorf("failed to inspect docker container: %w", err)
 			}
 			if res.State.Running {
+				client, err := m.GetHostClient("local", host)
+				if err != nil {
+					return nil, err
+				}
+				if err := client.WaitForHostReady(); err != nil {
+					return nil, err
+				}
 				return &apiv1.HostInstance{
 					Name: host,
 				}, nil

pkg/app/instances/gce.go

@@ -78,8 +78,16 @@ func (m *GCEInstanceManager) ListZones() (*apiv1.ListZonesResponse, error) {
 	}, nil
 }
 
-func (m *GCEInstanceManager) GetHostAddr(zone string, host string) (string, error) {
-	instance, err := m.getHostInstance(zone, host)
+func (m *GCEInstanceManager) getHostAddrInner(zone string, host string, ins *compute.Instance) (string, error) {
+	var instance *compute.Instance
+	var err error
+
+	if ins == nil {
+		instance, err = m.getHostInstance(zone, host)
+	} else {
+		instance, err = ins, nil
+	}
+
 	if err != nil {
 		return "", err
 	}
@@ -94,14 +102,22 @@ func (m *GCEInstanceManager) GetHostAddr(zone string, host string) (string, erro
 	return instance.NetworkInterfaces[0].NetworkIP, nil
 }
 
-func (m *GCEInstanceManager) GetHostURL(zone string, host string) (*url.URL, error) {
-	addr, err := m.GetHostAddr(zone, host)
+func (m *GCEInstanceManager) GetHostAddr(zone string, host string) (string, error) {
+	return m.getHostAddrInner(zone, host, nil)
+}
+
+func (m *GCEInstanceManager) getHostURLInner(zone string, host string, ins *compute.Instance) (*url.URL, error) {
+	addr, err := m.getHostAddrInner(zone, host, nil)
 	if err != nil {
 		return nil, err
 	}
 	return url.Parse(fmt.Sprintf("%s://%s:%d", m.Config.HostOrchestratorProtocol, addr, m.Config.GCP.HostOrchestratorPort))
 }
 
+func (m *GCEInstanceManager) GetHostURL(zone string, host string) (*url.URL, error) {
+	return m.getHostURLInner(zone, host, nil)
+}
+
 const operationStatusDone = "DONE"
 
 func (m *GCEInstanceManager) CreateHost(zone string, req *apiv1.CreateHostRequest, user accounts.User) (*apiv1.Operation, error) {
@@ -253,18 +269,27 @@ func (m *GCEInstanceManager) WaitOperation(zone string, user accounts.User, name
 	if op.Status != operationStatusDone {
 		return nil, errors.NewServiceUnavailableError("Wait for operation timed out", nil)
 	}
-	getter := opResultGetter{Service: m.Service, Op: op}
+	getter := opResultGetter{
+		Service:         m.Service,
+		Op:              op,
+		InstanceManager: m,
+		Zone:            zone,
+	}
 	return getter.Get()
 }
 
-func (m *GCEInstanceManager) GetHostClient(zone string, host string) (HostClient, error) {
-	url, err := m.GetHostURL(zone, host)
+func (m *GCEInstanceManager) getHostClientInner(zone string, host string, ins *compute.Instance) (HostClient, error) {
+	url, err := m.getHostURLInner(zone, host, ins)
 	if err != nil {
 		return nil, err
 	}
 	return NewNetHostClient(url, m.Config.AllowSelfSignedHostSSLCertificate), nil
 }
 
+func (m *GCEInstanceManager) GetHostClient(zone string, host string) (HostClient, error) {
+	return m.getHostClientInner(zone, host, nil)
+}
+
 func (m *GCEInstanceManager) getHostInstance(zone string, host string) (*compute.Instance, error) {
 	ins, err := m.Service.Instances.
 		Get(m.Config.GCP.ProjectID, zone, host).
@@ -324,8 +349,10 @@ var (
 )
 
 type opResultGetter struct {
-	Service *compute.Service
-	Op      *compute.Operation
+	Service         *compute.Service
+	Op              *compute.Operation
+	InstanceManager *GCEInstanceManager
+	Zone            string
 }
 
 func (g *opResultGetter) Get() (any, error) {
@@ -362,7 +389,19 @@ func (g *opResultGetter) buildCreateInstanceResult() (*apiv1.HostInstance, error
 	if err != nil {
 		return nil, toAppError(err)
 	}
-	return BuildHostInstance(ins)
+
+	host, err := BuildHostInstance(ins)
+	if err != nil {
+		return nil, err
+	}
+	client, err := g.InstanceManager.getHostClientInner(g.Zone, host.Name, ins)
+	if err != nil {
+		return nil, err
+	}
+	if err := client.WaitForHostReady(); err != nil {
+		return nil, err
+	}
+	return host, nil
 }
 
 // Converts compute API errors to AppError if relevant, return the same error otherwise

pkg/app/instances/gce_test.go

@@ -24,6 +24,7 @@ import (
 	"net/http/httptest"
 	"net/url"
 	"reflect"
+	"strconv"
 	"strings"
 	"testing"
 
@@ -572,9 +573,14 @@ func TestWaitCreateInstanceOperationSucceeds(t *testing.T) {
 		Name:           "foo",
 		MachineType:    "mt",
 		MinCpuPlatform: "mcp",
+		NetworkInterfaces: []*compute.NetworkInterface{
+			{NetworkIP: "127.0.0.1"},
+		},
 	}
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		switch path := r.URL.Path; path {
+		case "/":
+			replyJSON(w, map[string]string{})
 		case "/projects/google.com:test-project/zones/us-central1-a/operations/operation-1/wait":
 			replyJSON(w, operation)
 		case "/projects/google.com:test-project/zones/us-central1-a/instances/foo":
@@ -584,7 +590,18 @@ func TestWaitCreateInstanceOperationSucceeds(t *testing.T) {
 		}
 	}))
 	defer ts.Close()
-	im := NewGCEInstanceManager(testConfig, buildTestService(t, ts), testNameGenerator)
+
+	tsURL, _ := url.Parse(ts.URL)
+	port, _ := strconv.Atoi(tsURL.Port())
+
+	cfg := testConfig
+	cfg.HostOrchestratorProtocol = "http"
+
+	gcpCfg := *cfg.GCP
+	gcpCfg.HostOrchestratorPort = port
+	cfg.GCP = &gcpCfg
+
+	im := NewGCEInstanceManager(cfg, buildTestService(t, ts), testNameGenerator)
 
 	res, _ := im.WaitOperation(zone, &TestUser{}, opName)
 

pkg/app/instances/hostclient.go

@@ -23,6 +23,7 @@ import (
 	"net/http"
 	"net/http/httputil"
 	"net/url"
+	"time"
 
 	apiv1 "github.com/google/cloud-android-orchestration/api/v1"
 )
@@ -94,6 +95,21 @@ func (c *NetHostClient) Post(path, query string, bodyJSON any, out *HostResponse
 	return res.StatusCode, err
 }
 
+func (c *NetHostClient) WaitForHostReady() error {
+	maxWait := 2 * time.Minute
+	retryDelay := 5 * time.Second
+	deadline := time.Now().Add(maxWait)
+
+	for time.Now().Before(deadline) {
+		status, err := c.Get("/", "", nil)
+		if err == nil && status != http.StatusBadGateway {
+			return nil
+		}
+		time.Sleep(retryDelay)
+	}
+	return fmt.Errorf("wait for host orchestrator timed out")
+}
+
 func (c *NetHostClient) GetReverseProxy() *httputil.ReverseProxy {
 	devProxy := httputil.NewSingleHostReverseProxy(c.url)
 	if c.client != http.DefaultClient {

pkg/app/instances/instances.go

@@ -44,6 +44,7 @@ type HostClient interface {
 	Get(URLPath, URLQuery string, res *HostResponse) (int, error)
 	Post(URLPath, URLQuery string, bodyJSON any, res *HostResponse) (int, error)
 	GetReverseProxy() *httputil.ReverseProxy
+	WaitForHostReady() error
 }
 
 type HostResponse struct {

pkg/client/client.go

@@ -126,19 +126,6 @@ func (c *clientImpl) CreateHost(req *apiv1.CreateHostRequest) (*apiv1.HostInstan
 		return nil, err
 	}
 
-	// There is a short delay between the creation of the host and the availability of the host
-	// orchestrator. This call ensures the host orchestrator had time to start before returning
-	// from the this function.
-	retryOpts := hoclient.RetryOptions{
-		StatusCodes: []int{http.StatusBadGateway},
-		RetryDelay:  5 * time.Second,
-		MaxWait:     2 * time.Minute,
-	}
-	hostPath := fmt.Sprintf("/hosts/%s/", ins.Name)
-	if err := c.httpHelper.NewGetRequest(hostPath).JSONResDoWithRetries(nil, retryOpts); err != nil {
-		return nil, fmt.Errorf("unable to communicate with host orchestrator: %w", err)
-	}
-
 	return ins, nil
 }