Move waiting logic for host orchestrator settlement to backend

Context: b/501288123

Author: k311093

pkg/app/app_test.go

@@ -111,6 +111,10 @@ func (hc *testHostClient) GetReverseProxy() *httputil.ReverseProxy {
 	return httputil.NewSingleHostReverseProxy(hc.url)
 }
 
+func (hc *testHostClient) WaitForHostReady() error {
+	return nil
+}
+
 func TestListZonesSucceeds(t *testing.T) {
 	controller := NewApp(&testInstanceManager{}, &testAccountManager{}, nil, nil, nil, "", nil, apiv1.InfraConfig{}, &config.Config{})
 	ts := httptest.NewServer(controller.Handler())

pkg/app/instances/docker.go

@@ -188,6 +188,13 @@ func (m *DockerInstanceManager) waitCreateHostOperation(host string) (*apiv1.Hos
 				return nil, fmt.Errorf("failed to inspect docker container: %w", err)
 			}
 			if res.State.Running {
+				client, err := m.GetHostClient("local", host)
+				if err != nil {
+					return nil, err
+				}
+				if err := client.WaitForHostReady(); err != nil {
+					return nil, err
+				}
 				return &apiv1.HostInstance{
 					Name: host,
 				}, nil

pkg/app/instances/gce.go

@@ -78,8 +78,16 @@ func (m *GCEInstanceManager) ListZones() (*apiv1.ListZonesResponse, error) {
 	}, nil
 }
 
-func (m *GCEInstanceManager) GetHostAddr(zone string, host string) (string, error) {
-	instance, err := m.getHostInstance(zone, host)
+func (m *GCEInstanceManager) getHostAddrInner(zone string, host string, ins *compute.Instance) (string, error) {
+	var instance *compute.Instance
+	var err error
+
+	if ins == nil {
+		instance, err = m.getHostInstance(zone, host)
+	} else {
+		instance, err = ins, nil
+	}
+
 	if err != nil {
 		return "", err
 	}
@@ -94,14 +102,22 @@ func (m *GCEInstanceManager) GetHostAddr(zone string, host string) (string, erro
 	return instance.NetworkInterfaces[0].NetworkIP, nil
 }
 
-func (m *GCEInstanceManager) GetHostURL(zone string, host string) (*url.URL, error) {
-	addr, err := m.GetHostAddr(zone, host)
+func (m *GCEInstanceManager) GetHostAddr(zone string, host string) (string, error) {
+	return m.getHostAddrInner(zone, host, nil)
+}
+
+func (m *GCEInstanceManager) getHostURLInner(zone string, host string, ins *compute.Instance) (*url.URL, error) {
+	addr, err := m.getHostAddrInner(zone, host, nil)
 	if err != nil {
 		return nil, err
 	}
 	return url.Parse(fmt.Sprintf("%s://%s:%d", m.Config.HostOrchestratorProtocol, addr, m.Config.GCP.HostOrchestratorPort))
 }
 
+func (m *GCEInstanceManager) GetHostURL(zone string, host string) (*url.URL, error) {
+	return m.getHostURLInner(zone, host, nil)
+}
+
 const operationStatusDone = "DONE"
 
 func (m *GCEInstanceManager) CreateHost(zone string, req *apiv1.CreateHostRequest, user accounts.User) (*apiv1.Operation, error) {
@@ -253,18 +269,27 @@ func (m *GCEInstanceManager) WaitOperation(zone string, user accounts.User, name
 	if op.Status != operationStatusDone {
 		return nil, errors.NewServiceUnavailableError("Wait for operation timed out", nil)
 	}
-	getter := opResultGetter{Service: m.Service, Op: op}
+	getter := opResultGetter{
+		Service:         m.Service,
+		Op:              op,
+		InstanceManager: m,
+		Zone:            zone,
+	}
 	return getter.Get()
 }
 
-func (m *GCEInstanceManager) GetHostClient(zone string, host string) (HostClient, error) {
-	url, err := m.GetHostURL(zone, host)
+func (m *GCEInstanceManager) getHostClientInner(zone string, host string, ins *compute.Instance) (HostClient, error) {
+	url, err := m.getHostURLInner(zone, host, ins)
 	if err != nil {
 		return nil, err
 	}
 	return NewNetHostClient(url, m.Config.AllowSelfSignedHostSSLCertificate), nil
 }
 
+func (m *GCEInstanceManager) GetHostClient(zone string, host string) (HostClient, error) {
+	return m.getHostClientInner(zone, host, nil)
+}
+
 func (m *GCEInstanceManager) getHostInstance(zone string, host string) (*compute.Instance, error) {
 	ins, err := m.Service.Instances.
 		Get(m.Config.GCP.ProjectID, zone, host).
@@ -324,8 +349,10 @@ var (
 )
 
 type opResultGetter struct {
-	Service *compute.Service
-	Op      *compute.Operation
+	Service         *compute.Service
+	Op              *compute.Operation
+	InstanceManager *GCEInstanceManager
+	Zone            string
 }
 
 func (g *opResultGetter) Get() (any, error) {
@@ -362,7 +389,19 @@ func (g *opResultGetter) buildCreateInstanceResult() (*apiv1.HostInstance, error
 	if err != nil {
 		return nil, toAppError(err)
 	}
-	return BuildHostInstance(ins)
+
+	host, err := BuildHostInstance(ins)
+	if err != nil {
+		return nil, err
+	}
+	client, err := g.InstanceManager.getHostClientInner(g.Zone, host.Name, ins)
+	if err != nil {
+		return nil, err
+	}
+	if err := client.WaitForHostReady(); err != nil {
+		return nil, err
+	}
+	return host, nil
 }
 
 // Converts compute API errors to AppError if relevant, return the same error otherwise

pkg/app/instances/gce_test.go

@@ -24,6 +24,7 @@ import (
 	"net/http/httptest"
 	"net/url"
 	"reflect"
+	"strconv"
 	"strings"
 	"testing"
 
@@ -572,9 +573,14 @@ func TestWaitCreateInstanceOperationSucceeds(t *testing.T) {
 		Name:           "foo",
 		MachineType:    "mt",
 		MinCpuPlatform: "mcp",
+		NetworkInterfaces: []*compute.NetworkInterface{
+			{NetworkIP: "127.0.0.1"},
+		},
 	}
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		switch path := r.URL.Path; path {
+		case "/":
+			replyJSON(w, map[string]string{})
 		case "/projects/google.com:test-project/zones/us-central1-a/operations/operation-1/wait":
 			replyJSON(w, operation)
 		case "/projects/google.com:test-project/zones/us-central1-a/instances/foo":
@@ -584,7 +590,18 @@ func TestWaitCreateInstanceOperationSucceeds(t *testing.T) {
 		}
 	}))
 	defer ts.Close()
-	im := NewGCEInstanceManager(testConfig, buildTestService(t, ts), testNameGenerator)
+
+	tsURL, _ := url.Parse(ts.URL)
+	port, _ := strconv.Atoi(tsURL.Port())
+
+	cfg := testConfig
+	cfg.HostOrchestratorProtocol = "http"
+
+	gcpCfg := *cfg.GCP
+	gcpCfg.HostOrchestratorPort = port
+	cfg.GCP = &gcpCfg
+
+	im := NewGCEInstanceManager(cfg, buildTestService(t, ts), testNameGenerator)
 
 	res, _ := im.WaitOperation(zone, &TestUser{}, opName)
 

pkg/app/instances/hostclient.go

@@ -23,6 +23,7 @@ import (
 	"net/http"
 	"net/http/httputil"
 	"net/url"
+	"time"
 
 	apiv1 "github.com/google/cloud-android-orchestration/api/v1"
 )
@@ -94,6 +95,21 @@ func (c *NetHostClient) Post(path, query string, bodyJSON any, out *HostResponse
 	return res.StatusCode, err
 }
 
+func (c *NetHostClient) WaitForHostReady() error {
+	maxWait := 2 * time.Minute
+	retryDelay := 5 * time.Second
+	deadline := time.Now().Add(maxWait)
+
+	for time.Now().Before(deadline) {
+		status, err := c.Get("/", "", nil)
+		if err == nil && status != http.StatusBadGateway {
+			return nil
+		}
+		time.Sleep(retryDelay)
+	}
+	return fmt.Errorf("wait for host orchestrator timed out")
+}
+
 func (c *NetHostClient) GetReverseProxy() *httputil.ReverseProxy {
 	devProxy := httputil.NewSingleHostReverseProxy(c.url)
 	if c.client != http.DefaultClient {

pkg/app/instances/instances.go

@@ -44,6 +44,7 @@ type HostClient interface {
 	Get(URLPath, URLQuery string, res *HostResponse) (int, error)
 	Post(URLPath, URLQuery string, bodyJSON any, res *HostResponse) (int, error)
 	GetReverseProxy() *httputil.ReverseProxy
+	WaitForHostReady() error
 }
 
 type HostResponse struct {

pkg/client/client.go

@@ -126,19 +126,6 @@ func (c *clientImpl) CreateHost(req *apiv1.CreateHostRequest) (*apiv1.HostInstan
 		return nil, err
 	}
 
-	// There is a short delay between the creation of the host and the availability of the host
-	// orchestrator. This call ensures the host orchestrator had time to start before returning
-	// from the this function.
-	retryOpts := hoclient.RetryOptions{
-		StatusCodes: []int{http.StatusBadGateway},
-		RetryDelay:  5 * time.Second,
-		MaxWait:     2 * time.Minute,
-	}
-	hostPath := fmt.Sprintf("/hosts/%s/", ins.Name)
-	if err := c.httpHelper.NewGetRequest(hostPath).JSONResDoWithRetries(nil, retryOpts); err != nil {
-		return nil, fmt.Errorf("unable to communicate with host orchestrator: %w", err)
-	}
-
 	return ins, nil
 }