Add waiting logic for host orchestrator settlement to backend for Docker

While interacting with host orchestrator right after creating host API is returned, there is a possibility of getting 503 service temporarily unavailable error because host is created and booted but host orchestrator is not executed / ready yet.

Handling logic for waiting host orchestrator readiness is currently in the client logic. Moving this logic into cloud orchestrator API endpoint makes the client simpler and reduces redundant API endpoints.

Apply it first for Docker instance manager.

Context: b/501288123

Author: k311093

pkg/app/app_test.go

@@ -107,6 +107,7 @@ func (hc *testHostClient) Post(path, query string, bodyJSON any, res *instances.
 	return 200, nil
 }
 
+
 func (hc *testHostClient) GetReverseProxy() *httputil.ReverseProxy {
 	return httputil.NewSingleHostReverseProxy(hc.url)
 }

pkg/app/instances/docker.go

@@ -188,6 +188,18 @@ func (m *DockerInstanceManager) waitCreateHostOperation(host string) (*apiv1.Hos
 				return nil, fmt.Errorf("failed to inspect docker container: %w", err)
 			}
 			if res.State.Running {
+				client, err := m.GetHostClient("local", host)
+				if err != nil {
+					return nil, err
+				}
+				// There is a delay between host creation and its readiness, wait for host readiness and return when it is ready.
+				readinessChecker, ok := client.(ReadinessChecker)
+				if !ok {
+					readinessChecker = &defaultReadinessChecker{client}
+				}
+				if err := WaitForHostReady(readinessChecker, 2*time.Minute, 5*time.Second); err != nil {
+					return nil, err
+				}
 				return &apiv1.HostInstance{
 					Name: host,
 				}, nil

pkg/app/instances/helper.go

@@ -0,0 +1,50 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package instances
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/google/cloud-android-orchestration/pkg/app/errors"
+)
+
+func WaitForHostReady(client ReadinessChecker, maxWait time.Duration, retryDelay time.Duration) error {
+	deadline := time.Now().Add(maxWait)
+
+	for time.Now().Before(deadline) {
+		ready, err := client.IsHostReady()
+		if err != nil {
+			return err
+		}
+		if ready {
+			return nil
+		}
+		time.Sleep(retryDelay)
+	}
+	return errors.NewServiceUnavailableError("wait for host orchestrator timed out", nil)
+}
+
+type defaultReadinessChecker struct {
+	client HostClient
+}
+
+func (c *defaultReadinessChecker) IsHostReady() (bool, error) {
+	status, err := c.client.Get("/", "", nil)
+	if err != nil || status == http.StatusBadGateway || status == http.StatusServiceUnavailable {
+		return false, nil
+	}
+	return true, nil
+}

pkg/app/instances/helper_test.go

@@ -0,0 +1,94 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package instances
+
+import (
+	"net/http"
+	"net/http/httputil"
+	"testing"
+	"time"
+)
+
+type mockHostClient struct {
+	getFunc func(string, string, *HostResponse) (int, error)
+}
+
+func (c *mockHostClient) Get(path, query string, res *HostResponse) (int, error) {
+	return c.getFunc(path, query, res)
+}
+
+func (c *mockHostClient) Post(path, query string, body any, res *HostResponse) (int, error) {
+	return 0, nil
+}
+
+
+func (c *mockHostClient) GetReverseProxy() *httputil.ReverseProxy {
+	return nil
+}
+
+func TestWaitForHostReadySucceeds(t *testing.T) {
+	client := &mockHostClient{
+		getFunc: func(path, query string, res *HostResponse) (int, error) {
+			return http.StatusOK, nil
+		},
+	}
+
+	err := WaitForHostReady(&defaultReadinessChecker{client}, 100*time.Millisecond, 1*time.Millisecond)
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+}
+
+func TestWaitForHostReadyRetriesOnBadGateway(t *testing.T) {
+	calls := 0
+	client := &mockHostClient{
+		getFunc: func(path, query string, res *HostResponse) (int, error) {
+			calls++
+			if calls == 1 {
+				return http.StatusBadGateway, nil
+			}
+			return http.StatusOK, nil
+		},
+	}
+
+	err := WaitForHostReady(&defaultReadinessChecker{client}, 100*time.Millisecond, 1*time.Millisecond)
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	if calls != 2 {
+		t.Errorf("expected 2 calls, got %d", calls)
+	}
+}
+
+func TestWaitForHostReadyRetriesOnServiceUnavailable(t *testing.T) {
+	calls := 0
+	client := &mockHostClient{
+		getFunc: func(path, query string, res *HostResponse) (int, error) {
+			calls++
+			if calls == 1 {
+				return http.StatusServiceUnavailable, nil
+			}
+			return http.StatusOK, nil
+		},
+	}
+
+	err := WaitForHostReady(&defaultReadinessChecker{client}, 100*time.Millisecond, 1*time.Millisecond)
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+	}
+	if calls != 2 {
+		t.Errorf("expected 2 calls, got %d", calls)
+	}
+}

pkg/app/instances/hostclient.go

@@ -108,6 +108,7 @@ func (c *NetHostClient) GetReverseProxy() *httputil.ReverseProxy {
 	return devProxy
 }
 
+
 func parseReply(res *http.Response, resObj any, resErr *apiv1.Error) error {
 	var err error
 	dec := json.NewDecoder(res.Body)

pkg/app/instances/instances.go

@@ -38,6 +38,10 @@ type Manager interface {
 	GetHostClient(zone string, host string) (HostClient, error)
 }
 
+type ReadinessChecker interface {
+	IsHostReady() (bool, error)
+}
+
 type HostClient interface {
 	// Get and Post requests return the HTTP status code or an error.
 	// The response body is parsed into the res output parameter if provided.