test: make health-checker quieter (#8671)

Instead of logging each check, log final failures. Also, if a service is
slow to become healthy, log that.

Right now, the `boulder-ra-sct-provider` instances take about 5s to
become healthy in our CI runs, and health-checker spams the logs a lot
during that period. We should improve the startup time, but this is a
quick fix to reduce the log spam.

Author: jsha

test/health-checker/main.go

@@ -59,7 +59,8 @@ func main() {
 	ctx, cancel := context.WithTimeout(context.Background(), 10*c.GRPC.Timeout.Duration)
 	defer cancel()
 
-	for {
+	start := time.Now()
+	for i := 1; ; i++ {
 		select {
 		case <-ticker.C:
 			_, hostOverride, err := c.GRPC.MakeTargetAndHostOverride()
@@ -84,8 +85,11 @@ func main() {
 				if strings.Contains(err.Error(), "authentication handshake failed") {
 					cmd.Fail(fmt.Sprintf("health checking %s (%s): %s\n", c.GRPC.HostOverride, *serverAddr, err))
 				}
-				fmt.Fprintf(os.Stderr, "health checking %s (%s): %s\n", c.GRPC.HostOverride, *serverAddr, err)
 			} else if resp.Status == healthpb.HealthCheckResponse_SERVING {
+				elapsed := time.Since(start)
+				if elapsed > 1*time.Second {
+					fmt.Printf("service %s is healthy after %s with %d tries\n", *serverAddr, elapsed, i)
+				}
 				return
 			} else {
 				cmd.Fail(fmt.Sprintf("service %s failed health check with status %s", *serverAddr, resp.Status))