Skip to content

Rework health report collection #810

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ekharkunov merged 1 commit into from
Nov 17, 2025
Merged

Rework health report collection #810

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 23 additions & 21 deletions server/src/main/java/com/defold/extender/services/HealthReporterService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
@Service
public class HealthReporterService {

@Value("${extender.health-reporter.connection-timeout:1500}") int connectionTimeout;
@Value("${extender.health-reporter.connection-timeout:5000}") int connectionTimeout;
public enum OperationalStatus {
Unreachable,
NotFullyOperational,
Expand All @@ -59,27 +59,30 @@ public String collectHealthReport(boolean isRemoteBuildEnabled, Map<String, Remo
// we collect information by platform. If one of the builder is unreachable - set status to "not fully operational"
Map<String, OperationalStatus> platformOperationalStatus = new HashMap<>();
JSONObject result = new JSONObject();
JSONParser parser = new JSONParser();
Map<String, CompletableFuture<Boolean>> reportResults = new HashMap<>(remoteBuilderPlatformMappings.size());
List<HttpGet> runningRequests = new ArrayList<>();
for (Map.Entry<String, RemoteInstanceConfig> entry : remoteBuilderPlatformMappings.entrySet()) {
CompletableFuture<Boolean> statusRequest = CompletableFuture.supplyAsync(() -> {
String instanceId = entry.getValue().getInstanceId();
// if instance controlled by instanceService and is currently suspended or suspending - mark it as 'operational'
if (instanceService != null && instanceService.isInstanceControlled(instanceId)) {
if (instanceService.isInstanceSuspended(instanceId)
|| instanceService.isInstanceSuspending(instanceId)
|| instanceService.isInstanceStarting(instanceId)) {
return Boolean.TRUE;
} else if (!instanceService.isInstanceRunning(instanceId)) {
return Boolean.FALSE;
}
}
final String healthUrl = String.format("%s/health_report", entry.getValue().getUrl());
final HttpGet request = new HttpGet(healthUrl);
synchronized(runningRequests) {
runningRequests.add(request);
String instanceId = entry.getValue().getInstanceId();
String platform = getPlatform(entry.getKey());
// if instance controlled by instanceService and is currently suspended or suspending - mark it as 'operational'
if (instanceService != null && instanceService.isInstanceControlled(instanceId)) {
if (instanceService.isInstanceSuspended(instanceId)
|| instanceService.isInstanceSuspending(instanceId)
|| instanceService.isInstanceStarting(instanceId)) {
updateOperationalStatus(platformOperationalStatus, platform, true);
continue;
} else if (!instanceService.isInstanceRunning(instanceId)) {
updateOperationalStatus(platformOperationalStatus, platform, false);
continue;
}
}

// if instance is not located in GCP - make http request to it asynchronously
final String healthUrl = String.format("%s/health_report", entry.getValue().getUrl());
final HttpGet request = new HttpGet(healthUrl);
runningRequests.add(request);
CompletableFuture<Boolean> innerRequest = CompletableFuture.supplyAsync(() -> {
JSONParser parser = new JSONParser();
try {
HttpResponse response = httpClient.execute(request);
if (response.getStatusLine().getStatusCode() == org.apache.http.HttpStatus.SC_OK) {
Expand All @@ -97,9 +100,8 @@ public String collectHealthReport(boolean isRemoteBuildEnabled, Map<String, Remo
} catch(Exception exc) {
return Boolean.FALSE;
}
});
statusRequest.completeOnTimeout(Boolean.FALSE, this.connectionTimeout, TimeUnit.MILLISECONDS);
reportResults.put(entry.getKey(), statusRequest);
}).completeOnTimeout(Boolean.FALSE, this.connectionTimeout, TimeUnit.MILLISECONDS);
reportResults.put(entry.getKey(), innerRequest);
}
for (Map.Entry<String, CompletableFuture<Boolean>> status : reportResults.entrySet()) {
String platform = getPlatform(status.getKey());
Expand Down
177 changes: 177 additions & 0 deletions server/src/test/java/com/defold/extender/services/HealthReporterServiceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,177 @@
package com.defold.extender.services;

import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
import static com.github.tomakehurst.wiremock.client.WireMock.get;
import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
import static org.junit.jupiter.api.Assertions.assertEquals;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collections;
import java.util.Map;
import java.util.Optional;

import org.apache.commons.io.FileUtils;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;

import com.defold.extender.remote.RemoteInstanceConfig;
import com.defold.extender.services.HealthReporterService.OperationalStatus;
import com.github.tomakehurst.wiremock.WireMockServer;
import com.github.tomakehurst.wiremock.core.WireMockConfiguration;

import io.micrometer.tracing.propagation.Propagator;
import io.micrometer.tracing.test.simple.SimpleTracer;

public class HealthReporterServiceTest {
private static final String OPERATIONAL_RESPONSE = JSONObject.toJSONString(Collections.singletonMap("status", OperationalStatus.Operational.toString()));
private static final String UNREACHABLE_RESPONSE = JSONObject.toJSONString(Collections.singletonMap("status", OperationalStatus.Unreachable.toString()));

private static Path tmpHTTPRoot = Path.of("/tmp/__health_check_http");

private static WireMockServer normalServer1;
private static WireMockServer normalServer2;
private static WireMockServer unreachableServer;
private static WireMockServer timeoutServer;

private static HealthReporterService service;

@BeforeAll
public static void beforeAll() throws IOException {
Files.createDirectories(tmpHTTPRoot);

normalServer1 = new WireMockServer(WireMockConfiguration.options()
.port(9678)
.withRootDirectory(tmpHTTPRoot.toString())
);
normalServer1.start();

normalServer1.stubFor(get(urlEqualTo("/health_report"))
.willReturn(aResponse()
.withStatus(200)
.withBody(OPERATIONAL_RESPONSE)
.withHeader("Content-Type", "application/json")));

normalServer2 = new WireMockServer(WireMockConfiguration.options()
.port(9679)
.withRootDirectory(tmpHTTPRoot.toString())
);
normalServer2.start();

normalServer2.stubFor(get(urlEqualTo("/health_report"))
.willReturn(aResponse()
.withStatus(200)
.withBody(OPERATIONAL_RESPONSE)
.withHeader("Content-Type", "application/json")));

unreachableServer = new WireMockServer(WireMockConfiguration.options()
.port(9680)
.withRootDirectory(tmpHTTPRoot.toString())
);
unreachableServer.start();

unreachableServer.stubFor(get(urlEqualTo("/health_report"))
.willReturn(aResponse()
.withStatus(200)
.withBody(UNREACHABLE_RESPONSE)
.withHeader("Content-Type", "application/json")));

timeoutServer = new WireMockServer(WireMockConfiguration.options()
.port(9681)
.withRootDirectory(tmpHTTPRoot.toString())
);
timeoutServer.start();

timeoutServer.stubFor(get(urlEqualTo("/health_report"))
.willReturn(aResponse()
.withFixedDelay(15000)
.withStatus(200)
.withBody(OPERATIONAL_RESPONSE)
.withHeader("Content-Type", "application/json")));

service = new HealthReporterService(Optional.empty(), new SimpleTracer(), Propagator.NOOP);
service.connectionTimeout = 5000;
}

@AfterAll
public static void afterAll() throws IOException {
if (normalServer1 != null) {
normalServer1.stop();
}
if (normalServer2 != null) {
normalServer2.stop();
}
if (unreachableServer != null) {
unreachableServer.stop();
}
if (timeoutServer != null) {
timeoutServer.stop();
}

FileUtils.deleteDirectory(tmpHTTPRoot.toFile());
}

@Test
public void testSingleNode() {
String result = service.collectHealthReport(false, Map.of());
assertEquals(result, OPERATIONAL_RESPONSE);
}

@Test
public void testRemoteNodesNormal() throws ParseException {
Map<String, RemoteInstanceConfig> conf = Map.of(
"linux-latest", new RemoteInstanceConfig("http://localhost:9678", "linux-latest", true),
"emsdk-406", new RemoteInstanceConfig("http://localhost:9679", "emsdk-406", true)
);
JSONObject expected = new JSONObject(Map.of(
"linux", "Operational",
"emsdk", "Operational"
));
String response = service.collectHealthReport(true, conf);
JSONParser jsonParser = new JSONParser();
JSONObject result = (JSONObject)jsonParser.parse(response);
assertEquals(expected, result);
}

@Test
public void testRemoteNodesUnreachable() {
Map<String, RemoteInstanceConfig> conf = Map.of(
"android-latest", new RemoteInstanceConfig("http://localhost:9680", "android-latest", true)
);
Map<String, String> expected = Map.of(
"android", "Unreachable"
);
String result = service.collectHealthReport(true, conf);
assertEquals(JSONObject.toJSONString(expected), result);
}

@Test
public void testRemoteNodesTimeout() {
Map<String, RemoteInstanceConfig> conf = Map.of(
"windows-latest", new RemoteInstanceConfig("http://localhost:9681", "windows-latest", true)
);
Map<String, Object> expected = Map.of(
"windows", "Unreachable"
);
String result = service.collectHealthReport(true, conf);
assertEquals(JSONObject.toJSONString(expected), result);
}

@Test
public void testRemoteNodesNotFullyOperational() {
Map<String, RemoteInstanceConfig> conf = Map.of(
"android-ndk25", new RemoteInstanceConfig("http://localhost:9679", "android-ndk25", true),
"android-latest", new RemoteInstanceConfig("http://localhost:9681", "android-latest", true)
);
Map<String, Object> expected = Map.of(
"android", "NotFullyOperational"
);
String result = service.collectHealthReport(true, conf);
assertEquals(JSONObject.toJSONString(expected), result);
}
}