Skip to content

Commit 58b53ba

Browse files
ekharkunovekharkunov
authored
Rework health report collection. Added tests (#810)
1 parent 7dc1d48 commit 58b53ba

2 files changed

Lines changed: 200 additions & 21 deletions

File tree

‎server/src/main/java/com/defold/extender/services/HealthReporterService.java‎

Lines changed: 23 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
@Service
3333
public class HealthReporterService {
3434

35-
@Value("${extender.health-reporter.connection-timeout:1500}") int connectionTimeout;
35+
@Value("${extender.health-reporter.connection-timeout:5000}") int connectionTimeout;
3636
public enum OperationalStatus {
3737
Unreachable,
3838
NotFullyOperational,
@@ -59,27 +59,30 @@ public String collectHealthReport(boolean isRemoteBuildEnabled, Map<String, Remo
5959
// we collect information by platform. If one of the builder is unreachable - set status to "not fully operational"
6060
Map<String, OperationalStatus> platformOperationalStatus = new HashMap<>();
6161
JSONObject result = new JSONObject();
62-
JSONParser parser = new JSONParser();
6362
Map<String, CompletableFuture<Boolean>> reportResults = new HashMap<>(remoteBuilderPlatformMappings.size());
6463
List<HttpGet> runningRequests = new ArrayList<>();
6564
for (Map.Entry<String, RemoteInstanceConfig> entry : remoteBuilderPlatformMappings.entrySet()) {
66-
CompletableFuture<Boolean> statusRequest = CompletableFuture.supplyAsync(() -> {
67-
String instanceId = entry.getValue().getInstanceId();
68-
// if instance controlled by instanceService and is currently suspended or suspending - mark it as 'operational'
69-
if (instanceService != null && instanceService.isInstanceControlled(instanceId)) {
70-
if (instanceService.isInstanceSuspended(instanceId)
71-
|| instanceService.isInstanceSuspending(instanceId)
72-
|| instanceService.isInstanceStarting(instanceId)) {
73-
return Boolean.TRUE;
74-
} else if (!instanceService.isInstanceRunning(instanceId)) {
75-
return Boolean.FALSE;
76-
}
77-
}
78-
final String healthUrl = String.format("%s/health_report", entry.getValue().getUrl());
79-
final HttpGet request = new HttpGet(healthUrl);
80-
synchronized(runningRequests) {
81-
runningRequests.add(request);
65+
String instanceId = entry.getValue().getInstanceId();
66+
String platform = getPlatform(entry.getKey());
67+
// if instance controlled by instanceService and is currently suspended or suspending - mark it as 'operational'
68+
if (instanceService != null && instanceService.isInstanceControlled(instanceId)) {
69+
if (instanceService.isInstanceSuspended(instanceId)
70+
|| instanceService.isInstanceSuspending(instanceId)
71+
|| instanceService.isInstanceStarting(instanceId)) {
72+
updateOperationalStatus(platformOperationalStatus, platform, true);
73+
continue;
74+
} else if (!instanceService.isInstanceRunning(instanceId)) {
75+
updateOperationalStatus(platformOperationalStatus, platform, false);
76+
continue;
8277
}
78+
}
79+
80+
// if instance is not located in GCP - make http request to it asynchronously
81+
final String healthUrl = String.format("%s/health_report", entry.getValue().getUrl());
82+
final HttpGet request = new HttpGet(healthUrl);
83+
runningRequests.add(request);
84+
CompletableFuture<Boolean> innerRequest = CompletableFuture.supplyAsync(() -> {
85+
JSONParser parser = new JSONParser();
8386
try {
8487
HttpResponse response = httpClient.execute(request);
8588
if (response.getStatusLine().getStatusCode() == org.apache.http.HttpStatus.SC_OK) {
@@ -97,9 +100,8 @@ public String collectHealthReport(boolean isRemoteBuildEnabled, Map<String, Remo
97100
} catch(Exception exc) {
98101
return Boolean.FALSE;
99102
}
100-
});
101-
statusRequest.completeOnTimeout(Boolean.FALSE, this.connectionTimeout, TimeUnit.MILLISECONDS);
102-
reportResults.put(entry.getKey(), statusRequest);
103+
}).completeOnTimeout(Boolean.FALSE, this.connectionTimeout, TimeUnit.MILLISECONDS);
104+
reportResults.put(entry.getKey(), innerRequest);
103105
}
104106
for (Map.Entry<String, CompletableFuture<Boolean>> status : reportResults.entrySet()) {
105107
String platform = getPlatform(status.getKey());

‎server/src/test/java/com/defold/extender/services/HealthReporterServiceTest.java‎

Lines changed: 177 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
package com.defold.extender.services;
2+
3+
import static com.github.tomakehurst.wiremock.client.WireMock.aResponse;
4+
import static com.github.tomakehurst.wiremock.client.WireMock.get;
5+
import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
6+
import static org.junit.jupiter.api.Assertions.assertEquals;
7+
8+
import java.io.IOException;
9+
import java.nio.file.Files;
10+
import java.nio.file.Path;
11+
import java.util.Collections;
12+
import java.util.Map;
13+
import java.util.Optional;
14+
15+
import org.apache.commons.io.FileUtils;
16+
import org.json.simple.JSONObject;
17+
import org.json.simple.parser.JSONParser;
18+
import org.json.simple.parser.ParseException;
19+
import org.junit.jupiter.api.AfterAll;
20+
import org.junit.jupiter.api.BeforeAll;
21+
import org.junit.jupiter.api.Test;
22+
23+
import com.defold.extender.remote.RemoteInstanceConfig;
24+
import com.defold.extender.services.HealthReporterService.OperationalStatus;
25+
import com.github.tomakehurst.wiremock.WireMockServer;
26+
import com.github.tomakehurst.wiremock.core.WireMockConfiguration;
27+
28+
import io.micrometer.tracing.propagation.Propagator;
29+
import io.micrometer.tracing.test.simple.SimpleTracer;
30+
31+
public class HealthReporterServiceTest {
32+
private static final String OPERATIONAL_RESPONSE = JSONObject.toJSONString(Collections.singletonMap("status", OperationalStatus.Operational.toString()));
33+
private static final String UNREACHABLE_RESPONSE = JSONObject.toJSONString(Collections.singletonMap("status", OperationalStatus.Unreachable.toString()));
34+
35+
private static Path tmpHTTPRoot = Path.of("/tmp/__health_check_http");
36+
37+
private static WireMockServer normalServer1;
38+
private static WireMockServer normalServer2;
39+
private static WireMockServer unreachableServer;
40+
private static WireMockServer timeoutServer;
41+
42+
private static HealthReporterService service;
43+
44+
@BeforeAll
45+
public static void beforeAll() throws IOException {
46+
Files.createDirectories(tmpHTTPRoot);
47+
48+
normalServer1 = new WireMockServer(WireMockConfiguration.options()
49+
.port(9678)
50+
.withRootDirectory(tmpHTTPRoot.toString())
51+
);
52+
normalServer1.start();
53+
54+
normalServer1.stubFor(get(urlEqualTo("/health_report"))
55+
.willReturn(aResponse()
56+
.withStatus(200)
57+
.withBody(OPERATIONAL_RESPONSE)
58+
.withHeader("Content-Type", "application/json")));
59+
60+
normalServer2 = new WireMockServer(WireMockConfiguration.options()
61+
.port(9679)
62+
.withRootDirectory(tmpHTTPRoot.toString())
63+
);
64+
normalServer2.start();
65+
66+
normalServer2.stubFor(get(urlEqualTo("/health_report"))
67+
.willReturn(aResponse()
68+
.withStatus(200)
69+
.withBody(OPERATIONAL_RESPONSE)
70+
.withHeader("Content-Type", "application/json")));
71+
72+
unreachableServer = new WireMockServer(WireMockConfiguration.options()
73+
.port(9680)
74+
.withRootDirectory(tmpHTTPRoot.toString())
75+
);
76+
unreachableServer.start();
77+
78+
unreachableServer.stubFor(get(urlEqualTo("/health_report"))
79+
.willReturn(aResponse()
80+
.withStatus(200)
81+
.withBody(UNREACHABLE_RESPONSE)
82+
.withHeader("Content-Type", "application/json")));
83+
84+
timeoutServer = new WireMockServer(WireMockConfiguration.options()
85+
.port(9681)
86+
.withRootDirectory(tmpHTTPRoot.toString())
87+
);
88+
timeoutServer.start();
89+
90+
timeoutServer.stubFor(get(urlEqualTo("/health_report"))
91+
.willReturn(aResponse()
92+
.withFixedDelay(15000)
93+
.withStatus(200)
94+
.withBody(OPERATIONAL_RESPONSE)
95+
.withHeader("Content-Type", "application/json")));
96+
97+
service = new HealthReporterService(Optional.empty(), new SimpleTracer(), Propagator.NOOP);
98+
service.connectionTimeout = 5000;
99+
}
100+
101+
@AfterAll
102+
public static void afterAll() throws IOException {
103+
if (normalServer1 != null) {
104+
normalServer1.stop();
105+
}
106+
if (normalServer2 != null) {
107+
normalServer2.stop();
108+
}
109+
if (unreachableServer != null) {
110+
unreachableServer.stop();
111+
}
112+
if (timeoutServer != null) {
113+
timeoutServer.stop();
114+
}
115+
116+
FileUtils.deleteDirectory(tmpHTTPRoot.toFile());
117+
}
118+
119+
@Test
120+
public void testSingleNode() {
121+
String result = service.collectHealthReport(false, Map.of());
122+
assertEquals(result, OPERATIONAL_RESPONSE);
123+
}
124+
125+
@Test
126+
public void testRemoteNodesNormal() throws ParseException {
127+
Map<String, RemoteInstanceConfig> conf = Map.of(
128+
"linux-latest", new RemoteInstanceConfig("http://localhost:9678", "linux-latest", true),
129+
"emsdk-406", new RemoteInstanceConfig("http://localhost:9679", "emsdk-406", true)
130+
);
131+
JSONObject expected = new JSONObject(Map.of(
132+
"linux", "Operational",
133+
"emsdk", "Operational"
134+
));
135+
String response = service.collectHealthReport(true, conf);
136+
JSONParser jsonParser = new JSONParser();
137+
JSONObject result = (JSONObject)jsonParser.parse(response);
138+
assertEquals(expected, result);
139+
}
140+
141+
@Test
142+
public void testRemoteNodesUnreachable() {
143+
Map<String, RemoteInstanceConfig> conf = Map.of(
144+
"android-latest", new RemoteInstanceConfig("http://localhost:9680", "android-latest", true)
145+
);
146+
Map<String, String> expected = Map.of(
147+
"android", "Unreachable"
148+
);
149+
String result = service.collectHealthReport(true, conf);
150+
assertEquals(JSONObject.toJSONString(expected), result);
151+
}
152+
153+
@Test
154+
public void testRemoteNodesTimeout() {
155+
Map<String, RemoteInstanceConfig> conf = Map.of(
156+
"windows-latest", new RemoteInstanceConfig("http://localhost:9681", "windows-latest", true)
157+
);
158+
Map<String, Object> expected = Map.of(
159+
"windows", "Unreachable"
160+
);
161+
String result = service.collectHealthReport(true, conf);
162+
assertEquals(JSONObject.toJSONString(expected), result);
163+
}
164+
165+
@Test
166+
public void testRemoteNodesNotFullyOperational() {
167+
Map<String, RemoteInstanceConfig> conf = Map.of(
168+
"android-ndk25", new RemoteInstanceConfig("http://localhost:9679", "android-ndk25", true),
169+
"android-latest", new RemoteInstanceConfig("http://localhost:9681", "android-latest", true)
170+
);
171+
Map<String, Object> expected = Map.of(
172+
"android", "NotFullyOperational"
173+
);
174+
String result = service.collectHealthReport(true, conf);
175+
assertEquals(JSONObject.toJSONString(expected), result);
176+
}
177+
}

0 commit comments

Comments
 (0)