Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 0 additions & 5 deletions server/configs/application-dev.yml

This file was deleted.

5 changes: 4 additions & 1 deletion server/configs/application-local-dev-android.yml
Original file line number Diff line number Diff line change
@@ -1 +1,4 @@
extender.gradle.enabled: true
extender:
instance-type: BUILDER_ONLY
gradle:
enabled: true
1 change: 1 addition & 0 deletions server/configs/application-local-dev-app.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
extender:
instance-type: FRONTEND_ONLY
remote-builder:
enabled: true
platforms:
Expand Down
1 change: 1 addition & 0 deletions server/configs/application-local-dev.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
extender:
instance-type: BUILDER_ONLY
# sdk.location: /usr/local/extender/sdk
cache:
enabled: true
Expand Down
1 change: 1 addition & 0 deletions server/configs/application-standalone-dev.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ server:
port: 9010

extender:
instance-type: MIXED
sdk:
location: /usr/local/extender/sdk
cache-clear-on-exit: false
Expand Down
1 change: 1 addition & 0 deletions server/configs/application-test-app.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
extender:
instance-type: FRONTEND_ONLY
remote-builder:
enabled: true
platforms:
Expand Down
48 changes: 37 additions & 11 deletions server/src/main/java/com/defold/extender/ExtenderController.java
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.eclipse.jetty.io.EofException;
import org.json.simple.JSONObject;
import org.json.simple.parser.ParseException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -46,9 +47,14 @@
import java.util.regex.Matcher;
import java.util.regex.Pattern;


@RestController
public class ExtenderController {
public enum InstanceType {
MIXED,
FRONTEND_ONLY,
BUILDER_ONLY
};

private static final Logger LOGGER = LoggerFactory.getLogger(ExtenderController.class);

private static final String LATEST = "latest";
Expand All @@ -65,6 +71,8 @@ public class ExtenderController {

private final RemoteEngineBuilder remoteEngineBuilder;
private Map<String, RemoteInstanceConfig> remoteBuilderPlatformMappings;
@Value("${extender.instance-type:MIXED}")
private InstanceType instanceType;
private final boolean remoteBuilderEnabled;

private static long maxPackageSize = 1024*1024*1024;
Expand Down Expand Up @@ -148,7 +156,7 @@ public void buildEngineAsync(HttpServletRequest _request,
HttpServletResponse response,
@PathVariable("platform") String platform,
@PathVariable("sdkVersion") String sdkVersionString)
throws ExtenderException, IOException, ParseException {
throws ExtenderException, IOException, ParseException, VersionNotSupportedException, PlatformNotSupportedException {

boolean isMultipart = JakartaServletFileUpload.isMultipartContent(_request);
if (!isMultipart) {
Expand Down Expand Up @@ -198,17 +206,33 @@ public void buildEngineAsync(HttpServletRequest _request,
// Regardless of success/fail status, we want to cache the uploaded files
DataCacheService.DataCacheServiceInfo uploadResultInfo = dataCacheService.cacheFiles(uploadDirectory);
metricsWriter.measureCacheUpload(uploadResultInfo.cachedFileSize.longValue(), uploadResultInfo.cachedFileCount.intValue());

String[] buildEnvDescription = ExtenderUtil.getSdksForPlatform(platform, defoldSdkService.getPlatformSdkMappings(sdkVersion));
// Build engine locally or on remote builder
if (remoteBuilderEnabled && isRemotePlatform(buildEnvDescription[0], buildEnvDescription[1])) {
LOGGER.info("Building engine on remote builder");
RemoteInstanceConfig remoteInstanceConfig = getRemoteBuilderConfig(buildEnvDescription[0], buildEnvDescription[1]);
this.remoteEngineBuilder.buildAsync(remoteInstanceConfig, uploadDirectory, platform, sdkVersion, jobDirectory, buildDirectory, metricsWriter);
} else {

if (instanceType.equals(InstanceType.BUILDER_ONLY)) {

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If instance marked as "BUILDER_ONLY" - skip request for sdk mappings and start building.

asyncBuilder.asyncBuildEngine(metricsWriter, platform, sdkVersion, jobDirectory, uploadDirectory, buildDirectory);
} else {
String[] buildEnvDescription = null;
try {
JSONObject mappings = defoldSdkService.getPlatformSdkMappings(sdkVersion);
buildEnvDescription = ExtenderUtil.getSdksForPlatform(platform, mappings);
} catch(ExtenderException exc) {

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Exception happened if we can't find sdk mappings for given hash.

if (instanceType.equals(InstanceType.FRONTEND_ONLY)) {
LOGGER.error("Unsupported engine version {}", sdkVersion);
throw new VersionNotSupportedException(sdkVersion);
}
}
// Build engine locally or on remote builder
if (remoteBuilderEnabled && buildEnvDescription != null && isRemotePlatform(buildEnvDescription[0], buildEnvDescription[1])) {
LOGGER.info("Building engine on remote builder");
RemoteInstanceConfig remoteInstanceConfig = getRemoteBuilderConfig(buildEnvDescription[0], buildEnvDescription[1]);
this.remoteEngineBuilder.buildAsync(remoteInstanceConfig, uploadDirectory, platform, sdkVersion, jobDirectory, buildDirectory, metricsWriter);
} else if (instanceType.equals(InstanceType.MIXED)) {

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If sdk mappings exist but there are no required sdk version defined and instance marked as "MIXED" - try build on current instance.

asyncBuilder.asyncBuildEngine(metricsWriter, platform, sdkVersion, jobDirectory, uploadDirectory, buildDirectory);
} else {
// no remote builder was found and current instance can't build
LOGGER.error("Unsupported build platform {}", platform);
throw new PlatformNotSupportedException(platform);
}
}

response.getWriter().write(jobDirectory.getName());
response.getWriter().flush();
response.getWriter().close();
Expand All @@ -218,6 +242,8 @@ public void buildEngineAsync(HttpServletRequest _request,
throw new ExtenderException(e, "Client closed connection prematurely, build aborted");
} catch(FileUploadException e) {
throw new ExtenderException(e, "Bad request: " + e.getMessage());
} catch(VersionNotSupportedException|PlatformNotSupportedException exc) {
throw exc;
} catch(Exception e) {
LOGGER.error(String.format("Exception while building or sending response - SDK: %s", sdkVersion));
throw e;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package com.defold.extender;

public class PlatformNotSupportedException extends ExtenderException {
private static String ERROR_MESSAGE = "Platform %s is not supported on the current server. Please, check build server address. If error will persist - create task here https://github.com/defold/extender/issues";

public PlatformNotSupportedException(String platform) {
super(String.format(ERROR_MESSAGE, platform));
}

public PlatformNotSupportedException(Exception e, String platform) {
super(e, String.format(ERROR_MESSAGE, platform));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package com.defold.extender;

public class VersionNotSupportedException extends ExtenderException {
private static String ERROR_MESSAGE = "Engine version %s is not supported on the current server. Please, use latest stable version. https://github.com/defold/defold/releases/latest";

public VersionNotSupportedException(String version) {
super(String.format(ERROR_MESSAGE, version));
}

public VersionNotSupportedException(Exception e, String version) {
super(e, String.format(ERROR_MESSAGE, version));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -356,7 +356,9 @@ private JSONObject getRemotePlatformSdkMappings(String hash) throws IOException,
try {
JSONObject result = operation.get();
if (result == null) {
throw new ExtenderException(String.format("Cannot find or parse platform sdks mappings for hash: %s", hash));
String msg = String.format("Cannot find or parse platform sdks mappings for hash: %s", hash);
LOGGER.error(msg);
throw new ExtenderException(msg);
}
return result;
} catch (InterruptedException|ExecutionException exc) {
Expand Down
3 changes: 3 additions & 0 deletions server/src/main/resources/application.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,9 @@ extender:
location: /tmp/results
cleanup-period: 20000
lifetime: 1200000
# see ExtenderController.InstanceType enum
# FRONTEND_ONLY, BUILDER_ONLY, MIXED
instance-type: MIXED

spring:
application:
Expand Down
16 changes: 16 additions & 0 deletions server/src/test/java/com/defold/extender/IntegrationTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import org.jf.dexlib2.iface.DexFile;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
Expand Down Expand Up @@ -575,4 +576,19 @@ public void buildEngineWithDynamicLibs(TestConfiguration configuration) throws I
assertNotNull(zipFile.getEntry(getDynamicLibName(configuration.platform, "dynamic_specific2")));
}
}

@Test
public void testUnsupportedVersion() throws IOException, ExtenderClientException {
TestConfiguration configuration = new TestConfiguration(new DefoldVersion("non-exist", new Version(2, 10, 1), new String[]{ "x86_64-linux" }) , "x86_64-linux");
List<ExtenderResource> sourceFiles = Lists.newArrayList(
new FileExtenderResource("test-data/AndroidManifest.xml", "AndroidManifest.xml"),
new FileExtenderResource("test-data/ext2/ext.manifest"),
new FileExtenderResource("test-data/ext2/src/test_ext.cpp"),
new FileExtenderResource(String.format("test-data/ext2/lib/%s/%s", configuration.platform, getLibName(configuration.platform, "alib"))),
new FileExtenderResource(String.format("test-data/ext2/lib/%s/%s", configuration.platform, getLibName(configuration.platform, "blib")))
);

ExtenderClientException exc = assertThrows(ExtenderClientException.class, () -> doBuild(sourceFiles, configuration));
assertTrue(exc.getMessage().contains("Unsupported engine version"));
}
}