-
Notifications
You must be signed in to change notification settings - Fork 24
Implemented instance type #712
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
653ab20
93a3cf4
d537c98
370ee23
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
This file was deleted.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1 +1,4 @@ | ||
| extender.gradle.enabled: true | ||
| extender: | ||
| instance-type: BUILDER_ONLY | ||
| gradle: | ||
| enabled: true |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,4 +1,5 @@ | ||
| extender: | ||
| instance-type: FRONTEND_ONLY | ||
| remote-builder: | ||
| enabled: true | ||
| platforms: | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,4 +1,5 @@ | ||
| extender: | ||
| instance-type: BUILDER_ONLY | ||
| # sdk.location: /usr/local/extender/sdk | ||
| cache: | ||
| enabled: true | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,4 +1,5 @@ | ||
| extender: | ||
| instance-type: FRONTEND_ONLY | ||
| remote-builder: | ||
| enabled: true | ||
| platforms: | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -16,6 +16,7 @@ | |
| import org.apache.commons.io.FileUtils; | ||
| import org.apache.commons.io.IOUtils; | ||
| import org.eclipse.jetty.io.EofException; | ||
| import org.json.simple.JSONObject; | ||
| import org.json.simple.parser.ParseException; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
@@ -46,9 +47,14 @@ | |
| import java.util.regex.Matcher; | ||
| import java.util.regex.Pattern; | ||
|
|
||
|
|
||
| @RestController | ||
| public class ExtenderController { | ||
| public enum InstanceType { | ||
| MIXED, | ||
| FRONTEND_ONLY, | ||
| BUILDER_ONLY | ||
| }; | ||
|
|
||
| private static final Logger LOGGER = LoggerFactory.getLogger(ExtenderController.class); | ||
|
|
||
| private static final String LATEST = "latest"; | ||
|
|
@@ -65,6 +71,8 @@ public class ExtenderController { | |
|
|
||
| private final RemoteEngineBuilder remoteEngineBuilder; | ||
| private Map<String, RemoteInstanceConfig> remoteBuilderPlatformMappings; | ||
| @Value("${extender.instance-type:MIXED}") | ||
| private InstanceType instanceType; | ||
| private final boolean remoteBuilderEnabled; | ||
|
|
||
| private static long maxPackageSize = 1024*1024*1024; | ||
|
|
@@ -148,7 +156,7 @@ public void buildEngineAsync(HttpServletRequest _request, | |
| HttpServletResponse response, | ||
| @PathVariable("platform") String platform, | ||
| @PathVariable("sdkVersion") String sdkVersionString) | ||
| throws ExtenderException, IOException, ParseException { | ||
| throws ExtenderException, IOException, ParseException, VersionNotSupportedException, PlatformNotSupportedException { | ||
|
|
||
| boolean isMultipart = JakartaServletFileUpload.isMultipartContent(_request); | ||
| if (!isMultipart) { | ||
|
|
@@ -198,17 +206,33 @@ public void buildEngineAsync(HttpServletRequest _request, | |
| // Regardless of success/fail status, we want to cache the uploaded files | ||
| DataCacheService.DataCacheServiceInfo uploadResultInfo = dataCacheService.cacheFiles(uploadDirectory); | ||
| metricsWriter.measureCacheUpload(uploadResultInfo.cachedFileSize.longValue(), uploadResultInfo.cachedFileCount.intValue()); | ||
|
|
||
| String[] buildEnvDescription = ExtenderUtil.getSdksForPlatform(platform, defoldSdkService.getPlatformSdkMappings(sdkVersion)); | ||
| // Build engine locally or on remote builder | ||
| if (remoteBuilderEnabled && isRemotePlatform(buildEnvDescription[0], buildEnvDescription[1])) { | ||
| LOGGER.info("Building engine on remote builder"); | ||
| RemoteInstanceConfig remoteInstanceConfig = getRemoteBuilderConfig(buildEnvDescription[0], buildEnvDescription[1]); | ||
| this.remoteEngineBuilder.buildAsync(remoteInstanceConfig, uploadDirectory, platform, sdkVersion, jobDirectory, buildDirectory, metricsWriter); | ||
| } else { | ||
|
|
||
| if (instanceType.equals(InstanceType.BUILDER_ONLY)) { | ||
| asyncBuilder.asyncBuildEngine(metricsWriter, platform, sdkVersion, jobDirectory, uploadDirectory, buildDirectory); | ||
| } else { | ||
| String[] buildEnvDescription = null; | ||
| try { | ||
| JSONObject mappings = defoldSdkService.getPlatformSdkMappings(sdkVersion); | ||
| buildEnvDescription = ExtenderUtil.getSdksForPlatform(platform, mappings); | ||
| } catch(ExtenderException exc) { | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Exception happened if we can't find sdk mappings for given hash. |
||
| if (instanceType.equals(InstanceType.FRONTEND_ONLY)) { | ||
| LOGGER.error("Unsupported engine version {}", sdkVersion); | ||
| throw new VersionNotSupportedException(sdkVersion); | ||
| } | ||
| } | ||
| // Build engine locally or on remote builder | ||
| if (remoteBuilderEnabled && buildEnvDescription != null && isRemotePlatform(buildEnvDescription[0], buildEnvDescription[1])) { | ||
| LOGGER.info("Building engine on remote builder"); | ||
| RemoteInstanceConfig remoteInstanceConfig = getRemoteBuilderConfig(buildEnvDescription[0], buildEnvDescription[1]); | ||
| this.remoteEngineBuilder.buildAsync(remoteInstanceConfig, uploadDirectory, platform, sdkVersion, jobDirectory, buildDirectory, metricsWriter); | ||
| } else if (instanceType.equals(InstanceType.MIXED)) { | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If sdk mappings exist but there are no required sdk version defined and instance marked as "MIXED" - try build on current instance. |
||
| asyncBuilder.asyncBuildEngine(metricsWriter, platform, sdkVersion, jobDirectory, uploadDirectory, buildDirectory); | ||
| } else { | ||
| // no remote builder was found and current instance can't build | ||
| LOGGER.error("Unsupported build platform {}", platform); | ||
| throw new PlatformNotSupportedException(platform); | ||
| } | ||
| } | ||
|
|
||
| response.getWriter().write(jobDirectory.getName()); | ||
| response.getWriter().flush(); | ||
| response.getWriter().close(); | ||
|
|
@@ -218,6 +242,8 @@ public void buildEngineAsync(HttpServletRequest _request, | |
| throw new ExtenderException(e, "Client closed connection prematurely, build aborted"); | ||
| } catch(FileUploadException e) { | ||
| throw new ExtenderException(e, "Bad request: " + e.getMessage()); | ||
| } catch(VersionNotSupportedException|PlatformNotSupportedException exc) { | ||
| throw exc; | ||
| } catch(Exception e) { | ||
| LOGGER.error(String.format("Exception while building or sending response - SDK: %s", sdkVersion)); | ||
| throw e; | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| package com.defold.extender; | ||
|
|
||
| public class PlatformNotSupportedException extends ExtenderException { | ||
| private static String ERROR_MESSAGE = "Platform %s is not supported on the current server. Please, check build server address. If error will persist - create task here https://github.com/defold/extender/issues"; | ||
|
|
||
| public PlatformNotSupportedException(String platform) { | ||
| super(String.format(ERROR_MESSAGE, platform)); | ||
| } | ||
|
|
||
| public PlatformNotSupportedException(Exception e, String platform) { | ||
| super(e, String.format(ERROR_MESSAGE, platform)); | ||
| } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,13 @@ | ||
| package com.defold.extender; | ||
|
|
||
| public class VersionNotSupportedException extends ExtenderException { | ||
| private static String ERROR_MESSAGE = "Engine version %s is not supported on the current server. Please, use latest stable version. https://github.com/defold/defold/releases/latest"; | ||
|
|
||
| public VersionNotSupportedException(String version) { | ||
| super(String.format(ERROR_MESSAGE, version)); | ||
| } | ||
|
|
||
| public VersionNotSupportedException(Exception e, String version) { | ||
| super(e, String.format(ERROR_MESSAGE, version)); | ||
| } | ||
| } |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
If instance marked as "BUILDER_ONLY" - skip request for sdk mappings and start building.