diff --git a/server/envs/macos.env b/server/envs/macos.env index 91029cf5..de31402b 100644 --- a/server/envs/macos.env +++ b/server/envs/macos.env @@ -1,6 +1,6 @@ SWIFT_5_5_VERSION=5.5 IOS_VERSION_MIN=11.0 -MACOS_VERSION_MIN=10.13 +MACOS_VERSION_MIN=10.15 XCODE_16_VERSION=16.2 XCODE_16_CLANG_VERSION=16.0.0 MACOS_15_VERSION=15.2 diff --git a/server/src/main/java/com/defold/extender/Extender.java b/server/src/main/java/com/defold/extender/Extender.java index dc50128f..46e9eade 100644 --- a/server/src/main/java/com/defold/extender/Extender.java +++ b/server/src/main/java/com/defold/extender/Extender.java @@ -83,11 +83,6 @@ class Extender { static final String FOLDER_COMMON_SRC = "commonsrc";// common source shared between both types static final String APPMANIFEST_FILENAME = "app.manifest"; - static final String FRAMEWORK_RE = "(.+)\\.framework"; - static final String JAR_RE = "(.+\\.jar)"; - static final String JS_RE = "(.+\\.js)"; - static final String PROTO_RE = "(?i).*(\\.proto)"; - static final String ENGINE_JAR_RE = "(?:.*)\\/share\\/java\\/[\\w\\-\\.]*\\.jar$"; private static final String MANIFEST_IOS = "Info.plist"; private static final String MANIFEST_OSX = "Info.plist"; @@ -409,9 +404,9 @@ private Map createContext(Map src) throws Extend private List getFrameworks(File dir) { List frameworks = new ArrayList<>(); final String[] platformParts = buildState.fullPlatform.split("-"); - frameworks.addAll(ExtenderUtil.collectDirsByName(new File(dir, "lib" + File.separator + buildState.fullPlatform), FRAMEWORK_RE)); // e.g. armv64-ios + frameworks.addAll(ExtenderUtil.collectDirsByName(new File(dir, "lib" + File.separator + buildState.fullPlatform), ExtenderConst.FRAMEWORK_RE)); // e.g. armv64-ios if (platformParts.length == 2) { - frameworks.addAll(ExtenderUtil.collectDirsByName(new File(dir, "lib" + File.separator + platformParts[1]), FRAMEWORK_RE)); // e.g. "ios" + frameworks.addAll(ExtenderUtil.collectDirsByName(new File(dir, "lib" + File.separator + platformParts[1]), ExtenderConst.FRAMEWORK_RE)); // e.g. "ios" } return frameworks; } @@ -439,10 +434,10 @@ private List getFrameworkPaths(File dir) { private List getExtensionLibJars(File extDir) { List jars = new ArrayList<>(); - jars.addAll(ExtenderUtil.collectFilesByPath(new File(extDir, "lib" + File.separator + buildState.fullPlatform), JAR_RE)); // e.g. armv7-android + jars.addAll(ExtenderUtil.collectFilesByPath(new File(extDir, "lib" + File.separator + buildState.fullPlatform), ExtenderConst.JAR_RE)); // e.g. armv7-android String[] platformParts = buildState.fullPlatform.split("-"); if (platformParts.length == 2) { - jars.addAll(ExtenderUtil.collectFilesByPath(new File(extDir, "lib" + File.separator + platformParts[1]), JAR_RE)); // e.g. "android" + jars.addAll(ExtenderUtil.collectFilesByPath(new File(extDir, "lib" + File.separator + platformParts[1]), ExtenderConst.JAR_RE)); // e.g. "android" } return jars; } @@ -1229,7 +1224,7 @@ private List buildExtensionInternal(File manifest, Map man srcFiles.addAll(ExtenderUtil.listFiles(srcDirs, platformConfig.zigSourceRe)); // Generate C++ files first (output into the source folder) - List protoFiles = ExtenderUtil.listFiles(srcDirs, PROTO_RE); + List protoFiles = ExtenderUtil.listFiles(srcDirs, ExtenderConst.PROTO_RE); List generated = generateProtoCxxForEngine(extDir, manifestContext, protoFiles); if (!protoFiles.isEmpty() && generated.isEmpty()) { throw new ExtenderException(String.format("%s:1: error: Protofiles didn't generate any output engine cpp files!", ExtenderUtil.getRelativePath(buildState.uploadDir, protoFiles.get(0)) )); @@ -1285,7 +1280,7 @@ private List buildPipelineExtension(File manifest, Map man extBuildDir.mkdir(); File[] srcDirs = { new File(extDir, FOLDER_COMMON_SRC), new File(extDir, FOLDER_PLUGIN_SRC) }; - List protoFiles = ExtenderUtil.listFiles(srcDirs, PROTO_RE); + List protoFiles = ExtenderUtil.listFiles(srcDirs, ExtenderConst.PROTO_RE); List outputFiles = new ArrayList<>(); @@ -1425,7 +1420,7 @@ private void getProjectPaths(Map mainContext, Map mainContext, Map getManifestContext(String platform, ManifestCo private File buildMainDexList(List jars) throws ExtenderException { // Find the engine libraries (**/share/java/*.jar) - List mainListJars = ExtenderUtil.filterStrings(jars, ENGINE_JAR_RE); + List mainListJars = ExtenderUtil.filterStrings(jars, ExtenderConst.ENGINE_JAR_RE); if (mainListJars.isEmpty()) { - throw new ExtenderException("Regex failed to find any engine jars: " + ENGINE_JAR_RE); + throw new ExtenderException("Regex failed to find any engine jars: " + ExtenderConst.ENGINE_JAR_RE); } List mainClassNames = new ArrayList(); diff --git a/server/src/main/java/com/defold/extender/ExtenderConst.java b/server/src/main/java/com/defold/extender/ExtenderConst.java new file mode 100644 index 00000000..a234bbf1 --- /dev/null +++ b/server/src/main/java/com/defold/extender/ExtenderConst.java @@ -0,0 +1,11 @@ +package com.defold.extender; + +public class ExtenderConst { + public static final String SOURCE_CODE_ARCHIVE_MAGIC_NAME = "__source_code__.zip"; + public static final String FRAMEWORK_RE = "(.+)\\.framework"; + public static final String JAR_RE = "(.+\\.jar)"; + public static final String JS_RE = "(.+\\.js)"; + public static final String PROTO_RE = "(?i).*(\\.proto)"; + public static final String ENGINE_JAR_RE = "(?:.*)\\/share\\/java\\/[\\w\\-\\.]*\\.jar$"; + +} diff --git a/server/src/main/java/com/defold/extender/ExtenderController.java b/server/src/main/java/com/defold/extender/ExtenderController.java index 4c525e28..77c78ebc 100644 --- a/server/src/main/java/com/defold/extender/ExtenderController.java +++ b/server/src/main/java/com/defold/extender/ExtenderController.java @@ -56,8 +56,6 @@ public class ExtenderController { // Used to verify the uploaded filenames private static final Pattern FILENAME_RE = Pattern.compile("^([\\w ](?:[\\w+\\-\\/ @]|(?:\\.[\\w+\\-\\/ ]*))+)$"); - private static final String SOURCE_CODE_ARCHIVE_MAGIC_NAME = "__source_code__.zip"; - private final DefoldSdkService defoldSdkService; private final DataCacheService dataCacheService; private final MeterRegistry meterRegistry; @@ -388,7 +386,7 @@ static void receiveUpload(MultipartHttpServletRequest request, File uploadDirect } // check if source code archive presented - File sourceCodeArchive = new File(uploadDirectory, SOURCE_CODE_ARCHIVE_MAGIC_NAME); + File sourceCodeArchive = new File(uploadDirectory, ExtenderConst.SOURCE_CODE_ARCHIVE_MAGIC_NAME); if (sourceCodeArchive.exists()) { LOGGER.debug("Source code archive found. Unarchiving..."); ZipUtils.unzip(new FileInputStream(sourceCodeArchive), uploadDirectory.toPath()); diff --git a/server/src/main/java/com/defold/extender/remote/RemoteEngineBuilder.java b/server/src/main/java/com/defold/extender/remote/RemoteEngineBuilder.java index 292565ae..dad00937 100644 --- a/server/src/main/java/com/defold/extender/remote/RemoteEngineBuilder.java +++ b/server/src/main/java/com/defold/extender/remote/RemoteEngineBuilder.java @@ -1,7 +1,10 @@ package com.defold.extender.remote; import com.defold.extender.BuilderConstants; +import com.defold.extender.ExtenderConst; +import com.defold.extender.ExtenderException; import com.defold.extender.metrics.MetricsWriter; +import com.defold.extender.services.DataCacheService; import com.defold.extender.services.GCPInstanceService; import com.defold.extender.tracing.ExtenderTracerInterceptor; @@ -19,10 +22,8 @@ import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.ContentType; import org.apache.http.entity.mime.MultipartEntityBuilder; -import org.apache.http.entity.mime.content.FileBody; -import org.apache.http.entity.mime.content.AbstractContentBody; +import org.apache.http.entity.mime.content.ByteArrayBody; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.util.EntityUtils; import org.slf4j.Logger; @@ -34,13 +35,17 @@ import java.io.OutputStream; import java.io.PrintWriter; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.util.Optional; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; import java.nio.charset.Charset; @Service @@ -91,7 +96,7 @@ public void buildAsync(final RemoteInstanceConfig remoteInstanceConfig, try { httpEntity = buildHttpEntity(projectDirectory); - } catch(IllegalStateException|IOException e) { + } catch(IllegalStateException | IOException | ExtenderException e) { throw new RemoteBuildException("Failed to add files to multipart request", e); } @@ -176,19 +181,34 @@ public void buildAsync(final RemoteInstanceConfig remoteInstanceConfig, } } - HttpEntity buildHttpEntity(final File projectDirectory) throws IOException { - final MultipartEntityBuilder builder = MultipartEntityBuilder.create() - .setContentType(ContentType.MULTIPART_FORM_DATA); - - Files.walk(projectDirectory.toPath()) - .filter(Files::isRegularFile) - .forEach(path -> { - String relativePath = path.toFile().getAbsolutePath().substring(projectDirectory.getAbsolutePath().length() + 1); - AbstractContentBody body = new FileBody(path.toFile(), ContentType.DEFAULT_BINARY, relativePath); - builder.addPart(relativePath, body); - }); - - return builder.build(); + HttpEntity buildHttpEntity(final File projectDirectory) throws IOException, ExtenderException { + MultipartEntityBuilder entityBuilder = MultipartEntityBuilder.create(); + entityBuilder.setStrictMode(); + + ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + try (ZipOutputStream zipStream = new ZipOutputStream(byteStream)) { + Path projectDirectoryPath = projectDirectory.toPath(); + Files.walk(projectDirectoryPath) + .filter(Files::isRegularFile) + .filter(path -> { return !path.getFileName().toString().equals(ExtenderConst.SOURCE_CODE_ARCHIVE_MAGIC_NAME); }) + .filter(path -> { return !path.getFileName().toString().equals(DataCacheService.FILE_CACHE_INFO_FILE); }) + .forEach(res -> { + Path path = projectDirectoryPath.relativize(res); + LOGGER.warn(path.toString()); + try { + ZipEntry entry = new ZipEntry(path.toString()); + zipStream.putNextEntry(entry); + zipStream.write(Files.readAllBytes(res)); + zipStream.closeEntry(); + } catch (IOException e) { + LOGGER.warn("Exception during add entry to source code archive", e); + } + }); + } catch (IOException exc) { + throw new ExtenderException(exc, "Failed to create source code archive"); + } + entityBuilder.addPart(ExtenderConst.SOURCE_CODE_ARCHIVE_MAGIC_NAME, new ByteArrayBody(byteStream.toByteArray(), ExtenderConst.SOURCE_CODE_ARCHIVE_MAGIC_NAME)); + return entityBuilder.build(); } private void touchInstance(String instanceId) { diff --git a/server/src/main/java/com/defold/extender/services/DataCacheService.java b/server/src/main/java/com/defold/extender/services/DataCacheService.java index 811983af..1e89d83d 100644 --- a/server/src/main/java/com/defold/extender/services/DataCacheService.java +++ b/server/src/main/java/com/defold/extender/services/DataCacheService.java @@ -30,7 +30,7 @@ public class DataCacheService { private static final Logger LOGGER = LoggerFactory.getLogger(DataCacheService.class); - static final String FILE_CACHE_INFO_FILE = "ne-cache-info.json"; + public static final String FILE_CACHE_INFO_FILE = "ne-cache-info.json"; static final String FILE_CACHE_INFO_HASH_TYPE = "sha256"; static final int FILE_CACHE_INFO_VERSION = 1; diff --git a/server/src/main/java/com/defold/extender/services/cocoapods/ResolvedPods.java b/server/src/main/java/com/defold/extender/services/cocoapods/ResolvedPods.java index 1992dd7d..1461bf3b 100644 --- a/server/src/main/java/com/defold/extender/services/cocoapods/ResolvedPods.java +++ b/server/src/main/java/com/defold/extender/services/cocoapods/ResolvedPods.java @@ -16,6 +16,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.defold.extender.ExtenderConst; import com.defold.extender.ExtenderException; import com.defold.extender.ExtenderUtil; import com.defold.extender.services.cocoapods.PlistBuddyWrapper.CreateBundlePlistArgs; @@ -23,7 +24,6 @@ public class ResolvedPods { private static final Logger LOGGER = LoggerFactory.getLogger(ResolvedPods.class); - static final String FRAMEWORK_RE = "(.+)\\.framework"; private List pods = new ArrayList<>(); private File podsDir; private File frameworksDir; @@ -127,7 +127,7 @@ List collectAllPodFrameworks() throws IOException { } // collect unpacked xcframeworks - Pattern pattern = Pattern.compile(FRAMEWORK_RE); + Pattern pattern = Pattern.compile(ExtenderConst.FRAMEWORK_RE); Files.walk(frameworksDir.toPath()) .filter(Files::isDirectory) .forEach(path -> { @@ -143,7 +143,7 @@ List collectAllPodFrameworks() throws IOException { List collectAllPodsDynamicFrameworks() throws IOException { Set dynamicFrameworks = new HashSet<>(); // collect unpacked xcframeworks - Pattern pattern = Pattern.compile(FRAMEWORK_RE); + Pattern pattern = Pattern.compile(ExtenderConst.FRAMEWORK_RE); Files.walk(frameworksDir.toPath()) .filter(Files::isDirectory) .forEach(path -> { diff --git a/server/src/test/java/com/defold/extender/ExtenderTest.java b/server/src/test/java/com/defold/extender/ExtenderTest.java index beafe880..799673c9 100644 --- a/server/src/test/java/com/defold/extender/ExtenderTest.java +++ b/server/src/test/java/com/defold/extender/ExtenderTest.java @@ -222,7 +222,7 @@ public void testCollectLibraries() { checkArray(expected, result); } { - List result = ExtenderUtil.collectDirsByName(new File("test-data/ext/lib/x86_64-osx"), Extender.FRAMEWORK_RE); + List result = ExtenderUtil.collectDirsByName(new File("test-data/ext/lib/x86_64-osx"), ExtenderConst.FRAMEWORK_RE); String[] expected = {"blib"}; checkArray(expected, result); } @@ -238,7 +238,7 @@ public void testCollectJars() { String[] endings = {"test-data/ext/lib/android/Dummy.jar", "test-data/ext/lib/android/JarDep.jar", "test-data/ext/lib/android/VeryLarge1.jar", "test-data/ext/lib/android/VeryLarge2.jar", "test-data/ext/lib/android/meta-inf.jar"}; - List paths = ExtenderUtil.collectFilesByPath(new File("test-data/ext/lib/android"), Extender.JAR_RE); + List paths = ExtenderUtil.collectFilesByPath(new File("test-data/ext/lib/android"), ExtenderConst.JAR_RE); assertEquals(endings.length, paths.size()); @@ -256,7 +256,7 @@ public void testCollectJars() { @Test public void testCollectJsFiles() { - List result = ExtenderUtil.collectFilesByPath(new File("test-data/ext/lib/js-web"), Extender.JS_RE); + List result = ExtenderUtil.collectFilesByPath(new File("test-data/ext/lib/js-web"), ExtenderConst.JS_RE); assertEquals(1, result.size()); assertTrue(result.get(0).endsWith("test-data/ext/lib/js-web/library_dummy.js")); }