Upload source code to remote builder as archive

server/envs/macos.env

@@ -1,6 +1,6 @@
 SWIFT_5_5_VERSION=5.5
 IOS_VERSION_MIN=11.0
-MACOS_VERSION_MIN=10.13
+MACOS_VERSION_MIN=10.15
 XCODE_16_VERSION=16.2
 XCODE_16_CLANG_VERSION=16.0.0
 MACOS_15_VERSION=15.2

server/src/main/java/com/defold/extender/Extender.java

@@ -83,11 +83,6 @@ class Extender {
     static final String FOLDER_COMMON_SRC = "commonsrc";// common source shared between both types
 
     static final String APPMANIFEST_FILENAME = "app.manifest";
-    static final String FRAMEWORK_RE = "(.+)\\.framework";
-    static final String JAR_RE = "(.+\\.jar)";
-    static final String JS_RE = "(.+\\.js)";
-    static final String PROTO_RE = "(?i).*(\\.proto)";
-    static final String ENGINE_JAR_RE = "(?:.*)\\/share\\/java\\/[\\w\\-\\.]*\\.jar$";
 
     private static final String MANIFEST_IOS    = "Info.plist";
     private static final String MANIFEST_OSX    = "Info.plist";
@@ -409,9 +404,9 @@ private Map<String, Object> createContext(Map<String, Object> src) throws Extend
     private List<String> getFrameworks(File dir) {
         List<String> frameworks = new ArrayList<>();
         final String[] platformParts = buildState.fullPlatform.split("-");
-        frameworks.addAll(ExtenderUtil.collectDirsByName(new File(dir, "lib" + File.separator + buildState.fullPlatform), FRAMEWORK_RE)); // e.g. armv64-ios
+        frameworks.addAll(ExtenderUtil.collectDirsByName(new File(dir, "lib" + File.separator + buildState.fullPlatform), ExtenderConst.FRAMEWORK_RE)); // e.g. armv64-ios
         if (platformParts.length == 2) {
-            frameworks.addAll(ExtenderUtil.collectDirsByName(new File(dir, "lib" + File.separator + platformParts[1]), FRAMEWORK_RE)); // e.g. "ios"
+            frameworks.addAll(ExtenderUtil.collectDirsByName(new File(dir, "lib" + File.separator + platformParts[1]), ExtenderConst.FRAMEWORK_RE)); // e.g. "ios"
         }
         return frameworks;
     }
@@ -439,10 +434,10 @@ private List<String> getFrameworkPaths(File dir) {
 
     private List<String> getExtensionLibJars(File extDir) {
         List<String> jars = new ArrayList<>();
-        jars.addAll(ExtenderUtil.collectFilesByPath(new File(extDir, "lib" + File.separator + buildState.fullPlatform), JAR_RE)); // e.g. armv7-android
+        jars.addAll(ExtenderUtil.collectFilesByPath(new File(extDir, "lib" + File.separator + buildState.fullPlatform), ExtenderConst.JAR_RE)); // e.g. armv7-android
         String[] platformParts = buildState.fullPlatform.split("-");
         if (platformParts.length == 2) {
-            jars.addAll(ExtenderUtil.collectFilesByPath(new File(extDir, "lib" + File.separator + platformParts[1]), JAR_RE)); // e.g. "android"
+            jars.addAll(ExtenderUtil.collectFilesByPath(new File(extDir, "lib" + File.separator + platformParts[1]), ExtenderConst.JAR_RE)); // e.g. "android"
         }
         return jars;
     }
@@ -1229,7 +1224,7 @@ private List<File> buildExtensionInternal(File manifest, Map<String, Object> man
             srcFiles.addAll(ExtenderUtil.listFiles(srcDirs, platformConfig.zigSourceRe));
 
         // Generate C++ files first (output into the source folder)
-        List<File> protoFiles = ExtenderUtil.listFiles(srcDirs, PROTO_RE);
+        List<File> protoFiles = ExtenderUtil.listFiles(srcDirs, ExtenderConst.PROTO_RE);
         List<File> generated = generateProtoCxxForEngine(extDir, manifestContext, protoFiles);
         if (!protoFiles.isEmpty() && generated.isEmpty()) {
             throw new ExtenderException(String.format("%s:1: error: Protofiles didn't generate any output engine cpp files!", ExtenderUtil.getRelativePath(buildState.uploadDir, protoFiles.get(0)) ));
@@ -1285,7 +1280,7 @@ private List<File> buildPipelineExtension(File manifest, Map<String, Object> man
         extBuildDir.mkdir();
         File[] srcDirs = { new File(extDir, FOLDER_COMMON_SRC), new File(extDir, FOLDER_PLUGIN_SRC) };
 
-        List<File> protoFiles = ExtenderUtil.listFiles(srcDirs, PROTO_RE);
+        List<File> protoFiles = ExtenderUtil.listFiles(srcDirs, ExtenderConst.PROTO_RE);
 
         List<File> outputFiles = new ArrayList<>();
 
@@ -1425,7 +1420,7 @@ private void getProjectPaths(Map<String, Object> mainContext, Map<String, Object
 
             extShLibs.addAll(ExtenderUtil.collectFilesByName(libDir, platformConfig.shlibRe));
             extLibs.addAll(ExtenderUtil.collectFilesByName(libDir, platformConfig.stlibRe));
-            extJsLibs.addAll(ExtenderUtil.collectFilesByPath(libDir, JS_RE));
+            extJsLibs.addAll(ExtenderUtil.collectFilesByPath(libDir, ExtenderConst.JS_RE));
 
             extFrameworks.addAll(getFrameworks(extDir));
 
@@ -1440,7 +1435,7 @@ private void getProjectPaths(Map<String, Object> mainContext, Map<String, Object
 
                 extShLibs.addAll(ExtenderUtil.collectFilesByName(libCommonDir, platformConfig.shlibRe));
                 extLibs.addAll(ExtenderUtil.collectFilesByName(libCommonDir, platformConfig.stlibRe));
-                extJsLibs.addAll(ExtenderUtil.collectFilesByPath(libCommonDir, JS_RE));
+                extJsLibs.addAll(ExtenderUtil.collectFilesByPath(libCommonDir, ExtenderConst.JS_RE));
                 extFrameworkPaths.addAll(getFrameworkPaths(extDir));
             }
         }
@@ -2140,10 +2135,10 @@ public static Map<String, Object> getManifestContext(String platform, ManifestCo
     private File buildMainDexList(List<String> jars) throws ExtenderException {
 
         // Find the engine libraries (**/share/java/*.jar)
-        List<String> mainListJars = ExtenderUtil.filterStrings(jars, ENGINE_JAR_RE);
+        List<String> mainListJars = ExtenderUtil.filterStrings(jars, ExtenderConst.ENGINE_JAR_RE);
 
         if (mainListJars.isEmpty()) {
-            throw new ExtenderException("Regex failed to find any engine jars: " + ENGINE_JAR_RE);
+            throw new ExtenderException("Regex failed to find any engine jars: " + ExtenderConst.ENGINE_JAR_RE);
         }
 
         List<String> mainClassNames = new ArrayList<String>();

server/src/main/java/com/defold/extender/ExtenderConst.java

@@ -0,0 +1,11 @@
+package com.defold.extender;
+
+public class ExtenderConst {
+    public static final String SOURCE_CODE_ARCHIVE_MAGIC_NAME = "__source_code__.zip";
+    public static final String FRAMEWORK_RE = "(.+)\\.framework";
+    public static final String JAR_RE = "(.+\\.jar)";
+    public static final String JS_RE = "(.+\\.js)";
+    public static final String PROTO_RE = "(?i).*(\\.proto)";
+    public static final String ENGINE_JAR_RE = "(?:.*)\\/share\\/java\\/[\\w\\-\\.]*\\.jar$";
+
+}

server/src/main/java/com/defold/extender/ExtenderController.java

@@ -56,8 +56,6 @@ public class ExtenderController {
     // Used to verify the uploaded filenames
     private static final Pattern FILENAME_RE = Pattern.compile("^([\\w ](?:[\\w+\\-\\/ @]|(?:\\.[\\w+\\-\\/ ]*))+)$");
 
-    private static final String SOURCE_CODE_ARCHIVE_MAGIC_NAME = "__source_code__.zip";
-
     private final DefoldSdkService defoldSdkService;
     private final DataCacheService dataCacheService;
     private final MeterRegistry meterRegistry;
@@ -388,7 +386,7 @@ static void receiveUpload(MultipartHttpServletRequest request, File uploadDirect
         }
 
         // check if source code archive presented
-        File sourceCodeArchive = new File(uploadDirectory, SOURCE_CODE_ARCHIVE_MAGIC_NAME);
+        File sourceCodeArchive = new File(uploadDirectory, ExtenderConst.SOURCE_CODE_ARCHIVE_MAGIC_NAME);
         if (sourceCodeArchive.exists()) {
             LOGGER.debug("Source code archive found. Unarchiving...");
             ZipUtils.unzip(new FileInputStream(sourceCodeArchive), uploadDirectory.toPath());

server/src/main/java/com/defold/extender/remote/RemoteEngineBuilder.java

@@ -1,7 +1,10 @@
 package com.defold.extender.remote;
 
 import com.defold.extender.BuilderConstants;
+import com.defold.extender.ExtenderConst;
+import com.defold.extender.ExtenderException;
 import com.defold.extender.metrics.MetricsWriter;
+import com.defold.extender.services.DataCacheService;
 import com.defold.extender.services.GCPInstanceService;
 import com.defold.extender.tracing.ExtenderTracerInterceptor;
 
@@ -19,10 +22,8 @@
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
-import org.apache.http.entity.ContentType;
 import org.apache.http.entity.mime.MultipartEntityBuilder;
-import org.apache.http.entity.mime.content.FileBody;
-import org.apache.http.entity.mime.content.AbstractContentBody;
+import org.apache.http.entity.mime.content.ByteArrayBody;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.util.EntityUtils;
 import org.slf4j.Logger;
@@ -34,13 +35,17 @@
 
 import java.io.OutputStream;
 import java.io.PrintWriter;
+import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.file.Files;
+import java.nio.file.Path;
 import java.nio.file.StandardCopyOption;
 import java.util.Optional;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
 import java.nio.charset.Charset;
 
 @Service
@@ -91,7 +96,7 @@ public void buildAsync(final RemoteInstanceConfig remoteInstanceConfig,
 
         try {
             httpEntity = buildHttpEntity(projectDirectory);
-        } catch(IllegalStateException|IOException e) {
+        } catch(IllegalStateException | IOException | ExtenderException e) {
             throw new RemoteBuildException("Failed to add files to multipart request", e);
         }
 
@@ -176,19 +181,34 @@ public void buildAsync(final RemoteInstanceConfig remoteInstanceConfig,
         }
     }
 
-    HttpEntity buildHttpEntity(final File projectDirectory) throws IOException {
-        final MultipartEntityBuilder builder = MultipartEntityBuilder.create()
-                .setContentType(ContentType.MULTIPART_FORM_DATA);
-
-        Files.walk(projectDirectory.toPath())
-                .filter(Files::isRegularFile)
-                .forEach(path -> {
-                    String relativePath = path.toFile().getAbsolutePath().substring(projectDirectory.getAbsolutePath().length() + 1);
-                    AbstractContentBody body = new FileBody(path.toFile(), ContentType.DEFAULT_BINARY, relativePath);
-                    builder.addPart(relativePath, body);
-                });
-
-        return builder.build();
+    HttpEntity buildHttpEntity(final File projectDirectory) throws IOException, ExtenderException {
+        MultipartEntityBuilder entityBuilder = MultipartEntityBuilder.create();
+        entityBuilder.setStrictMode();
+
+        ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
+        try (ZipOutputStream zipStream = new ZipOutputStream(byteStream)) {
+            Path projectDirectoryPath = projectDirectory.toPath();
+            Files.walk(projectDirectoryPath)
+            .filter(Files::isRegularFile)
+            .filter(path -> { return !path.getFileName().toString().equals(ExtenderConst.SOURCE_CODE_ARCHIVE_MAGIC_NAME); })
+            .filter(path -> { return !path.getFileName().toString().equals(DataCacheService.FILE_CACHE_INFO_FILE); })
+            .forEach(res -> {
+                Path path = projectDirectoryPath.relativize(res);
+                LOGGER.warn(path.toString());
+                try {
+                    ZipEntry entry = new ZipEntry(path.toString());
+                    zipStream.putNextEntry(entry);
+                    zipStream.write(Files.readAllBytes(res));
+                    zipStream.closeEntry();
+                } catch (IOException e) {
+                    LOGGER.warn("Exception during add entry to source code archive", e);
+                }
+            });
+        } catch (IOException exc) {
+            throw new ExtenderException(exc, "Failed to create source code archive");
+        }
+        entityBuilder.addPart(ExtenderConst.SOURCE_CODE_ARCHIVE_MAGIC_NAME, new ByteArrayBody(byteStream.toByteArray(), ExtenderConst.SOURCE_CODE_ARCHIVE_MAGIC_NAME));
+        return entityBuilder.build();
     }
 
     private void touchInstance(String instanceId) {

server/src/main/java/com/defold/extender/services/DataCacheService.java

@@ -30,7 +30,7 @@ public class DataCacheService {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(DataCacheService.class);
 
-    static final String FILE_CACHE_INFO_FILE = "ne-cache-info.json";
+    public static final String FILE_CACHE_INFO_FILE = "ne-cache-info.json";
     static final String FILE_CACHE_INFO_HASH_TYPE = "sha256";
     static final int    FILE_CACHE_INFO_VERSION = 1;
 

server/src/main/java/com/defold/extender/services/cocoapods/ResolvedPods.java

@@ -16,14 +16,14 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.defold.extender.ExtenderConst;
 import com.defold.extender.ExtenderException;
 import com.defold.extender.ExtenderUtil;
 import com.defold.extender.services.cocoapods.PlistBuddyWrapper.CreateBundlePlistArgs;
 import com.defold.extender.utils.FrameworkUtil;
 
 public class ResolvedPods {
     private static final Logger LOGGER = LoggerFactory.getLogger(ResolvedPods.class);
-    static final String FRAMEWORK_RE = "(.+)\\.framework";
     private List<PodBuildSpec> pods = new ArrayList<>();
     private File podsDir;
     private File frameworksDir;
@@ -127,7 +127,7 @@ List<String> collectAllPodFrameworks() throws IOException {
         }
 
         // collect unpacked xcframeworks
-        Pattern pattern = Pattern.compile(FRAMEWORK_RE);
+        Pattern pattern = Pattern.compile(ExtenderConst.FRAMEWORK_RE);
         Files.walk(frameworksDir.toPath())
             .filter(Files::isDirectory)
             .forEach(path -> {
@@ -143,7 +143,7 @@ List<String> collectAllPodFrameworks() throws IOException {
     List<File> collectAllPodsDynamicFrameworks() throws IOException {
         Set<File> dynamicFrameworks = new HashSet<>();
         // collect unpacked xcframeworks
-        Pattern pattern = Pattern.compile(FRAMEWORK_RE);
+        Pattern pattern = Pattern.compile(ExtenderConst.FRAMEWORK_RE);
         Files.walk(frameworksDir.toPath())
             .filter(Files::isDirectory)
             .forEach(path -> {

server/src/test/java/com/defold/extender/ExtenderTest.java

@@ -222,7 +222,7 @@ public void testCollectLibraries() {
             checkArray(expected, result);
         }
         {
-            List<String> result = ExtenderUtil.collectDirsByName(new File("test-data/ext/lib/x86_64-osx"), Extender.FRAMEWORK_RE);
+            List<String> result = ExtenderUtil.collectDirsByName(new File("test-data/ext/lib/x86_64-osx"), ExtenderConst.FRAMEWORK_RE);
             String[] expected = {"blib"};
             checkArray(expected, result);
         }
@@ -238,7 +238,7 @@ public void testCollectJars() {
         String[] endings = {"test-data/ext/lib/android/Dummy.jar", "test-data/ext/lib/android/JarDep.jar",
                             "test-data/ext/lib/android/VeryLarge1.jar", "test-data/ext/lib/android/VeryLarge2.jar",
                             "test-data/ext/lib/android/meta-inf.jar"};
-        List<String> paths = ExtenderUtil.collectFilesByPath(new File("test-data/ext/lib/android"), Extender.JAR_RE);
+        List<String> paths = ExtenderUtil.collectFilesByPath(new File("test-data/ext/lib/android"), ExtenderConst.JAR_RE);
         assertEquals(endings.length, paths.size());
 
 
@@ -256,7 +256,7 @@ public void testCollectJars() {
 
     @Test
     public void testCollectJsFiles() {
-        List<String> result = ExtenderUtil.collectFilesByPath(new File("test-data/ext/lib/js-web"), Extender.JS_RE);
+        List<String> result = ExtenderUtil.collectFilesByPath(new File("test-data/ext/lib/js-web"), ExtenderConst.JS_RE);
         assertEquals(1, result.size());
         assertTrue(result.get(0).endsWith("test-data/ext/lib/js-web/library_dummy.js"));
     }