Pod build frameworks (#765)

* Preparation before pod build refactoring

* Test compilation

* Hack headermap generation

* Make intermediate PodSpec which includes all subspecs that should be built. Fix case when builad pod as framework but pod consists of several specs

* Added headermap generation. Reworked temp file creating (source list case)

* Added VFS support. Restructure project a bit

* Generate Info.plist for framework from template

* Cleanup docs and env setup scripts

* Cleanup

* Split PodSpec into static and runtime part

* Fix tests. Added new tests

* Code for backward compatibility

Author: ekharkunov

README.md

@@ -11,16 +11,16 @@ Extender is a build server that builds native extensions of the Defold engine. T
 The stand-alone server is currently used on a machine runing macOS. The server is used to build darwin targets (macOS+iOS) using the Apple tools (XCode+Apple Clang). It is also possible to use this setup when developing on macOS.
 
 ### Prerequisites
+Ensure that on host is installed:
+* brew
+* curl
+
 Before running Extender you need to have prepackaged toolchains and sdks. Full instruction how it can be done you can find [here](https://github.com/defold/defold/tree/dev/scripts/package).
 
 Ensure that you have the following tools packaged:
-* iPhoneOS17.5.sdk
 * iPhoneOS18.2.sdk
-* iPhoneSimulator17.5.sdk
 * iPhoneSimulator18.2.sdk
-* MacOSX14.5.sdk
 * MacOSX15.2.sdk
-* XcodeDefault15.4.xctoolchain.darwin
 * XcodeDefault16.2.xctoolchain.darwin
 
 NOTE: Complete list of needed packages see [link](./server/scripts/standalone/setup-standalone-env.sh)

server/envs/macos.env

@@ -1,10 +1,6 @@
 SWIFT_5_5_VERSION=5.5
 IOS_VERSION_MIN=11.0
 MACOS_VERSION_MIN=10.13
-XCODE_15_VERSION=15.4
-XCODE_15_CLANG_VERSION=15.0.0
-MACOS_14_VERSION=14.5
-IOS_17_VERSION=17.5
 XCODE_16_VERSION=16.2
 XCODE_16_CLANG_VERSION=16.0.0
 MACOS_15_VERSION=15.2

server/scripts/standalone/setup-standalone-env.sh

@@ -186,13 +186,9 @@ if [[ $(uname) == "Darwin" ]]; then
 
     # Keep Apple's naming convention to avoid bugs
     PACKAGES=(
-        iPhoneOS${IOS_17_VERSION}.sdk
         iPhoneOS${IOS_18_VERSION}.sdk
-        iPhoneSimulator${IOS_17_VERSION}.sdk
         iPhoneSimulator${IOS_18_VERSION}.sdk
-        MacOSX${MACOS_14_VERSION}.sdk
         MacOSX${MACOS_15_VERSION}.sdk
-        XcodeDefault${XCODE_15_VERSION}.xctoolchain.darwin
         XcodeDefault${XCODE_16_VERSION}.xctoolchain.darwin
     )
     function download_packages() {
@@ -217,3 +213,6 @@ download_zig ${ZIG_URL} ${ZIG_PACKAGE_NAME} ${ZIG_PATH_0_11}
 
 echo "[setup] Installing dotnet"
 install_dotnet
+
+echo "[setup] Install hmap utility"
+brew install milend/taps/hmap

server/src/main/java/com/defold/extender/AsyncBuilder.java

@@ -85,7 +85,7 @@ public void asyncBuildEngine(MetricsWriter metricsWriter, String platform, Strin
             File jobDirectory, File uploadDirectory, File buildDirectory) throws IOException {
         String jobName = jobDirectory.getName();
         Thread.currentThread().setName(String.format("async-build-%s", jobName));
-        File resultDir = new File(jobResultLocation.getAbsolutePath(), jobName);
+        File resultDir = new File(jobResultLocation, jobName);
         resultDir.mkdir();
         Extender extender = null;
         Boolean isSuccefull = true;
@@ -112,7 +112,7 @@ public void asyncBuildEngine(MetricsWriter metricsWriter, String platform, Strin
                 }
 
                 // Resolve CocoaPods dependencies
-                if (platform.contains("ios") || platform.contains("osx")) {
+                if (ExtenderUtil.isAppleTarget(platform)) {
                     extender.resolve(cocoaPodsService);
                     metricsWriter.measureCocoaPodsInstallation();
                 }

server/src/main/java/com/defold/extender/Extender.java

@@ -0,0 +1,111 @@
+package com.defold.extender;
+
+import java.io.File;
+
+public class ExtenderBuildState {
+    static final String APPMANIFEST_BASE_VARIANT_KEYWORD = "baseVariant";
+    static final String APPMANIFEST_WITH_SYMBOLS_KEYWORD = "withSymbols";
+    static final String APPMANIFEST_BUILD_ARTIFACTS_KEYWORD = "buildArtifacts";
+    static final String APPMANIFEST_JETIFIER_KEYWORD = "jetifier";
+    static final String APPMANIFEST_DEBUG_SOURCE_PATH = "debugSourcePath";
+
+    File jobDir;
+    File uploadDir;
+    File buildDir;
+    File sdk;
+    String buildConfiguration;  // debug/release/headless
+    String fullPlatform;
+    String arch;
+    String hostPlatform;
+    private final String buildArtifacts;
+    private final String debugSourcePath;
+
+    private final Boolean withSymbols;
+    private final Boolean useJetifier;
+
+    ExtenderBuildState(Extender.Builder builder, AppManifestConfiguration appManifest) throws ExtenderException {
+        jobDir = builder.jobDirectory;
+        buildDir = builder.buildDirectory;
+        uploadDir = builder.uploadDirectory;
+        fullPlatform = builder.platform;
+        sdk = builder.sdk;
+        arch = fullPlatform.split("-")[0];
+
+        String baseVariant = ExtenderUtil.getAppManifestContextString(appManifest, APPMANIFEST_BASE_VARIANT_KEYWORD, null);
+
+        this.useJetifier = ExtenderUtil.getAppManifestBoolean(appManifest, builder.platform, APPMANIFEST_JETIFIER_KEYWORD, true);
+        this.withSymbols = ExtenderUtil.getAppManifestContextBoolean(appManifest, APPMANIFEST_WITH_SYMBOLS_KEYWORD, true);
+        this.buildArtifacts = ExtenderUtil.getAppManifestContextString(appManifest, APPMANIFEST_BUILD_ARTIFACTS_KEYWORD, "");
+        this.debugSourcePath = ExtenderUtil.getAppManifestContextString(appManifest, APPMANIFEST_DEBUG_SOURCE_PATH, null);
+        // assign configuration names started with upper letter because it used for cocoapods
+        if (baseVariant != null && (baseVariant.equals("release") || baseVariant.equals("headless"))) {
+            this.buildConfiguration = "Release";
+        } else {
+            this.buildConfiguration = "Debug";
+        }
+
+        String os = System.getProperty("os.name");
+        String arch = System.getProperty("os.arch");
+
+        // These host names are using the Defold SDK names
+        if (os.contains("Mac")) {
+            if (arch.contains("aarch64")) {
+                this.hostPlatform = "arm64-macos";
+            } else {
+                this.hostPlatform = "x86_64-macos";
+            }
+        } else if (os.contains("Windows")) {
+            this.hostPlatform = "x86_64-win32";
+        } else {
+            if (arch.contains("aarch64")) {
+                this.hostPlatform = "arm64-linux";
+            } else {
+                this.hostPlatform = "x86_64-linux";
+            }
+        }
+    }
+
+    public File getJobDir() {
+        return jobDir;
+    }
+
+    public File getBuildDir() {
+        return buildDir;
+    }
+
+    public File getUploadDir() {
+        return uploadDir;
+    }
+
+    public String getBuildPlatform() {
+        return fullPlatform;
+    }
+
+    public String getBuildArch() {
+        return arch;
+    }
+
+    public String getBuildConfiguration() {
+        return buildConfiguration;
+    }
+
+    public String getHostPlatform() {
+        return hostPlatform;
+    }
+
+    public String getBuildArtifacts() {
+        return buildArtifacts;
+    }
+
+    public String getDebugSourcePath() {
+        return debugSourcePath;
+    }
+
+    public Boolean isNeedSymbols() {
+        return withSymbols;
+    }
+
+    public Boolean isUsedJetifier() {
+        return useJetifier;
+    }
+}

server/src/main/java/com/defold/extender/ExtenderBuildState.java

@@ -32,6 +32,7 @@
 import org.apache.commons.io.filefilter.DirectoryFileFilter;
 import org.apache.commons.io.filefilter.RegexFileFilter;
 import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.commons.lang3.RandomStringUtils;
 import org.apache.commons.text.StringEscapeUtils;
 import org.json.simple.JSONObject;
 import org.springframework.core.io.Resource;
@@ -60,7 +61,7 @@ static Pattern createPattern(String expression) {
         return Pattern.compile(convertStringToLiteral(expression));
     }
 
-    static private List<String> filterItems(List<String> input, List<String> expressions, boolean keep) {
+    static private List<String> filterItems(Collection<String> input, Collection<String> expressions, boolean keep) {
         List<String> items = new ArrayList<>();
 
         List<Pattern> patterns = new ArrayList<>();
@@ -101,16 +102,16 @@ static private List<String> filterItems(List<String> input, List<String> express
 
 
     // Excludes items from input list that matches an item in the expressions list
-    static List<String> excludeItems(List<String> input, List<String> expressions) {
+    static List<String> excludeItems(Collection<String> input, Collection<String> expressions) {
         return filterItems(input, expressions, false);
     }
 
     // Keeps the matching items from input list that matches an item in the expressions list
-    static private List<String> matchItems(List<String> input, List<String> expressions) {
+    static private List<String> matchItems(Collection<String> input, Collection<String> expressions) {
         return filterItems(input, expressions, true);
     }
 
-    static List<String> pruneItems(List<String> input, List<String> includePatterns, List<String> excludePatterns) {
+    static List<String> pruneItems(Collection<String> input, Collection<String> includePatterns, Collection<String> excludePatterns) {
         List<String> includeItems = matchItems(input, includePatterns);
         List<String> items = excludeItems(input, excludePatterns);
         for( String item : includeItems) {
@@ -918,8 +919,8 @@ public static String calculateSHA256(InputStream input) throws NoSuchAlgorithmEx
         return hexString.toString();
     }
 
-    public static File writeSourceFilesListToTmpFile(Set<String> fileList) throws IOException {
-        File resultFile = Files.createTempFile(null, "sourcelist").toFile();
+    public static File writeSourceFilesListToTmpFile(File targetDir, Set<String> fileList) throws IOException {
+        File resultFile = new File(targetDir, String.format("%s.sourcelist", RandomStringUtils.insecure().nextAlphanumeric(30)));
         Set<String> escapedList = new HashSet<>();
         fileList.forEach((elem) -> {
             escapedList.add(StringEscapeUtils.escapeXSI(elem));

server/src/main/java/com/defold/extender/ExtenderUtil.java

@@ -7,6 +7,7 @@
 
 import org.springframework.stereotype.Service;
 
+import com.defold.extender.ExtenderBuildState;
 import com.defold.extender.ExtenderException;
 
 import io.micrometer.core.instrument.Gauge;
@@ -23,9 +24,9 @@ public GradleService(GradleServiceInterface service,
         Gauge.builder("extender.job.gradle.cacheSize", this, GradleService::getCacheSize).baseUnit(BaseUnits.BYTES).register(registry);
     }
 
-    public List<File> resolveDependencies(Map<String, Object> env, File cwd, File buildDirectory, Boolean useJetifier, List<File> outputFiles)
+    public List<File> resolveDependencies(ExtenderBuildState buildState, Map<String, Object> env, List<File> outputFiles)
         throws IOException, ExtenderException {
-        return gradleService.resolveDependencies(env, cwd, buildDirectory, useJetifier, outputFiles);
+        return gradleService.resolveDependencies(buildState, env, outputFiles);
     }
 
     public long getCacheSize() {

server/src/main/java/com/defold/extender/services/GradleService.java

@@ -5,11 +5,12 @@
 import java.util.List;
 import java.util.Map;
 
+import com.defold.extender.ExtenderBuildState;
 import com.defold.extender.ExtenderException;
 
 public interface GradleServiceInterface {
     // Resolve dependencies, download them, extract to
-    public List<File> resolveDependencies(Map<String, Object> env, File cwd, File buildDirectory, Boolean useJetifier, List<File> outputFiles) throws IOException, ExtenderException;
+    public List<File> resolveDependencies(ExtenderBuildState buildState, Map<String, Object> env, List<File> outputFiles) throws IOException, ExtenderException;
 
     public long getCacheSize() throws IOException;
 }

server/src/main/java/com/defold/extender/services/GradleServiceInterface.java

@@ -8,13 +8,14 @@
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.stereotype.Service;
 
+import com.defold.extender.ExtenderBuildState;
 import com.defold.extender.ExtenderException;
 
 @Service
 @ConditionalOnProperty(name = "extender.gradle.enabled", havingValue = "false", matchIfMissing = true)
 public class MockGradleService implements GradleServiceInterface {
     @Override
-    public List<File> resolveDependencies(Map<String, Object> env, File cwd, File buildDirectory, Boolean useJetifier, List<File> outputFiles)
+    public List<File> resolveDependencies(ExtenderBuildState buildState, Map<String, Object> env, List<File> outputFiles)
             throws IOException, ExtenderException {
         return List.of();
     }