diff --git a/README.md b/README.md index 307cd0c5..8b363e2c 100644 --- a/README.md +++ b/README.md @@ -11,16 +11,16 @@ Extender is a build server that builds native extensions of the Defold engine. T The stand-alone server is currently used on a machine runing macOS. The server is used to build darwin targets (macOS+iOS) using the Apple tools (XCode+Apple Clang). It is also possible to use this setup when developing on macOS. ### Prerequisites +Ensure that on host is installed: +* brew +* curl + Before running Extender you need to have prepackaged toolchains and sdks. Full instruction how it can be done you can find [here](https://github.com/defold/defold/tree/dev/scripts/package). Ensure that you have the following tools packaged: -* iPhoneOS17.5.sdk * iPhoneOS18.2.sdk -* iPhoneSimulator17.5.sdk * iPhoneSimulator18.2.sdk -* MacOSX14.5.sdk * MacOSX15.2.sdk -* XcodeDefault15.4.xctoolchain.darwin * XcodeDefault16.2.xctoolchain.darwin NOTE: Complete list of needed packages see [link](./server/scripts/standalone/setup-standalone-env.sh) diff --git a/server/envs/macos.env b/server/envs/macos.env index 786543ed..91029cf5 100644 --- a/server/envs/macos.env +++ b/server/envs/macos.env @@ -1,10 +1,6 @@ SWIFT_5_5_VERSION=5.5 IOS_VERSION_MIN=11.0 MACOS_VERSION_MIN=10.13 -XCODE_15_VERSION=15.4 -XCODE_15_CLANG_VERSION=15.0.0 -MACOS_14_VERSION=14.5 -IOS_17_VERSION=17.5 XCODE_16_VERSION=16.2 XCODE_16_CLANG_VERSION=16.0.0 MACOS_15_VERSION=15.2 diff --git a/server/scripts/standalone/setup-standalone-env.sh b/server/scripts/standalone/setup-standalone-env.sh index a4821cf1..311460e7 100755 --- a/server/scripts/standalone/setup-standalone-env.sh +++ b/server/scripts/standalone/setup-standalone-env.sh @@ -186,13 +186,9 @@ if [[ $(uname) == "Darwin" ]]; then # Keep Apple's naming convention to avoid bugs PACKAGES=( - iPhoneOS${IOS_17_VERSION}.sdk iPhoneOS${IOS_18_VERSION}.sdk - iPhoneSimulator${IOS_17_VERSION}.sdk iPhoneSimulator${IOS_18_VERSION}.sdk - MacOSX${MACOS_14_VERSION}.sdk MacOSX${MACOS_15_VERSION}.sdk - XcodeDefault${XCODE_15_VERSION}.xctoolchain.darwin XcodeDefault${XCODE_16_VERSION}.xctoolchain.darwin ) function download_packages() { @@ -217,3 +213,6 @@ download_zig ${ZIG_URL} ${ZIG_PACKAGE_NAME} ${ZIG_PATH_0_11} echo "[setup] Installing dotnet" install_dotnet + +echo "[setup] Install hmap utility" +brew install milend/taps/hmap \ No newline at end of file diff --git a/server/src/main/java/com/defold/extender/AsyncBuilder.java b/server/src/main/java/com/defold/extender/AsyncBuilder.java index 635015ba..47d611a9 100644 --- a/server/src/main/java/com/defold/extender/AsyncBuilder.java +++ b/server/src/main/java/com/defold/extender/AsyncBuilder.java @@ -85,7 +85,7 @@ public void asyncBuildEngine(MetricsWriter metricsWriter, String platform, Strin File jobDirectory, File uploadDirectory, File buildDirectory) throws IOException { String jobName = jobDirectory.getName(); Thread.currentThread().setName(String.format("async-build-%s", jobName)); - File resultDir = new File(jobResultLocation.getAbsolutePath(), jobName); + File resultDir = new File(jobResultLocation, jobName); resultDir.mkdir(); Extender extender = null; Boolean isSuccefull = true; @@ -112,7 +112,7 @@ public void asyncBuildEngine(MetricsWriter metricsWriter, String platform, Strin } // Resolve CocoaPods dependencies - if (platform.contains("ios") || platform.contains("osx")) { + if (ExtenderUtil.isAppleTarget(platform)) { extender.resolve(cocoaPodsService); metricsWriter.measureCocoaPodsInstallation(); } diff --git a/server/src/main/java/com/defold/extender/Extender.java b/server/src/main/java/com/defold/extender/Extender.java index 5f06a660..dc50128f 100644 --- a/server/src/main/java/com/defold/extender/Extender.java +++ b/server/src/main/java/com/defold/extender/Extender.java @@ -1,6 +1,7 @@ package com.defold.extender; import com.google.common.collect.ImmutableMap; + import org.apache.commons.io.FileUtils; import org.apache.commons.io.FilenameUtils; import org.apache.commons.io.filefilter.DirectoryFileFilter; @@ -41,22 +42,20 @@ import com.defold.extender.services.GradleService; import com.defold.extender.services.cocoapods.CocoaPodsService; -import com.defold.extender.services.cocoapods.PodSpec; +import com.defold.extender.services.cocoapods.PodBuildSpec; +import com.defold.extender.services.cocoapods.PodUtils; import com.defold.extender.services.cocoapods.ResolvedPods; +import com.defold.extender.utils.PodBuildUtil; import com.defold.extender.builders.CSharpBuilder; import com.defold.extender.log.Markers; import com.defold.extender.metrics.MetricsWriter; import com.defold.extender.process.ProcessExecutor; +import com.defold.extender.process.ProcessUtils; class Extender { private static final Logger LOGGER = LoggerFactory.getLogger(Extender.class); + private final ExtenderBuildState buildState; private final String appManifestPath; - private final String platform; - private final String hostPlatform; - private final File sdk; - private final File uploadDirectory; - private final File jobDirectory; - private final File buildDirectory; private final Configuration config; // build.yml from the defoldsdk private final PlatformConfig platformConfig; // "common", platform, arch-platform from build.yml private final PlatformConfig platformVariantConfig; // "common", platform, arch-platform from build_variant.yml @@ -65,11 +64,6 @@ class Extender { private final ProcessExecutor processExecutor = new ProcessExecutor(); private MetricsWriter metricsWriter; // context flags - private final Boolean withSymbols; - private final Boolean useJetifier; - private final String buildArtifacts; - private final String debugSourcePath; - private final String configuration; // debug/release/headless private Boolean needsCSLibraries = false; private Map manifestFiles; @@ -89,11 +83,6 @@ class Extender { static final String FOLDER_COMMON_SRC = "commonsrc";// common source shared between both types static final String APPMANIFEST_FILENAME = "app.manifest"; - static final String APPMANIFEST_BASE_VARIANT_KEYWORD = "baseVariant"; - static final String APPMANIFEST_WITH_SYMBOLS_KEYWORD = "withSymbols"; - static final String APPMANIFEST_BUILD_ARTIFACTS_KEYWORD = "buildArtifacts"; - static final String APPMANIFEST_JETIFIER_KEYWORD = "jetifier"; - static final String APPMANIFEST_DEBUG_SOURCE_PATH = "debugSourcePath"; static final String FRAMEWORK_RE = "(.+)\\.framework"; static final String JAR_RE = "(.+\\.jar)"; static final String JS_RE = "(.+\\.js)"; @@ -137,13 +126,13 @@ private class ProGuardContext { private static final boolean DM_DEBUG_DISABLE_PROGUARD = System.getenv("DM_DEBUG_DISABLE_PROGUARD") != null; static public class Builder { - private String platform; - private File sdk; - private File jobDirectory; - private File buildDirectory; - private File uploadDirectory; - private Map env = new HashMap(); - private MetricsWriter metricsWriter; + String platform; + File sdk; + File jobDirectory; + File buildDirectory; + File uploadDirectory; + Map env = new HashMap(); + MetricsWriter metricsWriter; public Builder() { } @@ -188,18 +177,15 @@ public Extender build() throws IOException, ExtenderException { }; private Extender(Builder builder) throws IOException, ExtenderException { - this.jobDirectory = builder.jobDirectory; - this.uploadDirectory = builder.uploadDirectory; - this.buildDirectory = builder.buildDirectory; this.metricsWriter = builder.metricsWriter; this.gradlePackages = new ArrayList<>(); this.outputFiles = new ArrayList<>(); // Read config from SDK - this.config = Extender.loadYaml(this.jobDirectory, new File(builder.sdk.getPath() + "/extender/build.yml"), Configuration.class); + this.config = Extender.loadYaml(builder.jobDirectory, new File(builder.sdk.getPath() + "/extender/build.yml"), Configuration.class); // Read the app manifest from the upload folder - Collection allFiles = FileUtils.listFiles(uploadDirectory, null, true); + Collection allFiles = FileUtils.listFiles(builder.uploadDirectory, null, true); List appManifests = allFiles.stream().filter(f -> f.getName().equals(APPMANIFEST_FILENAME)).collect(Collectors.toList()); if (appManifests.size() > 1 ) { throw new ExtenderException("Only one app.manifest allowed!"); @@ -213,8 +199,8 @@ private Extender(Builder builder) throws IOException, ExtenderException { this.appManifestPath = ""; appManifest = new AppManifestConfiguration(); } else { - this.appManifestPath = ExtenderUtil.getRelativePath(this.uploadDirectory, appManifests.get(0)); - appManifest = Extender.loadYaml(this.jobDirectory, appManifests.get(0), AppManifestConfiguration.class); + this.appManifestPath = ExtenderUtil.getRelativePath(builder.uploadDirectory, appManifests.get(0)); + appManifest = Extender.loadYaml(builder.jobDirectory, appManifests.get(0), AppManifestConfiguration.class); // An completely empty manifest will yield a null pointer in result from Extender.loadYaml appManifest = (appManifest != null) ? appManifest : new AppManifestConfiguration(); @@ -225,7 +211,7 @@ private Extender(Builder builder) throws IOException, ExtenderException { appManifest.platforms = new HashMap(); } - baseVariant = ExtenderUtil.getAppManifestContextString(appManifest, APPMANIFEST_BASE_VARIANT_KEYWORD, null); + baseVariant = ExtenderUtil.getAppManifestContextString(appManifest, ExtenderBuildState.APPMANIFEST_BASE_VARIANT_KEYWORD, null); if (baseVariant != null) { File baseVariantFile = new File(builder.sdk.getPath() + "/extender/variants/" + baseVariant + ".appmanifest"); @@ -235,53 +221,20 @@ private Extender(Builder builder) throws IOException, ExtenderException { } LOGGER.info("Using base variant: " + baseVariant); - baseVariantManifest = Extender.loadYaml(this.jobDirectory, baseVariantFile, AppManifestConfiguration.class); - } - } - - this.useJetifier = ExtenderUtil.getAppManifestBoolean(appManifest, builder.platform, APPMANIFEST_JETIFIER_KEYWORD, true); - this.withSymbols = ExtenderUtil.getAppManifestContextBoolean(appManifest, APPMANIFEST_WITH_SYMBOLS_KEYWORD, true); - this.buildArtifacts = ExtenderUtil.getAppManifestContextString(appManifest, APPMANIFEST_BUILD_ARTIFACTS_KEYWORD, ""); - this.debugSourcePath = ExtenderUtil.getAppManifestContextString(appManifest, APPMANIFEST_DEBUG_SOURCE_PATH, null); - // assign configuration names started with upper letter because it used for cocoapods - if (baseVariant != null && (baseVariant.equals("release") || baseVariant.equals("headless"))) { - this.configuration = "Release"; - } else { - this.configuration = "Debug"; - } - - this.platform = builder.platform; - this.sdk = builder.sdk; - - String os = System.getProperty("os.name"); - String arch = System.getProperty("os.arch"); - - // These host names are using the Defold SDK names - if (os.contains("Mac")) { - if (arch.contains("aarch64")) { - this.hostPlatform = "arm64-macos"; - } else { - this.hostPlatform = "x86_64-macos"; - } - } else if (os.contains("Windows")) { - this.hostPlatform = "x86_64-win32"; - } else { - if (arch.contains("aarch64")) { - this.hostPlatform = "arm64-linux"; - } else { - this.hostPlatform = "x86_64-linux"; + baseVariantManifest = Extender.loadYaml(builder.jobDirectory, baseVariantFile, AppManifestConfiguration.class); } } + this.buildState = new ExtenderBuildState(builder, appManifest); - if (config.platforms.get(platform) == null) { - throw new ExtenderException(String.format("Unsupported platform %s by this sdk", platform)); + if (config.platforms.get(buildState.fullPlatform) == null) { + throw new ExtenderException(String.format("Unsupported platform %s by this sdk", buildState.fullPlatform)); } // Merge the platform configs from build.yml into a single instance: common -> platform -> arch-platform this.platformConfig = new PlatformConfig(); this.platformConfig.context = new HashMap<>(config.context); // the context from build.yml - for (String platformAlt : ExtenderUtil.getPlatformAlternatives(platform)) { + for (String platformAlt : ExtenderUtil.getPlatformAlternatives(buildState.fullPlatform)) { PlatformConfig platformConfigAlt = config.platforms.get(platformAlt); if (platformConfigAlt == null) continue; @@ -292,7 +245,7 @@ private Extender(Builder builder) throws IOException, ExtenderException { // Merge the variant info into a single config this.platformVariantConfig = new PlatformConfig(); if (baseVariantManifest != null) { - for (String platformAlt : ExtenderUtil.getPlatformAlternatives(platform)) { + for (String platformAlt : ExtenderUtil.getPlatformAlternatives(buildState.fullPlatform)) { AppManifestPlatformConfig configAlt = baseVariantManifest.platforms.get(platformAlt); if (configAlt == null) continue; @@ -303,7 +256,7 @@ private Extender(Builder builder) throws IOException, ExtenderException { // Merge the app manifest info into a single config this.platformAppConfig = new PlatformConfig(); - for (String platformAlt : ExtenderUtil.getPlatformAlternatives(platform)) { + for (String platformAlt : ExtenderUtil.getPlatformAlternatives(buildState.fullPlatform)) { AppManifestPlatformConfig configAlt = appManifest.platforms.get(platformAlt); if (configAlt == null) { continue; @@ -313,17 +266,17 @@ private Extender(Builder builder) throws IOException, ExtenderException { ExtenderUtil.mergeObjects(this.platformAppConfig, platformConfigAlt); } - LOGGER.info("Using context for platform: " + platform); + LOGGER.info("Using context for platform: {}", buildState.fullPlatform); - processExecutor.setCwd(jobDirectory); + processExecutor.setCwd(buildState.jobDir); { HashMap envContext = new HashMap<>(); - envContext.put("build_folder", buildDirectory); - envContext.put("dynamo_home", sdk); + envContext.put("build_folder", buildState.buildDir); + envContext.put("dynamo_home", buildState.sdk); envContext.put("env.LD_LIBRARY_PATH", "."); // Easier when running a standalone local without such a variable - processExecutor.putEnv("DYNAMO_HOME", sdk.getAbsolutePath()); + processExecutor.putEnv("DYNAMO_HOME", buildState.sdk.getAbsolutePath()); String java_home = System.getenv("JAVA_HOME"); if (java_home != null) { @@ -360,7 +313,7 @@ private Extender(Builder builder) throws IOException, ExtenderException { ExtensionManifestValidator manifestValidator = new ExtensionManifestValidator(new WhitelistConfig(), this.platformConfig.allowedFlags, allowedSymbols); // Make sure the user hasn't input anything invalid in the manifest - manifestValidator.validate(this.appManifestPath, this.uploadDirectory, this.platformAppConfig.context); + manifestValidator.validate(this.appManifestPath, buildState.uploadDir, this.platformAppConfig.context); // Collect extension directories (used by both buildEngine and buildClassesDex) this.manifests = allFiles.stream().filter(f -> f.getName().equals("ext.manifest")).collect(Collectors.toList()); @@ -380,17 +333,13 @@ private String getNameUUID() { } private File createBuildFile(String name) { - return new File(buildDirectory, name); + return new File(buildState.buildDir, name); } private File uniqueTmpFile(String prefix, String suffix) { return createBuildFile(prefix + getNameUUID() + suffix); } - private File uniqueTmpFile(String pattern) { - return createBuildFile(String.format(pattern, getNameUUID())); - } - private String executeCommand(String template, Map context) throws ExtenderException { String command = templateExecutor.execute(template, context); try { @@ -449,9 +398,9 @@ private Map createContext(Map src) throws Extend Map context = new HashMap<>(src); // TODO: Create deep copy // Should not be allowed to be overridden by manifests - context.put("dynamo_home", ExtenderUtil.getRelativePath(jobDirectory, sdk)); - context.put("platform", this.platform); - context.put("host_platform", this.hostPlatform); + context.put("dynamo_home", ExtenderUtil.getRelativePath(buildState.jobDir, buildState.sdk)); + context.put("platform", buildState.fullPlatform); + context.put("host_platform", buildState.hostPlatform); resolveVariables(context); return context; @@ -459,8 +408,8 @@ private Map createContext(Map src) throws Extend private List getFrameworks(File dir) { List frameworks = new ArrayList<>(); - final String[] platformParts = this.platform.split("-"); - frameworks.addAll(ExtenderUtil.collectDirsByName(new File(dir, "lib" + File.separator + this.platform), FRAMEWORK_RE)); // e.g. armv64-ios + final String[] platformParts = buildState.fullPlatform.split("-"); + frameworks.addAll(ExtenderUtil.collectDirsByName(new File(dir, "lib" + File.separator + buildState.fullPlatform), FRAMEWORK_RE)); // e.g. armv64-ios if (platformParts.length == 2) { frameworks.addAll(ExtenderUtil.collectDirsByName(new File(dir, "lib" + File.separator + platformParts[1]), FRAMEWORK_RE)); // e.g. "ios" } @@ -470,8 +419,8 @@ private List getFrameworks(File dir) { // Get a list of subfolders matching the current platform private List getPlatformPaths(File dir) { List paths = new ArrayList<>(); - final String[] platformParts = this.platform.split("-"); - File libDir = new File(dir, this.platform); + final String[] platformParts = buildState.fullPlatform.split("-"); + File libDir = new File(dir, buildState.fullPlatform); if (libDir.exists()) { paths.add(libDir.getAbsolutePath()); } @@ -490,8 +439,8 @@ private List getFrameworkPaths(File dir) { private List getExtensionLibJars(File extDir) { List jars = new ArrayList<>(); - jars.addAll(ExtenderUtil.collectFilesByPath(new File(extDir, "lib" + File.separator + this.platform), JAR_RE)); // e.g. armv7-android - String[] platformParts = this.platform.split("-"); + jars.addAll(ExtenderUtil.collectFilesByPath(new File(extDir, "lib" + File.separator + buildState.fullPlatform), JAR_RE)); // e.g. armv7-android + String[] platformParts = buildState.fullPlatform.split("-"); if (platformParts.length == 2) { jars.addAll(ExtenderUtil.collectFilesByPath(new File(extDir, "lib" + File.separator + platformParts[1]), JAR_RE)); // e.g. "android" } @@ -533,7 +482,7 @@ else if(f.getName().endsWith(".aar")) { private List pruneNonExisting(List paths) { List existing = new ArrayList<>(); for (String path : paths) { - File f = new File(jobDirectory + File.separator + path); + File f = new File(buildState.jobDir + File.separator + path); if (f.exists()) existing.add(path); } @@ -543,22 +492,22 @@ private List pruneNonExisting(List paths) { private List getExtLocalIncludeDirs(File dir) { List includes = new ArrayList<>(); - includes.add( ExtenderUtil.getRelativePath(jobDirectory, new File(dir, "include" + File.separator + this.platform) ) ); + includes.add( ExtenderUtil.getRelativePath(buildState.jobDir, new File(dir, "include" + File.separator + buildState.fullPlatform) ) ); - String[] platformParts = this.platform.split("-"); + String[] platformParts = buildState.fullPlatform.split("-"); if (platformParts.length == 2) { - includes.add( ExtenderUtil.getRelativePath(jobDirectory, new File(dir, "include" + File.separator + platformParts[1]))); + includes.add( ExtenderUtil.getRelativePath(buildState.jobDir, new File(dir, "include" + File.separator + platformParts[1]))); } - includes.add( ExtenderUtil.getRelativePath(jobDirectory, new File(dir, "include") ) ); + includes.add( ExtenderUtil.getRelativePath(buildState.jobDir, new File(dir, "include") ) ); return includes; } private List getIncludeDirs(File extDir) { List includes = getExtLocalIncludeDirs(extDir); - includes.add( ExtenderUtil.getRelativePath(jobDirectory, new File(buildDirectory, extDir.getName())) ); // where we generate source from protobuf files - includes.add( ExtenderUtil.getRelativePath(jobDirectory, uploadDirectory) ); //TODO: Do we ever put stuff here? Isn't it more useful to include the build folder? + includes.add( ExtenderUtil.getRelativePath(buildState.jobDir, new File(buildState.buildDir, extDir.getName())) ); // where we generate source from protobuf files + includes.add( ExtenderUtil.getRelativePath(buildState.jobDir, buildState.uploadDir) ); //TODO: Do we ever put stuff here? Isn't it more useful to include the build folder? // Add the other extensions include folders for (File otherExtDir : this.getExtensionFolders()) { @@ -571,11 +520,13 @@ private List getIncludeDirs(File extDir) { return pruneNonExisting(includes); } - private List getPodIncludeDir(PodSpec pod) { + private List getPodIncludeDir(PodBuildSpec pod) { List result = new ArrayList<>(); - for (File path : pod.includePaths) { - result.add(path.toString()); + for (File f : pod.includePaths) { + result.add(f.toString()); } + result.add(pod.headerMapFile.toString()); + if (pod.swiftModuleHeader != null) { result.add(pod.swiftModuleHeader.toPath().getParent().toString()); } @@ -585,18 +536,17 @@ private List getPodIncludeDir(PodSpec pod) { // swiftc: https://gist.github.com/enomoto/7f11d57e4add7e702f9f84f34d3a0f8c // swift-frontend: https://gist.github.com/palaniraja/b4de1e64e874b68bda9e5236829cd8a6 - private void emitSwiftHeader(PodSpec pod, Map manifestContext, List commands) throws IOException, InterruptedException, ExtenderException { + private void emitSwiftHeader(PodBuildSpec pod, Map manifestContext, List commands) throws IOException, InterruptedException, ExtenderException { List includes = getIncludeDirs(pod.dir); includes.addAll(getPodIncludeDir(pod)); List frameworks = new ArrayList<>(); - frameworks.addAll(getFrameworks(pod.dir)); frameworks.addAll(resolvedPods.getFrameworks()); List frameworkPaths = new ArrayList<>(); frameworkPaths.addAll(getFrameworkPaths(pod.dir)); frameworkPaths.addAll(resolvedPods.getFrameworksSearchPaths()); - File sourceListFile = ExtenderUtil.writeSourceFilesListToTmpFile(pod.swiftSourceFilePaths); + File sourceListFile = ExtenderUtil.writeSourceFilesListToTmpFile(pod.intermediatedDir, pod.swiftSourceFilePaths); Map context = createContext(manifestContext); context.put("ext", ImmutableMap.of("includes", includes, "frameworks", frameworks, "frameworkPaths", frameworkPaths)); @@ -610,18 +560,17 @@ private void emitSwiftHeader(PodSpec pod, Map manifestContext, L commands.add(command); } - private void emitSwiftModule(PodSpec pod, Map manifestContext, List commands) throws IOException, InterruptedException, ExtenderException { + private void emitSwiftModule(PodBuildSpec pod, Map manifestContext, List commands) throws IOException, InterruptedException, ExtenderException { List includes = getIncludeDirs(pod.dir); includes.addAll(getPodIncludeDir(pod)); List frameworks = new ArrayList<>(); - frameworks.addAll(getFrameworks(pod.dir)); frameworks.addAll(resolvedPods.getFrameworks()); List frameworkPaths = new ArrayList<>(); frameworkPaths.addAll(getFrameworkPaths(pod.dir)); frameworkPaths.addAll(resolvedPods.getFrameworksSearchPaths()); - File sourceListFile = ExtenderUtil.writeSourceFilesListToTmpFile(pod.swiftSourceFilePaths); + File sourceListFile = ExtenderUtil.writeSourceFilesListToTmpFile(pod.intermediatedDir, pod.swiftSourceFilePaths); Map context = createContext(manifestContext); context.put("ext", ImmutableMap.of("includes", includes, "frameworks", frameworks, "frameworkPaths", frameworkPaths)); @@ -635,12 +584,12 @@ private void emitSwiftModule(PodSpec pod, Map manifestContext, L commands.add(command); } - void generateSwiftCompatabilityHeaders(PodSpec spec, File podDir) throws IOException { + void generateSwiftCompatabilityHeaders(PodBuildSpec spec, File podDir) throws IOException { LOGGER.debug("Generate Swift compatability header and modulemap for {}", spec.name); File podBuildDir = spec.swiftModuleHeader.toPath().getParent().getParent().toFile(); // copy objc modulemap which Cocopoapods generated - String podName = spec.getPodName(); + String podName = spec.name; Path sourceModuleMap = Path.of(podDir.toString(), "Target Support Files", podName, String.format("%s.modulemap", podName)); Path targetModuleMap = Path.of(podBuildDir.toString(), String.format("%s.modulemap", spec.moduleName)); // it's not a bug. Cocoapods installs module map in Target support Files with spec.name but in compiler options it waits spec.moduleName Files.copy(sourceModuleMap, targetModuleMap, StandardCopyOption.REPLACE_EXISTING); @@ -654,8 +603,8 @@ void generateSwiftCompatabilityHeaders(PodSpec spec, File podDir) throws IOExcep Files.writeString(targetModuleMap, spec.swiftModuleDefinition, StandardOpenOption.APPEND); } - private File addCompileFileSwift(PodSpec pod, int index, File src, Map manifestContext, List commands) throws IOException, InterruptedException, ExtenderException { - File o = new File(buildDirectory, String.format("%s_%d.o", src.getName(), index)); + private File addCompileFileSwift(PodBuildSpec pod, int index, File src, Map manifestContext, List commands) throws IOException, InterruptedException, ExtenderException { + File o = new File(buildState.buildDir, String.format("%s_%d.o", src.getName(), index)); // remove the primary source file from the set of all source files String swiftPrimarySourceFile = src.getAbsolutePath(); @@ -666,17 +615,15 @@ private File addCompileFileSwift(PodSpec pod, int index, File src, Map frameworks = new ArrayList<>(); - frameworks.addAll(getFrameworks(pod.dir)); frameworks.addAll(resolvedPods.getFrameworks()); List frameworkPaths = new ArrayList<>(); - frameworkPaths.addAll(getFrameworkPaths(pod.dir)); frameworkPaths.addAll(resolvedPods.getFrameworksSearchPaths()); - File sourceFileList = ExtenderUtil.writeSourceFilesListToTmpFile(swiftSourceFilePaths); - File primarySourceFile = ExtenderUtil.writeSourceFilesListToTmpFile(Set.of(swiftPrimarySourceFile)); + File sourceFileList = ExtenderUtil.writeSourceFilesListToTmpFile(pod.intermediatedDir, swiftSourceFilePaths); + File primarySourceFile = ExtenderUtil.writeSourceFilesListToTmpFile(pod.intermediatedDir, Set.of(swiftPrimarySourceFile)); Map context = createContext(manifestContext); context.put("ext", ImmutableMap.of("includes", includes, "frameworks", frameworks, "frameworkPaths", frameworkPaths)); - context.put("tgt", ExtenderUtil.getRelativePath(jobDirectory, o)); + context.put("tgt", ExtenderUtil.getRelativePath(buildState.jobDir, o)); context.put("moduleName", pod.moduleName); context.put("swiftPrimarySourceFile", String.format("@%s", primarySourceFile.getAbsolutePath())); context.put("swiftSourceFiles", String.format("@%s", sourceFileList.getAbsolutePath())); @@ -692,7 +639,7 @@ private File addCompileFileCpp_Internal(int index, File extDir, File src, Map includes = getIncludeDirs(extDir); includes.addAll(additionalIncludes); - File o = new File(buildDirectory, String.format("%s_%d.o", src.getName(), index)); + File o = new File(buildState.buildDir, String.format("%s_%d.o", src.getName(), index)); List frameworks = new ArrayList<>(); frameworks.addAll(getFrameworks(extDir)); @@ -704,8 +651,8 @@ private File addCompileFileCpp_Internal(int index, File extDir, File src, Map context = createContext(manifestContext); - context.put("src", ExtenderUtil.getRelativePath(jobDirectory, src)); - context.put("tgt", ExtenderUtil.getRelativePath(jobDirectory, o)); + context.put("src", ExtenderUtil.getRelativePath(buildState.jobDir, src)); + context.put("tgt", ExtenderUtil.getRelativePath(buildState.jobDir, o)); context.put("ext", ImmutableMap.of("includes", includes, "frameworks", frameworks, "frameworkPaths", frameworkPaths)); String command = templateExecutor.execute(cmd, context); @@ -744,7 +691,7 @@ private File linkCppShared(File extBuildDir, List objs, Map manifestContext) thro Map context = createContext(manifestContext); context.put("extension_name", "ENGINE_MAIN"); context.put("extension_name_upper", "ENGINE_MAIN"); - context.put("src", ExtenderUtil.getRelativePath(jobDirectory, maincpp)); - context.put("tgt", ExtenderUtil.getRelativePath(jobDirectory, o)); + context.put("src", ExtenderUtil.getRelativePath(buildState.jobDir, maincpp)); + context.put("tgt", ExtenderUtil.getRelativePath(buildState.jobDir, o)); executeCommand(platformConfig.compileCmd, context); return o; } @@ -773,8 +720,6 @@ private List getExtensionFolders() { private List generateProtoCxxForEngine(File extDir, Map manifestContext, List protoFiles) throws IOException, InterruptedException, ExtenderException { List generated = new ArrayList<>(); - Map tmpcontext = createContext(manifestContext); - if (platformConfig.protoEngineCxxCmd == null) { LOGGER.info("Not supporting .proto files"); return generated; @@ -782,7 +727,7 @@ private List generateProtoCxxForEngine(File extDir, Map ma LOGGER.info("Converting .proto to engine .cpp files for extension {}", extDir.getName()); - File extBuildDir = createDir(buildDirectory, extDir.getName()); + File extBuildDir = createDir(buildState.buildDir, extDir.getName()); List includes = getIncludeDirs(extDir); @@ -793,7 +738,7 @@ private List generateProtoCxxForEngine(File extDir, Map ma Map context = createContext(manifestContext); - context.put("src", ExtenderUtil.getRelativePath(jobDirectory, protoFile)); + context.put("src", ExtenderUtil.getRelativePath(buildState.jobDir, protoFile)); context.put("ext", ImmutableMap.of("includes", includes)); context.put("out_dir", extBuildDir); executeCommand(platformConfig.protoEngineCxxCmd, context); @@ -808,9 +753,7 @@ private List generateProtoCxxForEngine(File extDir, Map ma private List generateProtoSrcForPlugin(File extDir, Map manifestContext, List protoFiles, String language) throws IOException, InterruptedException, ExtenderException { List generated = new ArrayList<>(); - Map tmpcontext = createContext(manifestContext); - - File extBuildDir = createDir(buildDirectory, extDir.getName()); + File extBuildDir = createDir(buildState.buildDir, extDir.getName()); List includes = getIncludeDirs(extDir); LOGGER.info("Converting .proto to plugin {} files for extension {}", language, extDir.getName()); @@ -831,13 +774,13 @@ else if (language.equals("python")) File tgtFile = new File(extBuildDir, name); Map context = createContext(manifestContext); - context.put("src", ExtenderUtil.getRelativePath(jobDirectory, protoFile)); + context.put("src", ExtenderUtil.getRelativePath(buildState.jobDir, protoFile)); context.put("ext", ImmutableMap.of("includes", includes)); context.put("out_dir", extBuildDir); context.put("language", language); // Adding the source folderpath, so the output relative path gets stripped and the output becomes "extBuildDir/proto_file_name.pb.cc" - context.put("proto_path", ExtenderUtil.getRelativePath(jobDirectory, protoFile.getParentFile())); + context.put("proto_path", ExtenderUtil.getRelativePath(buildState.jobDir, protoFile.getParentFile())); executeCommand(platformConfig.protoPipelineCmd, context); @@ -855,14 +798,7 @@ private List compileExtensionSourceFiles(File extDir, Map objs = new ArrayList<>(); List commands = new ArrayList<>(); - List additionalIncludes = new ArrayList<>(); - if (resolvedPods != null) { - for (PodSpec spec : resolvedPods.getPodSpecs()) { - for (File path : spec.includePaths) { - additionalIncludes.add(path.toString()); - } - } - } + List additionalIncludes = resolvedPods != null ? resolvedPods.getAdditionalIncludePaths() : List.of(); for (File src : srcFiles) { final int i = getAndIncreaseNameCount(); @@ -878,14 +814,14 @@ else if (platformConfig.zigSourceRe != null && ExtenderUtil.matchesFile(src, pla if (o == null) { throw new ExtenderException(String.format("Source file '%s' didn't match a source builder.", src)); } - objs.add(ExtenderUtil.getRelativePath(jobDirectory, o)); + objs.add(ExtenderUtil.getRelativePath(buildState.jobDir, o)); } ProcessExecutor.executeCommands(processExecutor, commands); // in parallel return objs; } // compile the source files of a pod and return a list of object files - private List compilePodSourceFiles(PodSpec pod, Map manifestContext, Set podSourceLookup) throws IOException, InterruptedException, ExtenderException { + private List compilePodSourceFiles(PodBuildSpec pod, Map manifestContext) throws IOException, InterruptedException, ExtenderException { // clean up flags from context Map trimmedContext = ExtenderUtil.mergeContexts(manifestContext, new HashMap<>()); trimmedContext.put("flags", new ArrayList()); @@ -926,32 +862,62 @@ private List compilePodSourceFiles(PodSpec pod, Map mani List updatedSwiftFlags = new ArrayList(swiftFlags); updatedSwiftFlags.add("-enable-experimental-feature"); updatedSwiftFlags.add("AccessLevelOnImport"); +// ******************************** Added for backward comapatibility ***************************************** +// ******************************** Remove after 6 month ****************************************************** + for (File path : pod.frameworkSearchPaths) { + updatedSwiftFlags.add(String.format("-F%s", path.toString())); + } + if (resolvedPods.useFrameworks()) { + updatedSwiftFlags.add(String.format("-F%s", pod.buildDir.toString())); + } +// ************************************************************************************************************ mergedContextWithPodsForSwift.put("swiftFlags", updatedSwiftFlags); // generate headers from swift files List emitSwiftHeaderCommands = new ArrayList<>(); LOGGER.info("emit swift header"); emitSwiftHeader(pod, mergedContextWithPodsForSwift, emitSwiftHeaderCommands); +// ******************************** Added for backward comapatibility ***************************************** +// ******************************** Remove after 6 month ****************************************************** + for (int i = 0; i < emitSwiftHeaderCommands.size(); ++i) { + String cmd = emitSwiftHeaderCommands.get(i); + cmd = cmd.replaceAll("-DSWIFT_PACKAGE", ""); + emitSwiftHeaderCommands.set(i, cmd); + } +// ************************************************************************************************************ ProcessExecutor.executeCommands(processExecutor, emitSwiftHeaderCommands); // in parallel // generate swift module from swift files List emitSwiftModuleCommands = new ArrayList<>(); LOGGER.info("emit swift module"); emitSwiftModule(pod, mergedContextWithPodsForSwift, emitSwiftModuleCommands); +// ******************************** Added for backward comapatibility ***************************************** +// ******************************** Remove after 6 month ****************************************************** + for (int i = 0; i < emitSwiftModuleCommands.size(); ++i) { + String cmd = emitSwiftModuleCommands.get(i); + cmd = cmd.replaceAll("-DSWIFT_PACKAGE", ""); + emitSwiftModuleCommands.set(i, cmd); + } +// ************************************************************************************************************ ProcessExecutor.executeCommands(processExecutor, emitSwiftModuleCommands); // in parallel // compile swift source files one by one List compileSwiftCommands = new ArrayList<>(); for (File src : pod.swiftSourceFiles) { - if (!podSourceLookup.contains(src)) { - final int i = getAndIncreaseNameCount(); - File o = addCompileFileSwift(pod, i, src, mergedContextWithPodsForSwift, compileSwiftCommands); - String objPath = ExtenderUtil.getRelativePath(jobDirectory, o); - objs.add(objPath); - podSourceLookup.add(src); - } + final int i = getAndIncreaseNameCount(); + File o = addCompileFileSwift(pod, i, src, mergedContextWithPodsForSwift, compileSwiftCommands); + String objPath = ExtenderUtil.getRelativePath(buildState.jobDir, o); + objs.add(objPath); } LOGGER.info("compiling {} swift files", compileSwiftCommands.size()); +// ******************************** Added for backward comapatibility ***************************************** +// ******************************** Remove after 6 month ****************************************************** + for (int i = 0; i < compileSwiftCommands.size(); ++i) { + String cmd = compileSwiftCommands.get(i); + cmd = cmd.replaceAll("-DSWIFT_PACKAGE", ""); + compileSwiftCommands.set(i, cmd); + } +// ************************************************************************************************************ ProcessExecutor.executeCommands(processExecutor, compileSwiftCommands); // in parallel generateSwiftCompatabilityHeaders(pod, resolvedPods.getCurrentPodsDirectory()); @@ -959,27 +925,24 @@ private List compilePodSourceFiles(PodSpec pod, Map mani List commands = new ArrayList<>(); for (File src : pod.sourceFiles) { - if (!podSourceLookup.contains(src)) { - String extension = FilenameUtils.getExtension(src.getAbsolutePath()); - final int i = getAndIncreaseNameCount(); - File o = null; - // use the correct context depending on the source file language - if (extension.equals("c")) { - o = addCompileFileCppStatic(i, pod.dir, src, mergedContextWithPodsForC, commands, getPodIncludeDir(pod)); - } - else if (extension.equals("m")) { - o = addCompileFileCppStatic(i, pod.dir, src, mergedContextWithPodsForObjC, commands, getPodIncludeDir(pod)); - } - else if (extension.equals("mm")) { - o = addCompileFileCppStatic(i, pod.dir, src, mergedContextWithPodsForObjCpp, commands, getPodIncludeDir(pod)); - } - else { - o = addCompileFileCppStatic(i, pod.dir, src, mergedContextWithPodsForCpp, commands, getPodIncludeDir(pod)); - } - String objPath = ExtenderUtil.getRelativePath(jobDirectory, o); - objs.add(objPath); - podSourceLookup.add(src); + String extension = FilenameUtils.getExtension(src.getAbsolutePath()); + final int i = getAndIncreaseNameCount(); + File o = null; + // use the correct context depending on the source file language + if (extension.equals("c")) { + o = addCompileFileCppStatic(i, pod.dir, src, mergedContextWithPodsForC, commands, getPodIncludeDir(pod)); + } + else if (extension.equals("m")) { + o = addCompileFileCppStatic(i, pod.dir, src, mergedContextWithPodsForObjC, commands, getPodIncludeDir(pod)); + } + else if (extension.equals("mm")) { + o = addCompileFileCppStatic(i, pod.dir, src, mergedContextWithPodsForObjCpp, commands, getPodIncludeDir(pod)); } + else { + o = addCompileFileCppStatic(i, pod.dir, src, mergedContextWithPodsForCpp, commands, getPodIncludeDir(pod)); + } + String objPath = ExtenderUtil.getRelativePath(buildState.jobDir, o); + objs.add(objPath); } LOGGER.info("compiling {} source files", commands.size()); ProcessExecutor.executeCommands(processExecutor, commands); // in parallel @@ -987,6 +950,124 @@ else if (extension.equals("mm")) { return objs; } + void buildPodAsFramework(PodBuildSpec spec, String targetPlatform, File targetSupportFileDir) throws ExtenderException, IOException, InterruptedException { + // Collect resource bundles + List resourceBundles = ResolvedPods.createPodResourceBundles(spec, spec.buildDir, targetPlatform); + + if (PodUtils.hasSourceFiles(spec)) { + Map> enumeratedFiles = new HashMap<>(); + String podName = spec.name; + // Create output folder and output framework folder + File frameworkDir = new File(spec.buildDir, String.format("%s.framework", spec.moduleName)); + frameworkDir.mkdirs(); + File frameworkHeaders = new File(frameworkDir, "Headers"); + frameworkHeaders.mkdir(); + File frameworkModules = new File(frameworkDir, "Modules"); + frameworkModules.mkdir(); + + String frameworkHeadersDirPath = frameworkHeaders.toString(); + // Collect headers + for (File header : spec.publicHeaders) { + FileUtils.copyFileToDirectory(header, frameworkHeaders); + PodBuildUtil.putFileNameIntoVFS(enumeratedFiles, frameworkHeadersDirPath, header); + } + + // copy first time + File sourceModuleMap = Path.of(targetSupportFileDir.toString(), podName, String.format("%s.modulemap", podName)).toFile(); + FileUtils.copyFile(sourceModuleMap, new File(frameworkModules, "module.modulemap")); + PodBuildUtil.putFileNameIntoVFS(enumeratedFiles, frameworkModules.toString(), sourceModuleMap); + + File sourceUmbrellaHeader = Path.of(targetSupportFileDir.toString(), podName, String.format("%s-umbrella.h", podName)).toFile(); + FileUtils.copyFileToDirectory(sourceUmbrellaHeader, frameworkHeaders); + PodBuildUtil.putFileNameIntoVFS(enumeratedFiles, frameworkHeadersDirPath, sourceUmbrellaHeader); + + PodBuildUtil.generateVFSOverlay(spec, enumeratedFiles); + // Compile library + File library = buildPodLibrary(spec); + + FileUtils.copyFile(library, new File(frameworkDir, spec.moduleName)); + + // copy from generateSwiftCompatabilityHeader + if (spec.swiftModuleHeader != null && spec.swiftModuleHeader.exists()) { + FileUtils.copyFileToDirectory(spec.swiftModuleHeader, frameworkHeaders); + // copy the second time because during generating swift compatibility header modulemap was updated + File updatedModuleMap = new File(spec.buildDir, String.format("%s.modulemap", spec.moduleName)); + FileUtils.copyFile(updatedModuleMap, new File(frameworkModules, "module.modulemap"), StandardCopyOption.REPLACE_EXISTING); + } + + // Copy swift module + File swiftModule = new File(spec.buildDir, spec.moduleName + ".swiftmodule"); + if (swiftModule.exists()) { + File targetSwiftModuleDir = new File(frameworkModules, spec.moduleName + ".swiftmodule"); + targetSwiftModuleDir.mkdir(); + File targetSwiftModuleName = new File(targetSwiftModuleDir, String.format("%s.swiftmodule", PodUtils.swiftModuleNameFromPlatform(targetPlatform))); + FileUtils.copyFile(swiftModule, targetSwiftModuleName, StandardCopyOption.REPLACE_EXISTING); + } + // 4.2. Copy modulemap + for (File bundle : resourceBundles) { + FileUtils.copyDirectoryToDirectory(bundle, frameworkDir); + } + Set resources = new HashSet<>(); + ResolvedPods.addPodResources(spec, resources); + for (File res : resources) { + FileUtils.copyFileToDirectory(res, frameworkDir); + } + + // copy Info.plist + File sourceInfoPlist = Path.of(targetSupportFileDir.toString(), podName, String.format("%s-Info.plist", podName)).toFile(); + PodBuildUtil.generatedInfoPlistFromTemplate(sourceInfoPlist, Map.of( + "PODS_DEVELOPMENT_LANGUAGE", "en", + "EXECUTABLE_NAME", spec.moduleName, + "PRODUCT_BUNDLE_IDENTIFIER", String.format("org.cocoapods.%s", spec.moduleName), + "PRODUCT_NAME", spec.moduleName, + "CURRENT_PROJECT_VERSION", "1" + + ), new File(frameworkDir, "Info.plist")); + + // 5. Sign framework + ProcessUtils.execCommand(List.of( + "/usr/bin/codesign", + "--force", + "--sign", + "-", + "--timestamp=none", + "--generate-entitlement-der", + frameworkDir.getAbsolutePath() + ), null, null); + } + } + + /* + * @return resultLib File Result static/dynamic library or null is no source files + */ + File buildPodLibrary(PodBuildSpec spec) throws ExtenderException, IOException, InterruptedException { + LOGGER.info("buildPods - compiling pod source files for {}", spec.name); + // The source files of each pod will be compiled and built as a library. + // We use the same mechanism as when building the extension and create a + // manifest context for each pod + Map manifestContext = new HashMap<>(); + manifestContext = ExtenderUtil.mergeContexts(manifestContext, this.platformConfig.context); + manifestContext = ExtenderUtil.mergeContexts(manifestContext, this.platformVariantConfig.context); + manifestContext.put("extension_name", spec.name); + manifestContext.put("extension_name_upper", spec.name.toUpperCase()); + manifestContext.put("osMinVersion", resolvedPods.getPlatformMinVersion()); + manifestContext.put("env.IOS_VERSION_MIN", resolvedPods.getPlatformMinVersion()); + + // Compile pod source files + List objs = compilePodSourceFiles(spec, manifestContext); + if (!objs.isEmpty()) { + // Create c++ library + File lib = new File(spec.buildDir ,String.format(platformConfig.writeLibPattern, manifestContext.get("extension_name") + "_" + getNameUUID())); + Map context = createContext(manifestContext); + context.put("tgt", lib); + context.put("objs", objs); + LOGGER.info("creating library {} from {} objects", lib.getName(), objs.size()); + executeCommand(platformConfig.libCmd, context); + return lib; + } + return null; + } + // build the source files of each resolved pod file into a library private List buildPods() throws IOException, InterruptedException, ExtenderException { List outputFiles = new ArrayList<>(); @@ -995,36 +1076,19 @@ private List buildPods() throws IOException, InterruptedException, Extende return outputFiles; } - LOGGER.info("buildPods - compiling pod source files"); - Set podSourceLookup = new HashSet<>(); - for (PodSpec pod : resolvedPods.getPodSpecs()) { - LOGGER.info("buildPods - compiling pod source files for {}", pod.name); - // The source files of each pod will be compiled and built as a library. - // We use the same mechanism as when building the extension and create a - // manifest context for each pod - Map manifestContext = new HashMap<>(); - manifestContext = ExtenderUtil.mergeContexts(manifestContext, this.platformConfig.context); - manifestContext = ExtenderUtil.mergeContexts(manifestContext, this.platformVariantConfig.context); - manifestContext.put("extension_name", pod.name); - manifestContext.put("extension_name_upper", pod.name.toUpperCase()); - manifestContext.put("osMinVersion", resolvedPods.getPlatformMinVersion()); - manifestContext.put("env.IOS_VERSION_MIN", resolvedPods.getPlatformMinVersion()); - - // Compile pod source files - List objs = compilePodSourceFiles(pod, manifestContext, podSourceLookup); - if (!objs.isEmpty()) { - // Create c++ library - File lib = createBuildFile(String.format(platformConfig.writeLibPattern, manifestContext.get("extension_name") + "_" + getNameUUID())); - Map context = createContext(manifestContext); - context.put("tgt", lib); - context.put("objs", objs); - LOGGER.info("creating library {} from {} objects", lib.getName(), objs.size()); - executeCommand(platformConfig.libCmd, context); + boolean asFramework = resolvedPods.useFrameworks(); + LOGGER.info("buildPods - compiling pod source file as {}", asFramework ? "frameworks" : "libraries"); + for (PodBuildSpec pod : resolvedPods.getPodSpecs()) { + PodBuildUtil.generateHeaderMap(pod); + if (asFramework) { + buildPodAsFramework(pod, buildState.fullPlatform, resolvedPods.getTargetSupportFilesDir()); + } else { + buildPodLibrary(pod); } } LOGGER.info("buildPods - adding framework resource to build output"); - File resourcesBuildDir = new File(buildDirectory, "resources"); + File resourcesBuildDir = new File(buildState.buildDir, "resources"); resourcesBuildDir.mkdir(); List resources = resolvedPods.getAllPodResources(); @@ -1034,8 +1098,7 @@ private List buildPods() throws IOException, InterruptedException, Extende resourceDestFile.getParentFile().mkdirs(); Files.copy(resourceFile.toPath(), resourceDestFile.toPath()); outputFiles.add(resourceDestFile); - } - else { + } else { File resourceDestDir = new File(resourcesBuildDir, resourceFile.getName()); resourceDestDir.mkdirs(); FileUtils.copyDirectory(resourceFile, resourceDestDir); @@ -1044,10 +1107,10 @@ private List buildPods() throws IOException, InterruptedException, Extende } LOGGER.info("buildPods - creating and adding resource bundles"); - outputFiles.addAll(resolvedPods.createResourceBundles(resourcesBuildDir, platform)); + outputFiles.addAll(resolvedPods.createResourceBundles(resourcesBuildDir, buildState.fullPlatform)); LOGGER.info("buildPods - adding dynamic frameworks to build output"); - File frameworksBuildDir = new File(buildDirectory, "frameworks"); + File frameworksBuildDir = new File(buildState.buildDir, "frameworks"); frameworksBuildDir.mkdir(); List dynamicFrameworks = resolvedPods.getDynamicFrameworks(); @@ -1069,7 +1132,7 @@ public boolean accept(File pathname) { File podfileLock = resolvedPods.getPodfileLock(); if (podfileLock != null) { LOGGER.info("buildPods - adding Podfile.lock to build output"); - File destPodFileLock = new File(buildDirectory, "Podfile.lock"); + File destPodFileLock = new File(buildState.buildDir, "Podfile.lock"); FileUtils.copyFile(podfileLock, destPodFileLock); outputFiles.add(destPodFileLock); } @@ -1112,7 +1175,7 @@ private List buildExtensionInternal_CSharp(File manifest, Map buildExtensionInternal_CSharp(File manifest, Map buildExtensionInternal_CSharp(File manifest, Map "extension") List libs = (List)context.get("engineLibs"); - if (ExtenderUtil.isWindowsTarget(this.platform)) + if (ExtenderUtil.isWindowsTarget(buildState.fullPlatform)) { libs = new ArrayList<>(); for (String lib : (List)context.get("engineLibs")) { @@ -1144,11 +1207,11 @@ private List buildExtensionInternal_CSharp(File manifest, Map buildExtensionInternal(File manifest, Map man List protoFiles = ExtenderUtil.listFiles(srcDirs, PROTO_RE); List generated = generateProtoCxxForEngine(extDir, manifestContext, protoFiles); if (!protoFiles.isEmpty() && generated.isEmpty()) { - throw new ExtenderException(String.format("%s:1: error: Protofiles didn't generate any output engine cpp files!", ExtenderUtil.getRelativePath(this.uploadDirectory, protoFiles.get(0)) )); + throw new ExtenderException(String.format("%s:1: error: Protofiles didn't generate any output engine cpp files!", ExtenderUtil.getRelativePath(buildState.uploadDir, protoFiles.get(0)) )); } srcFiles.addAll(generated); @@ -1180,7 +1243,7 @@ private List buildExtensionInternal(File manifest, Map man } if (srcFiles.isEmpty() && srcCSFiles.isEmpty()) { - throw new ExtenderException(String.format("%s:1: error: Extension has no source!", ExtenderUtil.getRelativePath(this.uploadDirectory, manifest) )); + throw new ExtenderException(String.format("%s:1: error: Extension has no source!", ExtenderUtil.getRelativePath(buildState.uploadDir, manifest) )); } List outputFiles = new ArrayList<>(); @@ -1206,9 +1269,9 @@ private List buildExtension(File manifest, Map manifestCon private List buildLibrary(File manifest, Map manifestContext) throws IOException, InterruptedException, ExtenderException { LOGGER.info("buildLibrary"); List srcDirs = new ArrayList<>(); - srcDirs.add(uploadDirectory); + srcDirs.add(buildState.uploadDir); String libName = String.format(platformConfig.writeLibPattern, manifestContext.get("extension_name")); - return buildExtensionInternal(manifest, manifestContext, srcDirs, new File(buildDirectory, libName)); + return buildExtensionInternal(manifest, manifestContext, srcDirs, new File(buildState.buildDir, libName)); } private List buildPipelineExtension(File manifest, Map manifestContext) throws IOException, InterruptedException, ExtenderException { @@ -1218,7 +1281,7 @@ private List buildPipelineExtension(File manifest, Map man } File extDir = manifest.getParentFile(); - File extBuildDir = createDir(buildDirectory, extDir.getName()); + File extBuildDir = createDir(buildState.buildDir, extDir.getName()); extBuildDir.mkdir(); File[] srcDirs = { new File(extDir, FOLDER_COMMON_SRC), new File(extDir, FOLDER_PLUGIN_SRC) }; @@ -1248,7 +1311,7 @@ private List buildPipelineExtension(File manifest, Map man int i = getAndIncreaseNameCount(); for (File src : srcFiles) { File o = addCompileFileCppShared(i, extDir, src, manifestContext, commands); - objs.add(ExtenderUtil.getRelativePath(jobDirectory, o)); + objs.add(ExtenderUtil.getRelativePath(buildState.jobDir, o)); i++; } ProcessExecutor.executeCommands(processExecutor, commands); // in parallel @@ -1334,25 +1397,26 @@ private List patchLibs(List libs) { } private void getProjectPaths(Map mainContext, Map env) throws ExtenderException, IOException { - List extLibs = new ArrayList<>(); - List extShLibs = new ArrayList<>(); - List extLibPaths = new ArrayList<>(Arrays.asList(buildDirectory.toString())); - List extFrameworks = new ArrayList<>(); - List extFrameworkPaths = new ArrayList<>(Arrays.asList(buildDirectory.toString())); - List extJsLibs = new ArrayList<>(); + Collection extLibs = new HashSet<>(); + Collection extShLibs = new HashSet<>(); + Collection extLibPaths = new HashSet<>(Arrays.asList(buildState.buildDir.toString())); + Collection extFrameworks = new HashSet<>(); + Collection extFrameworkPaths = new HashSet<>(Arrays.asList(buildState.buildDir.toString())); + Collection extJsLibs = new HashSet<>(); - extShLibs.addAll(ExtenderUtil.collectFilesByName(buildDirectory, platformConfig.shlibRe)); - extLibs.addAll(ExtenderUtil.collectFilesByName(buildDirectory, platformConfig.stlibRe)); + extShLibs.addAll(ExtenderUtil.collectFilesByName(buildState.buildDir, platformConfig.shlibRe)); + extLibs.addAll(ExtenderUtil.collectFilesByName(buildState.buildDir, platformConfig.stlibRe)); if (resolvedPods != null) { extFrameworks.addAll(resolvedPods.getFrameworks()); + extFrameworks.addAll(resolvedPods.getBuiltFrameworks()); extFrameworkPaths.addAll(resolvedPods.getFrameworksSearchPaths()); extLibs.addAll(resolvedPods.getStaticLibraries()); extLibPaths.addAll(resolvedPods.getLibrarySearchPaths()); } for (File extDir : this.extDirs) { - File libDir = new File(extDir, "lib" + File.separator + this.platform); // e.g. arm64-ios + File libDir = new File(extDir, "lib" + File.separator + buildState.fullPlatform); // e.g. arm64-ios if (libDir.exists()) { extLibPaths.add(libDir.toString()); @@ -1365,7 +1429,7 @@ private void getProjectPaths(Map mainContext, Map mainContext, Map mainContext, Map linkEngine(List symbols, Map linkContext, File resourceFile) throws IOException, InterruptedException, ExtenderException { LOGGER.info("Linking engine"); - File maincpp = new File(buildDirectory , "main.cpp"); + File maincpp = new File(buildState.buildDir , "main.cpp"); List extSymbols = new ArrayList<>(); extSymbols.addAll(symbols); @@ -1424,12 +1490,12 @@ private List linkEngine(List symbols, Map linkCont File mainObject = compileMain(maincpp, linkContext); String writeExePattern = platformConfig.writeExePattern; - File exe = new File(buildDirectory, writeExePattern); + File exe = new File(buildState.buildDir, writeExePattern); List objects = new ArrayList<>(); - objects.add(ExtenderUtil.getRelativePath(jobDirectory, mainObject)); + objects.add(ExtenderUtil.getRelativePath(buildState.jobDir, mainObject)); if (resourceFile != null) { // For Win32 targets - objects.add(ExtenderUtil.getRelativePath(jobDirectory, resourceFile)); + objects.add(ExtenderUtil.getRelativePath(buildState.jobDir, resourceFile)); } Map env = new HashMap<>(); @@ -1437,7 +1503,7 @@ private List linkEngine(List symbols, Map linkCont Map context = createContext(linkContext); context.put("src", objects); - context.put("tgt", ExtenderUtil.getRelativePath(jobDirectory, exe)); + context.put("tgt", ExtenderUtil.getRelativePath(buildState.jobDir, exe)); context.put("ext", env); // WINE->clang transition pt1: in the transition period from link.exe -> lld, we want to make sure we can write "foo" as opposed to "foo.lib" @@ -1446,7 +1512,7 @@ private List linkEngine(List symbols, Map linkCont context.put("engineLibs", patchLibs((List) context.get("engineLibs"))); if (this.needsCSLibraries) { - CSharpBuilder.updateContext(platform, this.buildDirectory, context); + CSharpBuilder.updateContext(buildState.fullPlatform, buildState.buildDir, context); } List commands = platformConfig.linkCmds; // Used by e.g. the Switch platform @@ -1467,12 +1533,12 @@ private List linkEngine(List symbols, Map linkCont } // Extract symbols - if (this.withSymbols) { + if (buildState.isNeedSymbols()) { LOGGER.info("Extracting symbols"); String symbolCmd = platformConfig.symbolCmd; if (symbolCmd != null && !symbolCmd.isBlank()) { Map symbolContext = createContext(linkContext); - symbolContext.put("src", ExtenderUtil.getRelativePath(jobDirectory, exe)); + symbolContext.put("src", ExtenderUtil.getRelativePath(buildState.jobDir, exe)); executeCommand(symbolCmd, symbolContext); } @@ -1484,10 +1550,10 @@ private List linkEngine(List symbols, Map linkCont // collect dynamic libraries and copy to result folder List dynamicLibsPathes = new ArrayList<>(); for (File extDir : this.extDirs) { - File libDir = new File(extDir, "lib" + File.separator + this.platform); // e.g. arm64-ios + File libDir = new File(extDir, "lib" + File.separator + buildState.fullPlatform); // e.g. arm64-ios dynamicLibsPathes.addAll(ExtenderUtil.collectFilePathesByName(libDir, platformConfig.shlibRe)); - String[] platformParts = this.platform.split("-"); + String[] platformParts = buildState.fullPlatform.split("-"); if (platformParts.length == 2) { File libCommonDir = new File(extDir, "lib" + File.separator + platformParts[1]); // e.g. ios @@ -1496,7 +1562,7 @@ private List linkEngine(List symbols, Map linkCont } for (String filepath : dynamicLibsPathes) { - FileUtils.copyFileToDirectory(new File(filepath), buildDirectory); + FileUtils.copyFileToDirectory(new File(filepath), buildState.buildDir); } // Collect output/binaries @@ -1506,7 +1572,7 @@ private List linkEngine(List symbols, Map linkCont } // If we wish to grab the symbols, prepend the pattern (E.g. to "(.*dSYM)|(dmengine)") - if (this.withSymbols) { + if (buildState.isNeedSymbols()) { String symbolsPattern = platformConfig.symbolsPattern; if (symbolsPattern != null && !symbolsPattern.isBlank()) { zipContentPattern = symbolsPattern + "|" + zipContentPattern; @@ -1516,7 +1582,7 @@ private List linkEngine(List symbols, Map linkCont zipContentPattern = zipContentPattern + "|" + platformConfig.shlibRe; final Pattern p = Pattern.compile(zipContentPattern); - List outputFiles = Arrays.asList(buildDirectory.listFiles(new FileFilter(){ + List outputFiles = Arrays.asList(buildState.buildDir.listFiles(new FileFilter(){ @Override public boolean accept(File file) { return p.matcher(file.getName()).matches(); @@ -1548,7 +1614,7 @@ private List getAndroidJniFolders(String platform) { private List getAndroidResourceFolders(String platform) { // New feature from 1.2.165 - File packageDir = new File(uploadDirectory, "packages"); + File packageDir = new File(buildState.uploadDir, "packages"); if (!packageDir.exists()) { return new ArrayList<>(); } @@ -1600,7 +1666,7 @@ private static File createDir(File parent, String child) throws IOException { private File compileAndroidResources(List resourceDirectories, Map mergedAppContext) throws ExtenderException { LOGGER.info("Compiling Android resources"); - File outputDirectory = new File(buildDirectory, "compiledResources"); + File outputDirectory = new File(buildState.buildDir, "compiledResources"); outputDirectory.mkdirs(); try { Map context = createContext(mergedAppContext); @@ -1647,7 +1713,7 @@ private Map linkAndroidResources(File compiledResourcesDir, Map linkAndroidResources(File compiledResourcesDir, Map linkAndroidResources(File compiledResourcesDir, Map resourceDirectories, Map mergedAppContext) throws ExtenderException { - File rJavaDir = new File(uploadDirectory, "_app/rjava"); + File rJavaDir = new File(buildState.uploadDir, "_app/rjava"); if (rJavaDir.exists()) { LOGGER.info("Using pre-existing R.java files"); return rJavaDir; @@ -1699,7 +1765,7 @@ private File generateRJava(List resourceDirectories, Map LOGGER.info("Generating R.java files"); // From 1.2.165 - rJavaDir = new File(buildDirectory, "rjava"); + rJavaDir = new File(buildState.buildDir, "rjava"); rJavaDir.mkdir(); if (platformConfig.rjavaCmd == null) { @@ -1713,7 +1779,7 @@ private File generateRJava(List resourceDirectories, Map } // Use the merged manifest - File manifestFile = new File(buildDirectory, MANIFEST_ANDROID); + File manifestFile = new File(buildState.buildDir, MANIFEST_ANDROID); context.put("manifestFile", manifestFile.getAbsolutePath()); context.put("outputDirectory", rJavaDir.getAbsolutePath()); context.put("resourceDirectories", resourceDirectories); @@ -1802,7 +1868,7 @@ private Map.Entry buildJavaExtension(File manifest, Map buildProGuard(List allJars, Map allJarsMap = getProGuardMapping(allJars, extensionJarMap); @@ -2004,8 +2070,8 @@ private Map.Entry buildProGuard(List allJars, Map allPro = new ArrayList<>(); allPro.add(appProPath); - File targetFile = new File(buildDirectory, "dmengine.jar"); - File mappingFile = new File(buildDirectory, "mapping.txt"); + File targetFile = new File(buildState.buildDir, "dmengine.jar"); + File mappingFile = new File(buildState.buildDir, "mapping.txt"); List jarList = new ArrayList<>(); List jarLibrariesList = new ArrayList<>(); @@ -2096,7 +2162,7 @@ private File buildMainDexList(List jars) throws ExtenderException { // Only keep the .class files mainClassNames = ExtenderUtil.filterStrings(mainClassNames, "(?:.*)\\.class$"); - File mainList = new File(buildDirectory, "main_dex_list.txt"); + File mainList = new File(buildState.buildDir, "main_dex_list.txt"); try { mainList.createNewFile(); @@ -2112,13 +2178,13 @@ private File buildMainDexList(List jars) throws ExtenderException { } private File[] buildClassesDex(List jars, File mainDexList) throws ExtenderException { - LOGGER.info("Building classes.dex with extension source {}", uploadDirectory); + LOGGER.info("Building classes.dex with extension source {}", buildState.uploadDir); // The empty list is also present for backwards compatability with older build.yml List empty_list = new ArrayList<>(); Map context = createContext(mergedAppContext); - context.put("classes_dex_dir", buildDirectory.getAbsolutePath()); + context.put("classes_dex_dir", buildState.buildDir.getAbsolutePath()); context.put("jars", jars); context.put("engineJars", empty_list); context.put("mainDexList", mainDexList.getAbsolutePath()); @@ -2129,14 +2195,14 @@ private File[] buildClassesDex(List jars, File mainDexList) throws Exten platformConfig.dxCmd = platformConfig.dxCmd.replace("--main-dex-list", "--main-dex-rules"); executeCommand(platformConfig.dxCmd, context); - File[] classes = ExtenderUtil.listFilesMatching(buildDirectory, "^classes(|[0-9]+)\\.dex$"); + File[] classes = ExtenderUtil.listFilesMatching(buildState.buildDir, "^classes(|[0-9]+)\\.dex$"); return classes; } private File buildWin32Resources(Map mergedAppContext) throws ExtenderException { Map context = createContext(mergedAppContext); - File resourceFile = new File(buildDirectory, "dmengine.res"); - context.put("tgt", ExtenderUtil.getRelativePath(jobDirectory, resourceFile)); + File resourceFile = new File(buildState.buildDir, "dmengine.res"); + context.put("tgt", ExtenderUtil.getRelativePath(buildState.jobDir, resourceFile)); String command = templateExecutor.execute(platformConfig.windresCmd, context); if (command.equals("")) { @@ -2161,9 +2227,10 @@ private void loadManifests(ExtensionManifestValidator validator) throws IOExcept manifestFiles = new HashMap<>(); Map debugContext = new HashMap<>(); - if (this.debugSourcePath != null && !this.debugSourcePath.isEmpty()) { + String debugSourcePath = buildState.getDebugSourcePath(); + if (debugSourcePath != null && !debugSourcePath.isEmpty()) { List flags = new ArrayList<>(List.of( - String.format("-fdebug-compilation-dir=%s", this.debugSourcePath), + String.format("-fdebug-compilation-dir=%s", debugSourcePath), "-fdebug-prefix-map=upload=extensions", "-fdebug-prefix-map=build=generated" )); @@ -2172,7 +2239,7 @@ private void loadManifests(ExtensionManifestValidator validator) throws IOExcept HashMap _manifestConfigs = new HashMap<>(); for (File manifest : this.manifests) { - ManifestConfiguration manifestConfig = Extender.loadYaml(this.jobDirectory, manifest, ManifestConfiguration.class); + ManifestConfiguration manifestConfig = Extender.loadYaml(buildState.jobDir, manifest, ManifestConfiguration.class); if (manifestConfig == null) { throw new ExtenderException("Missing manifest file: " + manifest.getAbsolutePath()); @@ -2185,7 +2252,7 @@ private void loadManifests(ExtensionManifestValidator validator) throws IOExcept List symbols = getSortedKeys(_manifestConfigs.keySet()); for (String extensionSymbol : symbols) { File manifest = manifestFiles.get(extensionSymbol); - String relativePath = ExtenderUtil.getRelativePath(this.uploadDirectory, manifest); + String relativePath = ExtenderUtil.getRelativePath(buildState.uploadDir, manifest); ManifestConfiguration manifestConfig = _manifestConfigs.get(extensionSymbol); @@ -2193,7 +2260,7 @@ private void loadManifests(ExtensionManifestValidator validator) throws IOExcept manifestContext = ExtenderUtil.mergeContexts(manifestContext, this.platformConfig.context); manifestContext = ExtenderUtil.mergeContexts(manifestContext, this.platformVariantConfig.context); - for (String platformAlternative : ExtenderUtil.getPlatformAlternatives(platform)) { + for (String platformAlternative : ExtenderUtil.getPlatformAlternatives(buildState.fullPlatform)) { Map ctx = getManifestContext(platformAlternative, manifestConfig); validator.validate(relativePath, manifest.getParentFile(), ctx); @@ -2218,7 +2285,7 @@ private void loadManifests(ExtensionManifestValidator validator) throws IOExcept for (String extensionSymbol : symbols) { ManifestConfiguration manifestConfig = _manifestConfigs.get(extensionSymbol); - for (String platformAlternative : ExtenderUtil.getPlatformAlternatives(platform)) { + for (String platformAlternative : ExtenderUtil.getPlatformAlternatives(buildState.fullPlatform)) { Map ctx = getManifestContext(platformAlternative, manifestConfig); mergedAppContext = ExtenderUtil.mergeContexts(mergedAppContext, ctx); } @@ -2232,9 +2299,9 @@ private void loadManifests(ExtensionManifestValidator validator) throws IOExcept mergedAppContext.put("extension_name", "unknown"); mergedAppContext.put("extension_name_upper", "UNKNOWN"); - mergedAppContext.put("dynamo_home", ExtenderUtil.getRelativePath(jobDirectory, sdk)); - mergedAppContext.put("platform", this.platform); - mergedAppContext.put("host_platform", this.hostPlatform); + mergedAppContext.put("dynamo_home", ExtenderUtil.getRelativePath(buildState.jobDir, buildState.sdk)); + mergedAppContext.put("platform", buildState.fullPlatform); + mergedAppContext.put("host_platform", buildState.getHostPlatform()); //exclude fake `jar` path for extensions without java code List excludeJars = ExtenderUtil.getStringList(mergedAppContext, "excludeJars"); @@ -2271,12 +2338,12 @@ private List getSortedKeys(Set keyset) { } private boolean shouldBuildArtifact(String artifact) { - List artifacts = Arrays.asList(this.buildArtifacts.split(",")); + List artifacts = Arrays.asList(buildState.getBuildArtifacts().split(",")); return artifacts.contains(artifact); } private boolean shouldBuildEngine() { - return this.buildArtifacts.equals("") || shouldBuildArtifact("engine"); + return buildState.getBuildArtifacts().equals("") || shouldBuildArtifact("engine"); } private boolean shouldBuildPlugins() { return shouldBuildArtifact("plugins"); @@ -2291,7 +2358,7 @@ private List buildLibraries() throws ExtenderException { if (!shouldBuildLibrary()) { return new ArrayList<>(); } - LOGGER.info("Building library for platform {} with extension source {}", platform, uploadDirectory); + LOGGER.info("Building library for platform {} with extension source {}", buildState.fullPlatform, buildState.uploadDir); List outputFiles = new ArrayList<>(); try { @@ -2315,7 +2382,7 @@ private List buildEngine() throws ExtenderException { if (!shouldBuildEngine()) { return new ArrayList<>(); } - LOGGER.info("Building engine for platform {} with extension source {}", platform, uploadDirectory); + LOGGER.info("Building engine for platform {} with extension source {}", buildState.fullPlatform, buildState.uploadDir); List outputFiles = new ArrayList<>(); try { @@ -2342,7 +2409,7 @@ private List buildEngine() throws ExtenderException { } File resourceFile = null; - if (platform.endsWith("win32")) { + if (buildState.fullPlatform.endsWith("win32")) { resourceFile = buildWin32Resources(mergedAppContext); } @@ -2375,7 +2442,7 @@ private boolean isDesktopPlatform(String platform) { // Supported from 1.2.186 private List buildPipelinePlugin() throws ExtenderException { - if (!isDesktopPlatform(platform)) { + if (!isDesktopPlatform(buildState.fullPlatform)) { return new ArrayList<>(); } if (!shouldBuildPlugins()) { @@ -2385,7 +2452,7 @@ private List buildPipelinePlugin() throws ExtenderException { throw new ExtenderException("Trying to build plugins with an old sdk"); } - LOGGER.info("Building pipeline plugin for platform {} with extension source {}", platform, uploadDirectory); + LOGGER.info("Building pipeline plugin for platform {} with extension source {}", buildState.fullPlatform, buildState.uploadDir); try { List output = new ArrayList<>(); @@ -2410,7 +2477,7 @@ private List copyAndroidJniFolders(String platform) throws ExtenderExcepti if (jniFolders.isEmpty()) { return new ArrayList<>(); } - File targetDir = new File(buildDirectory, "jni"); + File targetDir = new File(buildState.buildDir, "jni"); targetDir.mkdir(); try { @@ -2429,7 +2496,7 @@ private List copyAndroidAssetFolders(String platform) throws ExtenderExcep if (assets.isEmpty()) { return new ArrayList<>(); } - File targetDir = new File(buildDirectory, "assets"); + File targetDir = new File(buildState.buildDir, "assets"); targetDir.mkdir(); try { @@ -2447,7 +2514,7 @@ private List copyAndroidResourceFolders(List androidResourceFolder if (androidResourceFolders.isEmpty()) { return new ArrayList<>(); } - File packagesDir = new File(buildDirectory, "packages"); + File packagesDir = new File(buildState.buildDir, "packages"); packagesDir.mkdir(); List packagesList = new ArrayList<>(); @@ -2473,7 +2540,7 @@ private List copyAndroidResourceFolders(List androidResourceFolder private Set copyMetaInformationFiles(List allJars) { Set result = new HashSet<>(); - File metaInfDir = new File(buildDirectory, "META-INF"); + File metaInfDir = new File(buildState.buildDir, "META-INF"); metaInfDir.mkdir(); for (String jarPath : allJars) { try (ZipFile jarFile = new ZipFile(jarPath)) { @@ -2486,7 +2553,7 @@ private Set copyMetaInformationFiles(List allJars) { if (ExtenderUtil.isMetaInfEntryValuable(entry)) { // Extract the file LOGGER.info(String.format("Extracting file: %s", entry.getName())); - result.add(ExtenderUtil.extractFile(jarFile, entry, buildDirectory)); + result.add(ExtenderUtil.extractFile(jarFile, entry, buildState.buildDir)); } } } catch (IOException exc) { @@ -2597,12 +2664,12 @@ private List buildApple(String platform) throws ExtenderException { LOGGER.info("Building Apple specific code"); List outputFiles = new ArrayList<>(); - File resourceDirectory = new File(buildDirectory, "resources"); + File resourceDirectory = new File(buildState.buildDir, "resources"); List resources = ExtenderUtil.listFiles(resourceDirectory, ".*"); outputFiles.addAll(resources); List privacyManifests = new ArrayList<>(); - privacyManifests.addAll(ExtenderUtil.listFilesMatchingRecursive(uploadDirectory, "PrivacyInfo.xcprivacy")); + privacyManifests.addAll(ExtenderUtil.listFilesMatchingRecursive(buildState.uploadDir, "PrivacyInfo.xcprivacy")); // no need to deal with PrivacyInfo manifests from pods because they will be packed into resource bundle // but that functionality saved for the backward compatability with older engine's versions (before 1.10.12) if (resolvedPods != null) { @@ -2615,7 +2682,7 @@ private List buildApple(String platform) throws ExtenderException { } // use the first privacy manifests as-is if it is the only one if (privacyManifests.size() == 1) { - File targetFile = new File(buildDirectory, "PrivacyInfo.xcprivacy"); + File targetFile = new File(buildState.buildDir, "PrivacyInfo.xcprivacy"); try { FileUtils.copyFile(privacyManifests.get(0), targetFile); outputFiles.add(targetFile); @@ -2628,7 +2695,7 @@ private List buildApple(String platform) throws ExtenderException { // use the first privacy manifest as the base and merge the rest into // it (and save it as a new final 'PrivacyInfo.xcprivacy' file) File mainManifest = privacyManifests.remove(0); - File targetManifest = new File(buildDirectory, "PrivacyInfo.xcprivacy"); + File targetManifest = new File(buildState.buildDir, "PrivacyInfo.xcprivacy"); outputFiles.add(targetManifest); // Merge the files @@ -2644,7 +2711,7 @@ private List buildApple(String platform) throws ExtenderException { } private String getBasePlatform(String platform) { - String[] platformParts = this.platform.split("-"); + String[] platformParts = buildState.fullPlatform.split("-"); return platformParts[1]; } @@ -2667,10 +2734,10 @@ private List buildManifests(String platform) throws ExtenderException { } // The merged output will end up here - File targetManifest = new File(buildDirectory, manifestName); + File targetManifest = new File(buildState.buildDir, manifestName); out.add(targetManifest); - File mainManifest = new File(uploadDirectory, manifestName); + File mainManifest = new File(buildState.uploadDir, manifestName); // Make sure they're in the "/manifests//" List allManifests = new ArrayList<>(); @@ -2724,7 +2791,7 @@ private List buildManifests(String platform) throws ExtenderException { } File writeLog() { - File logFile = new File(buildDirectory, "log.txt"); + File logFile = new File(buildState.buildDir, "log.txt"); try { LOGGER.info("Writing log file"); processExecutor.writeLog(logFile); @@ -2736,7 +2803,7 @@ File writeLog() { void resolve(GradleService gradleService) throws ExtenderException { try { - gradlePackages = gradleService.resolveDependencies(this.platformConfig.context, jobDirectory, buildDirectory, useJetifier, outputFiles); + gradlePackages = gradleService.resolveDependencies(this.buildState, this.platformConfig.context, outputFiles); } catch (IOException e) { throw new ExtenderException(e, "Failed to resolve Gradle dependencies. " + e.getMessage()); @@ -2745,7 +2812,7 @@ void resolve(GradleService gradleService) throws ExtenderException { void resolve(CocoaPodsService cocoaPodsService) throws ExtenderException { try { - resolvedPods = cocoaPodsService.resolveDependencies(platformConfig, jobDirectory, platform, configuration); + resolvedPods = cocoaPodsService.resolveDependencies(platformConfig, buildState); } catch (IOException e) { throw new ExtenderException(e, "Failed to resolve CocoaPod dependencies. " + e.getMessage()); @@ -2753,7 +2820,7 @@ void resolve(CocoaPodsService cocoaPodsService) throws ExtenderException { } void build() throws ExtenderException { - outputFiles.addAll(buildManifests(platform)); + outputFiles.addAll(buildManifests(buildState.fullPlatform)); if (shouldBuildLibrary()) { @@ -2762,11 +2829,11 @@ void build() throws ExtenderException { else { // TODO: Thread this step - if (platform.endsWith("android")) { - outputFiles.addAll(buildAndroid(platform)); + if (ExtenderUtil.isAndroidTarget(buildState.fullPlatform)) { + outputFiles.addAll(buildAndroid(buildState.fullPlatform)); } - else if (platform.endsWith("ios") || platform.endsWith("macos")) { - outputFiles.addAll(buildApple(platform)); + else if (ExtenderUtil.isAppleTarget(buildState.fullPlatform)) { + outputFiles.addAll(buildApple(buildState.fullPlatform)); } outputFiles.addAll(buildEngine()); diff --git a/server/src/main/java/com/defold/extender/ExtenderBuildState.java b/server/src/main/java/com/defold/extender/ExtenderBuildState.java new file mode 100644 index 00000000..bba87529 --- /dev/null +++ b/server/src/main/java/com/defold/extender/ExtenderBuildState.java @@ -0,0 +1,111 @@ +package com.defold.extender; + +import java.io.File; + +public class ExtenderBuildState { + static final String APPMANIFEST_BASE_VARIANT_KEYWORD = "baseVariant"; + static final String APPMANIFEST_WITH_SYMBOLS_KEYWORD = "withSymbols"; + static final String APPMANIFEST_BUILD_ARTIFACTS_KEYWORD = "buildArtifacts"; + static final String APPMANIFEST_JETIFIER_KEYWORD = "jetifier"; + static final String APPMANIFEST_DEBUG_SOURCE_PATH = "debugSourcePath"; + + File jobDir; + File uploadDir; + File buildDir; + File sdk; + String buildConfiguration; // debug/release/headless + String fullPlatform; + String arch; + String hostPlatform; + private final String buildArtifacts; + private final String debugSourcePath; + + private final Boolean withSymbols; + private final Boolean useJetifier; + + ExtenderBuildState(Extender.Builder builder, AppManifestConfiguration appManifest) throws ExtenderException { + jobDir = builder.jobDirectory; + buildDir = builder.buildDirectory; + uploadDir = builder.uploadDirectory; + fullPlatform = builder.platform; + sdk = builder.sdk; + arch = fullPlatform.split("-")[0]; + + String baseVariant = ExtenderUtil.getAppManifestContextString(appManifest, APPMANIFEST_BASE_VARIANT_KEYWORD, null); + + this.useJetifier = ExtenderUtil.getAppManifestBoolean(appManifest, builder.platform, APPMANIFEST_JETIFIER_KEYWORD, true); + this.withSymbols = ExtenderUtil.getAppManifestContextBoolean(appManifest, APPMANIFEST_WITH_SYMBOLS_KEYWORD, true); + this.buildArtifacts = ExtenderUtil.getAppManifestContextString(appManifest, APPMANIFEST_BUILD_ARTIFACTS_KEYWORD, ""); + this.debugSourcePath = ExtenderUtil.getAppManifestContextString(appManifest, APPMANIFEST_DEBUG_SOURCE_PATH, null); + // assign configuration names started with upper letter because it used for cocoapods + if (baseVariant != null && (baseVariant.equals("release") || baseVariant.equals("headless"))) { + this.buildConfiguration = "Release"; + } else { + this.buildConfiguration = "Debug"; + } + + String os = System.getProperty("os.name"); + String arch = System.getProperty("os.arch"); + + // These host names are using the Defold SDK names + if (os.contains("Mac")) { + if (arch.contains("aarch64")) { + this.hostPlatform = "arm64-macos"; + } else { + this.hostPlatform = "x86_64-macos"; + } + } else if (os.contains("Windows")) { + this.hostPlatform = "x86_64-win32"; + } else { + if (arch.contains("aarch64")) { + this.hostPlatform = "arm64-linux"; + } else { + this.hostPlatform = "x86_64-linux"; + } + } + } + + public File getJobDir() { + return jobDir; + } + + public File getBuildDir() { + return buildDir; + } + + public File getUploadDir() { + return uploadDir; + } + + public String getBuildPlatform() { + return fullPlatform; + } + + public String getBuildArch() { + return arch; + } + + public String getBuildConfiguration() { + return buildConfiguration; + } + + public String getHostPlatform() { + return hostPlatform; + } + + public String getBuildArtifacts() { + return buildArtifacts; + } + + public String getDebugSourcePath() { + return debugSourcePath; + } + + public Boolean isNeedSymbols() { + return withSymbols; + } + + public Boolean isUsedJetifier() { + return useJetifier; + } +} diff --git a/server/src/main/java/com/defold/extender/ExtenderUtil.java b/server/src/main/java/com/defold/extender/ExtenderUtil.java index 10593fc4..0ef3619f 100644 --- a/server/src/main/java/com/defold/extender/ExtenderUtil.java +++ b/server/src/main/java/com/defold/extender/ExtenderUtil.java @@ -32,6 +32,7 @@ import org.apache.commons.io.filefilter.DirectoryFileFilter; import org.apache.commons.io.filefilter.RegexFileFilter; import org.apache.commons.io.filefilter.TrueFileFilter; +import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.text.StringEscapeUtils; import org.json.simple.JSONObject; import org.springframework.core.io.Resource; @@ -60,7 +61,7 @@ static Pattern createPattern(String expression) { return Pattern.compile(convertStringToLiteral(expression)); } - static private List filterItems(List input, List expressions, boolean keep) { + static private List filterItems(Collection input, Collection expressions, boolean keep) { List items = new ArrayList<>(); List patterns = new ArrayList<>(); @@ -101,16 +102,16 @@ static private List filterItems(List input, List express // Excludes items from input list that matches an item in the expressions list - static List excludeItems(List input, List expressions) { + static List excludeItems(Collection input, Collection expressions) { return filterItems(input, expressions, false); } // Keeps the matching items from input list that matches an item in the expressions list - static private List matchItems(List input, List expressions) { + static private List matchItems(Collection input, Collection expressions) { return filterItems(input, expressions, true); } - static List pruneItems(List input, List includePatterns, List excludePatterns) { + static List pruneItems(Collection input, Collection includePatterns, Collection excludePatterns) { List includeItems = matchItems(input, includePatterns); List items = excludeItems(input, excludePatterns); for( String item : includeItems) { @@ -918,8 +919,8 @@ public static String calculateSHA256(InputStream input) throws NoSuchAlgorithmEx return hexString.toString(); } - public static File writeSourceFilesListToTmpFile(Set fileList) throws IOException { - File resultFile = Files.createTempFile(null, "sourcelist").toFile(); + public static File writeSourceFilesListToTmpFile(File targetDir, Set fileList) throws IOException { + File resultFile = new File(targetDir, String.format("%s.sourcelist", RandomStringUtils.insecure().nextAlphanumeric(30))); Set escapedList = new HashSet<>(); fileList.forEach((elem) -> { escapedList.add(StringEscapeUtils.escapeXSI(elem)); diff --git a/server/src/main/java/com/defold/extender/services/GradleService.java b/server/src/main/java/com/defold/extender/services/GradleService.java index eed89ce9..d3566270 100644 --- a/server/src/main/java/com/defold/extender/services/GradleService.java +++ b/server/src/main/java/com/defold/extender/services/GradleService.java @@ -7,6 +7,7 @@ import org.springframework.stereotype.Service; +import com.defold.extender.ExtenderBuildState; import com.defold.extender.ExtenderException; import io.micrometer.core.instrument.Gauge; @@ -23,9 +24,9 @@ public GradleService(GradleServiceInterface service, Gauge.builder("extender.job.gradle.cacheSize", this, GradleService::getCacheSize).baseUnit(BaseUnits.BYTES).register(registry); } - public List resolveDependencies(Map env, File cwd, File buildDirectory, Boolean useJetifier, List outputFiles) + public List resolveDependencies(ExtenderBuildState buildState, Map env, List outputFiles) throws IOException, ExtenderException { - return gradleService.resolveDependencies(env, cwd, buildDirectory, useJetifier, outputFiles); + return gradleService.resolveDependencies(buildState, env, outputFiles); } public long getCacheSize() { diff --git a/server/src/main/java/com/defold/extender/services/GradleServiceInterface.java b/server/src/main/java/com/defold/extender/services/GradleServiceInterface.java index 0b53abb9..caac966e 100644 --- a/server/src/main/java/com/defold/extender/services/GradleServiceInterface.java +++ b/server/src/main/java/com/defold/extender/services/GradleServiceInterface.java @@ -5,11 +5,12 @@ import java.util.List; import java.util.Map; +import com.defold.extender.ExtenderBuildState; import com.defold.extender.ExtenderException; public interface GradleServiceInterface { // Resolve dependencies, download them, extract to - public List resolveDependencies(Map env, File cwd, File buildDirectory, Boolean useJetifier, List outputFiles) throws IOException, ExtenderException; + public List resolveDependencies(ExtenderBuildState buildState, Map env, List outputFiles) throws IOException, ExtenderException; public long getCacheSize() throws IOException; } diff --git a/server/src/main/java/com/defold/extender/services/MockGradleService.java b/server/src/main/java/com/defold/extender/services/MockGradleService.java index 7cc2db1b..60b7e0c7 100644 --- a/server/src/main/java/com/defold/extender/services/MockGradleService.java +++ b/server/src/main/java/com/defold/extender/services/MockGradleService.java @@ -8,13 +8,14 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; +import com.defold.extender.ExtenderBuildState; import com.defold.extender.ExtenderException; @Service @ConditionalOnProperty(name = "extender.gradle.enabled", havingValue = "false", matchIfMissing = true) public class MockGradleService implements GradleServiceInterface { @Override - public List resolveDependencies(Map env, File cwd, File buildDirectory, Boolean useJetifier, List outputFiles) + public List resolveDependencies(ExtenderBuildState buildState, Map env, List outputFiles) throws IOException, ExtenderException { return List.of(); } diff --git a/server/src/main/java/com/defold/extender/services/RealGradleService.java b/server/src/main/java/com/defold/extender/services/RealGradleService.java index 3fdb2678..afd16340 100644 --- a/server/src/main/java/com/defold/extender/services/RealGradleService.java +++ b/server/src/main/java/com/defold/extender/services/RealGradleService.java @@ -1,5 +1,6 @@ package com.defold.extender.services; +import com.defold.extender.ExtenderBuildState; import com.defold.extender.ExtenderException; import com.defold.extender.ExtenderUtil; import com.defold.extender.TemplateExecutor; @@ -87,32 +88,35 @@ private Map createJobEnvContext(Map env) { } @Override - public List resolveDependencies(Map env, File cwd, File buildDirectory, Boolean useJetifier, List outputFiles) throws IOException, ExtenderException { + public List resolveDependencies(ExtenderBuildState buildState, Map env, List outputFiles) throws IOException, ExtenderException { + // cwd -> jobDir + File workDir = buildState.getJobDir(); + File buildDir = buildState.getBuildDir(); Map jobEnvContext = createJobEnvContext(env); // create build.gradle - File mainGradleFile = new File(cwd, "build.gradle"); - List gradleFiles = ExtenderUtil.listFilesMatchingRecursive(cwd, "build\\.gradle"); + File mainGradleFile = new File(workDir, "build.gradle"); + List gradleFiles = ExtenderUtil.listFilesMatchingRecursive(workDir, "build\\.gradle"); // This file might exist when testing and debugging the extender using a debug job folder gradleFiles.remove(mainGradleFile); createBuildGradleFile(mainGradleFile, gradleFiles, jobEnvContext); // create gradle.properties - File gradlePropertiesFile = new File(cwd, "gradle.properties"); - createGradlePropertiesFile(gradlePropertiesFile, useJetifier); + File gradlePropertiesFile = new File(workDir, "gradle.properties"); + createGradlePropertiesFile(gradlePropertiesFile, buildState.isUsedJetifier()); // create local.properties - File localPropertiesFile = new File(cwd, "local.properties"); + File localPropertiesFile = new File(workDir, "local.properties"); createLocalPropertiesFile(localPropertiesFile, jobEnvContext); // download, parse and unpack dependencies - List unpackedDependencies = downloadDependencies(cwd); + List unpackedDependencies = downloadDependencies(workDir); // add gradle lockfile to outputs // configured in template.build.gradle - outputFiles.add(new File(buildDirectory, "gradle.lockfile")); + outputFiles.add(new File(buildDir, "gradle.lockfile")); // write dependency tree and add to outputs - File dependencyTreeFile = new File(buildDirectory, "gradle.dependencytree"); - writeDependencyTree(dependencyTreeFile, cwd); + File dependencyTreeFile = new File(buildDir, "gradle.dependencytree"); + writeDependencyTree(dependencyTreeFile, workDir); outputFiles.add(dependencyTreeFile); return unpackedDependencies; diff --git a/server/src/main/java/com/defold/extender/services/cocoapods/CocoaPodsService.java b/server/src/main/java/com/defold/extender/services/cocoapods/CocoaPodsService.java index 7213f698..21959088 100644 --- a/server/src/main/java/com/defold/extender/services/cocoapods/CocoaPodsService.java +++ b/server/src/main/java/com/defold/extender/services/cocoapods/CocoaPodsService.java @@ -1,5 +1,6 @@ package com.defold.extender.services.cocoapods; +import com.defold.extender.ExtenderBuildState; import com.defold.extender.ExtenderException; import com.defold.extender.ExtenderUtil; import com.defold.extender.TemplateExecutor; @@ -8,6 +9,7 @@ import com.defold.extender.process.ProcessUtils; import org.apache.commons.io.FileUtils; +import org.json.simple.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; @@ -38,16 +40,19 @@ import java.util.HashSet; import java.util.LinkedHashSet; import java.util.UUID; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - @Service @ConditionalOnProperty(prefix = "extender", name = "cocoapods.enabled", havingValue = "true") public class CocoaPodsService { - static MainPodfile createMainPodfile() { - return new MainPodfile(); + static MainPodfile mainPodfileFromParseResult(PodfileParser.ParseResult parseResult, File workingDir) { + MainPodfile res = new MainPodfile(); + res.file = new File(workingDir, "Podfile"); + res.platform = parseResult.platform; + res.platformMinVersion = parseResult.minVersion; + res.podDefinitions = parseResult.podDefinitions; + res.useFrameworks = parseResult.useFrameworks; + return res; } private class InstalledPods { @@ -114,24 +119,6 @@ private static void dumpDir(File file, int indent) throws IOException { } } - // https://www.baeldung.com/java-comparing-versions#customSolution - private static int compareVersions(String version1, String version2) { - int result = 0; - String[] parts1 = version1.split("\\."); - String[] parts2 = version2.split("\\."); - int length = Math.max(parts1.length, parts2.length); - for (int i = 0; i < length; i++){ - Integer v1 = i < parts1.length ? Integer.parseInt(parts1[i]) : 0; - Integer v2 = i < parts2.length ? Integer.parseInt(parts2[i]) : 0; - int compare = v1.compareTo(v2); - if (compare != 0) { - result = compare; - break; - } - } - return result; - } - /** * Create the main Podfile with a list of all dependencies for all uploaded extensions * @param podFiles List of podfiles to merge into the main Pofile @@ -139,28 +126,26 @@ private static int compareVersions(String version1, String version2) { * @param workingDir The working directory where pods should be resolved * @param platform For which platform to resolve pods * @return Main pod file structure + * @throws PodfileParsingException + * @throws IOException */ - private MainPodfile createMainPodfile(List podFiles, File jobDirectory, File workingDir, String platform, Map jobEnvContext) throws IOException { - MainPodfile mainPodfile = createMainPodfile(); - - mainPodfile.file = new File(workingDir, "Podfile"); - - // This file might exist when testing and debugging the extender using a debug job folder - podFiles.remove(mainPodfile.file); - - mainPodfile.platformMinVersion = (platform.contains("ios") ? + private MainPodfile createMainPodfile(ExtenderBuildState buildState, CocoaPodsServiceBuildState cocoapodsBuildState, List podFiles, Map jobEnvContext) throws IOException, PodfileParsingException { + boolean isIOS = ExtenderUtil.isIOSTarget(buildState.getBuildPlatform()); + String podPlatform = isIOS ? "ios" : "osx"; + String defaultMinVersion = (isIOS ? jobEnvContext.get("env.IOS_VERSION_MIN").toString(): jobEnvContext.get("env.MACOS_VERSION_MIN").toString()); - mainPodfile.platform = (platform.contains("ios") ? "ios" : "osx"); // Load all Podfiles - List pods = parsePodfiles(mainPodfile, podFiles); + PodfileParser.ParseResult podParseResult = parsePodfiles(podFiles, podPlatform, defaultMinVersion); + MainPodfile mainPodfile = mainPodfileFromParseResult(podParseResult, cocoapodsBuildState.getWorkingDir()); // Create main Podfile contents HashMap envContext = new HashMap<>(); envContext.put("PLATFORM", mainPodfile.platform); envContext.put("PLATFORM_VERSION", mainPodfile.platformMinVersion); - envContext.put("PODS", pods); + envContext.put("PODS", mainPodfile.podDefinitions); + envContext.put("USE_FRAMEWORKS", mainPodfile.useFrameworks); String mainPodfileContents = templateExecutor.execute(podfileTemplateContents, envContext); LOGGER.info("Created main Podfile:\n{}", mainPodfileContents); @@ -169,38 +154,17 @@ private MainPodfile createMainPodfile(List podFiles, File jobDirectory, Fi return mainPodfile; } - // copy the files and folders of a directory recursively - // the function will also resolve symlinks while copying files and folders - private void copyDirectoryRecursively(File fromDir, File toDir) throws IOException { - toDir.mkdirs(); - File[] files = fromDir.toPath().toRealPath().toFile().listFiles(); - for (File file : files) { - if (file.isDirectory()) { - File sourceDir = file; - File destDir = new File(toDir, file.getName()); - copyDirectoryRecursively(sourceDir, destDir); - } - else { - // follow symlink (if one exists) and then copy file - File sourceFile = file.toPath().toRealPath().toFile(); - File destFile = new File(toDir, sourceFile.getName()); - Files.copy(sourceFile.toPath(), destFile.toPath()); - } - } - } - - private void unpackXCFrameworks(List pods, File podsDir, PlatformConfig config) throws IOException, ExtenderException { + private void unpackXCFrameworks(CocoaPodsServiceBuildState cocoapodsBuildState, List pods) throws IOException, ExtenderException { LOGGER.info("Unpack xcframeworks"); - File targetSupportFileDir = new File(podsDir, "Target Support Files"); Set handledPods = new HashSet<>(); - for (PodSpec spec : pods) { - String podName = spec.getPodName(); + for (PodBuildSpec spec : pods) { + String podName = spec.name; if (handledPods.contains(podName)) { continue; } handledPods.add(podName); - File unpackScript = Path.of(targetSupportFileDir.toString(), podName, String.format("%s-xcframeworks.sh", podName)).toFile(); + File unpackScript = Path.of(cocoapodsBuildState.getTargetSupportFilesDir().toString(), podName, String.format("%s-xcframeworks.sh", podName)).toFile(); if (unpackScript.exists()) { ProcessUtils.execCommand(List.of( unpackScript.getAbsolutePath() @@ -211,8 +175,8 @@ private void unpackXCFrameworks(List pods, File podsDir, PlatformConfig } } - void generateSwiftCompatabilityModule(List pods) { - for (PodSpec spec : pods) { + void generateSwiftCompatabilityModule(List pods) { + for (PodBuildSpec spec : pods) { if (spec.swiftSourceFiles.isEmpty()) { continue; } @@ -243,15 +207,13 @@ private Set getPodDeps(Map> specDepsMap, List jobEnvContext, String configuration) throws IOException, ExtenderException { + private InstalledPods installPods(ExtenderBuildState buildState, CocoaPodsServiceBuildState cocoapodsBuildState, + Map jobEnvContext) throws IOException, ExtenderException { LOGGER.info("Installing pods"); Path cacheDir; // store current cache dir into local variable to use the same value for all 'pod' runs @@ -260,6 +222,7 @@ private InstalledPods installPods(PodSpecParser.Platform selectedPlatform, Strin } InstalledPods installedPods = new InstalledPods(); + File workingDir = cocoapodsBuildState.getWorkingDir(); File podFile = new File(workingDir, "Podfile"); if (!podFile.exists()) { throw new ExtenderException("Unable to find Podfile " + podFile); @@ -366,18 +329,8 @@ public boolean accept(File f) { // "dependencies": { // "GoogleAppMeasurement": [ specJson = specJson.substring(specJson.indexOf("{", 0), specJson.length()); - File buildDir = new File(jobDir, "build"); - XCConfigParser parser = new XCConfigParser(buildDir, podsDir, selectedPlatform.toString().toLowerCase(), configuration, arch); - PodSpecParser.CreatePodSpecArgs args = new PodSpecParser.CreatePodSpecArgs.Builder() - .setSpecJson(specJson) - .setPodsDir(podsDir) - .setBuildDir(buildDir) - .setJobContext(jobEnvContext) - .setSelectedPlatform(selectedPlatform) - .setConfiguration(configuration) - .setConfigParser(parser) - .build(); - installedPods.podsMap.put(podName, PodSpecParser.createPodSpec(args)); + JSONObject spec = PodSpecParser.parseJson(specJson); + installedPods.podsMap.put(podName, PodSpecParser.createPodSpec(spec, cocoapodsBuildState.getSelectedPlatform(), null)); } else { LOGGER.warn("No version information for pod {}", podName); } @@ -406,12 +359,14 @@ private Map createJobEnvContext(Map env) { * @param configuration Build configuration ("debug", "release", "headless") * @return ResolvedPods instance with list of pods, install directory etc */ - public ResolvedPods resolveDependencies(PlatformConfig config, File jobDir, String platform, String configuration) throws IOException, ExtenderException { - if (!platform.contains("ios") && !platform.contains("osx")) { + public ResolvedPods resolveDependencies(PlatformConfig config, ExtenderBuildState buildState) throws IOException, ExtenderException { + String platform = buildState.getBuildPlatform(); + if (!ExtenderUtil.isAppleTarget(platform)) { throw new ExtenderException("Unsupported platform " + platform); } Map jobEnvContext = createJobEnvContext(config.context); + File jobDir = buildState.getJobDir(); // find all podfiles and filter down to a list of podfiles specifically // for the platform we are resolving pods for @@ -435,90 +390,75 @@ public ResolvedPods resolveDependencies(PlatformConfig config, File jobDir, Stri long methodStart = System.currentTimeMillis(); LOGGER.info("Resolving Cocoapod dependencies"); - File workingDir = new File(jobDir, "CocoaPodsService"); - File frameworksDir = Path.of(jobDir.toString(), "build", "Debugiphoneos", "XCFrameworkIntermediates").toFile(); - workingDir.mkdirs(); - frameworksDir.mkdirs(); - String[] platformParts = platform.split("-"); - PodSpecParser.Platform selectedPlatform = PodSpecParser.Platform.UNKNOWN; - String arch = platformParts[0]; - if (platform.contains("ios")) { - selectedPlatform = arch.equals("arm64") ? PodSpecParser.Platform.IPHONEOS : PodSpecParser.Platform.IPHONESIMULATOR; - } else if (platform.contains("osx")) { - selectedPlatform = PodSpecParser.Platform.MACOSX; - } + CocoaPodsServiceBuildState cocoapodsBuildState = new CocoaPodsServiceBuildState(buildState); + MainPodfile mainPodfile = createMainPodfile(buildState, cocoapodsBuildState, platformPodfiles, jobEnvContext); + InstalledPods installedPods = installPods(buildState, cocoapodsBuildState, jobEnvContext); - File podsDir = new File(workingDir, "Pods"); - MainPodfile mainPodfile = createMainPodfile(platformPodfiles, jobDir, workingDir, platform, jobEnvContext); - InstalledPods installedPods = installPods(selectedPlatform, arch, jobDir, workingDir, jobEnvContext, configuration); - List pods = new ArrayList<>(); + + XCConfigParser parser = new XCConfigParser(buildState, cocoapodsBuildState); + CreateBuildSpecArgs args = new CreateBuildSpecArgs.Builder() + .setJobContext(jobEnvContext) + .setConfigParser(parser) + .setExtenderBuildState(buildState) + .setCocoapodsBuildState(cocoapodsBuildState) + .build(); + Map tmpRegistry = new HashMap<>(); + List pods = new ArrayList<>(); for (String specName : installedPods.pods) { String podName = PodUtils.getPodName(specName); - PodSpec currentSpec = installedPods.podsMap.get(podName); + PodSpec podSpec = installedPods.podsMap.get(podName); String podnameparts[] = PodUtils.splitPodname(specName); if (podnameparts.length > 1) { for (int i = 1; i < podnameparts.length; i++) { String subspecname = podnameparts[i]; - PodSpec subspec = currentSpec.getSubspec(subspecname); + PodSpec subspec = podSpec.getSubspec(subspecname); if (subspec == null) { throw new ExtenderException(String.format("Unable to find subspec '%s' in pod '%s'", subspecname, podName)); } - currentSpec = subspec; + podSpec = subspec; } } - pods.add(currentSpec); + + PodBuildSpec buildSpec = null; + if (!tmpRegistry.containsKey(podName)) { + buildSpec = new PodBuildSpec(args, podSpec.parentSpec != null ? podSpec.parentSpec : podSpec); + tmpRegistry.put(podName, buildSpec); + pods.add(buildSpec); + } else { + buildSpec = tmpRegistry.get(podName); + } + if (podSpec.parentSpec != null) { + buildSpec.addSubSpec(podSpec); + } + + for (String subSpecName : podSpec.dependencies) { + String depPodName = PodUtils.getPodName(subSpecName); + PodBuildSpec depBuildSpec = tmpRegistry.get(depPodName); + assert(depBuildSpec != null); + buildSpec.dependantSpecs.add(depBuildSpec); + } } - unpackXCFrameworks(pods, podsDir, config); + unpackXCFrameworks(cocoapodsBuildState, pods); generateSwiftCompatabilityModule(pods); dumpDir(jobDir, 0); MetricsWriter.metricsTimer(meterRegistry, "extender.service.cocoapods.get", System.currentTimeMillis() - methodStart); - ResolvedPods resolvedPods = new ResolvedPods(podsDir, frameworksDir, pods, installedPods.podfileLock, mainPodfile.platformMinVersion); + ResolvedPods resolvedPods = new ResolvedPods(cocoapodsBuildState, pods, installedPods.podfileLock, mainPodfile); LOGGER.info("Resolved Cocoapod dependencies"); LOGGER.info(resolvedPods.toString()); return resolvedPods; } - static List parsePodfiles(MainPodfile mainPodfile, List podFiles) throws IOException { - // Load all Podfiles - Pattern podPattern = Pattern.compile("pod '([\\w|-]+)'.*"); - List pods = new ArrayList<>(); + static PodfileParser.ParseResult parsePodfiles(List podFiles, String platform, String defaultMinVersion) throws IOException, PodfileParsingException { + PodfileParser.ParseResult result = new PodfileParser.ParseResult(platform, defaultMinVersion); for (File podFile : podFiles) { - // Split each file into lines and go through them one by one - // Search for a Podfile platform and version configuration, examples: - // platform :ios, '9.0' - // platform :osx, '10.2' - // Get the version and figure out which is the highest version defined. This - // version will be used in the combined Podfile created by this function. - // Treat everything else as pods - List lines = Files.readAllLines(podFile.getAbsoluteFile().toPath()); - for (String line : lines) { - if (line.isEmpty()) { - continue; - } - if (line.startsWith("platform :")) { - String version = line.replaceFirst("platform :ios|platform :osx", "").replace(",", "").replace("'", "").trim(); - if (!version.isEmpty() && (compareVersions(version, mainPodfile.platformMinVersion) > 0)) { - mainPodfile.platformMinVersion = version; - } - } - else { - pods.add(line); - - Matcher matcher = podPattern.matcher(line); - if (matcher.matches()) { - // get the pod name from the line - // example: pod 'KSCrash', '1.17.4' -> KSCrash - String podname = matcher.group(1); - mainPodfile.podnames.add(podname); - } - } - } + PodfileParser.ParseResult res = PodfileParser.parsePodfile(podFile); + result.mergeWith(res); } - return pods; + return result; } private Path generateCacheDirPath() { diff --git a/server/src/main/java/com/defold/extender/services/cocoapods/CocoaPodsServiceBuildState.java b/server/src/main/java/com/defold/extender/services/cocoapods/CocoaPodsServiceBuildState.java new file mode 100644 index 00000000..0f91cbf3 --- /dev/null +++ b/server/src/main/java/com/defold/extender/services/cocoapods/CocoaPodsServiceBuildState.java @@ -0,0 +1,60 @@ +package com.defold.extender.services.cocoapods; + +import java.io.File; +import java.nio.file.Path; + +import com.defold.extender.ExtenderBuildState; +import com.defold.extender.ExtenderUtil; + +public class CocoaPodsServiceBuildState { + File workingDir; + File podsDir; + File unpackedFrameworksDir; + File targetSupportFilesDir; + PodUtils.Platform selectedPlatform; + + CocoaPodsServiceBuildState() { } + + CocoaPodsServiceBuildState(ExtenderBuildState extenderBuildState) { + this.workingDir = new File(extenderBuildState.getJobDir(), "CocoaPodsService"); + this.workingDir.mkdirs(); + this.podsDir = new File(workingDir, "Pods"); + this.podsDir.mkdirs(); + + this.selectedPlatform = PodUtils.Platform.UNKNOWN; + String platform = extenderBuildState.getBuildPlatform(); + if (ExtenderUtil.isIOSTarget(platform)) { + this.selectedPlatform = extenderBuildState.getBuildArch().equals("arm64") ? PodUtils.Platform.IPHONEOS : PodUtils.Platform.IPHONESIMULATOR; + } else if (ExtenderUtil.isMacOSTarget(platform)) { + this.selectedPlatform = PodUtils.Platform.MACOSX; + } + this.unpackedFrameworksDir = Path.of( + extenderBuildState.getBuildDir().toString(), + String.format("%s%s", extenderBuildState.getBuildConfiguration(), this.selectedPlatform.toString().toLowerCase()), + "XCFrameworkIntermediates" + ).toFile(); + this.unpackedFrameworksDir.mkdirs(); + + this.targetSupportFilesDir = new File(this.podsDir, "Target Support Files"); + } + + public File getWorkingDir() { + return workingDir; + } + + public File getPodsDir() { + return podsDir; + } + + public File getUnpackedFrameworksDir() { + return unpackedFrameworksDir; + } + + public PodUtils.Platform getSelectedPlatform() { + return selectedPlatform; + } + + public File getTargetSupportFilesDir() { + return targetSupportFilesDir; + } +} diff --git a/server/src/main/java/com/defold/extender/services/cocoapods/CreateBuildSpecArgs.java b/server/src/main/java/com/defold/extender/services/cocoapods/CreateBuildSpecArgs.java new file mode 100644 index 00000000..29566b8f --- /dev/null +++ b/server/src/main/java/com/defold/extender/services/cocoapods/CreateBuildSpecArgs.java @@ -0,0 +1,70 @@ +package com.defold.extender.services.cocoapods; + +import java.io.File; +import java.util.Map; + +import com.defold.extender.ExtenderBuildState; + +public class CreateBuildSpecArgs { + public static class Builder { + private File podsDir; + private File buildDir; + private PodUtils.Platform selectedPlatform; + private String configuration; + private Map jobEnvContext; + private IConfigParser configParser; + + public Builder() { } + + public Builder setJobContext(Map context) { + this.jobEnvContext = context; + return this; + } + + public Builder setConfigParser(IConfigParser parser) { + this.configParser = parser; + return this; + } + + public Builder setExtenderBuildState(ExtenderBuildState buildState) { + this.buildDir = buildState.getBuildDir(); + this.configuration = buildState.getBuildConfiguration(); + return this; + } + + public Builder setCocoapodsBuildState(CocoaPodsServiceBuildState buildState) { + this.podsDir = buildState.getPodsDir(); + this.selectedPlatform = buildState.getSelectedPlatform(); + return this; + } + + public CreateBuildSpecArgs build() { + return new CreateBuildSpecArgs(this); + } + } + + private CreateBuildSpecArgs(Builder builder) { + this.podsDir = builder.podsDir; + this.buildDir = builder.buildDir; + this.jobEnvContext = builder.jobEnvContext; + this.selectedPlatform = builder.selectedPlatform; + this.configuration = builder.configuration; + this.configParser = builder.configParser; + } + + private CreateBuildSpecArgs(CreateBuildSpecArgs copy) { + this.podsDir = copy.podsDir; + this.buildDir = copy.buildDir; + this.jobEnvContext = copy.jobEnvContext; + this.selectedPlatform = copy.selectedPlatform; + this.configuration = copy.configuration; + this.configParser = copy.configParser; + } + + protected File podsDir; + protected File buildDir; + protected PodUtils.Platform selectedPlatform; + protected String configuration; + protected Map jobEnvContext; + protected IConfigParser configParser; +} diff --git a/server/src/main/java/com/defold/extender/services/cocoapods/LanguageSet.java b/server/src/main/java/com/defold/extender/services/cocoapods/LanguageSet.java index 8c45db73..45e9df65 100644 --- a/server/src/main/java/com/defold/extender/services/cocoapods/LanguageSet.java +++ b/server/src/main/java/com/defold/extender/services/cocoapods/LanguageSet.java @@ -12,6 +12,16 @@ public class LanguageSet { public Set objcpp = new LinkedHashSet<>(); public List swift = new ArrayList<>(); + public LanguageSet() {} + + public LanguageSet(LanguageSet other) { + this.c.addAll(other.c); + this.cpp.addAll(other.cpp); + this.objc.addAll(other.objc); + this.objcpp.addAll(other.objcpp); + this.swift.addAll(other.swift); + } + public void add(String value) { c.add(value); cpp.add(value); diff --git a/server/src/main/java/com/defold/extender/services/cocoapods/MainPodfile.java b/server/src/main/java/com/defold/extender/services/cocoapods/MainPodfile.java index ccf52c86..6770d7af 100644 --- a/server/src/main/java/com/defold/extender/services/cocoapods/MainPodfile.java +++ b/server/src/main/java/com/defold/extender/services/cocoapods/MainPodfile.java @@ -1,23 +1,24 @@ package com.defold.extender.services.cocoapods; import java.io.File; -import java.util.ArrayList; -import java.util.List; +import java.util.Set; public class MainPodfile { - public List podnames = new ArrayList<>(); + public Set podDefinitions; public String platformMinVersion; public String platform; public File file; + public boolean useFrameworks = true; // left true by default for now for backward compatability @Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("file: " + file); - sb.append("pod count: " + podnames.size() + "\n"); - sb.append("pod names: " + podnames + "\n"); + sb.append("pod count: " + podDefinitions.size() + "\n"); + sb.append("pod definitions: " + podDefinitions + "\n"); sb.append("platform: " + platform + "\n"); sb.append("min version: " + platformMinVersion + "\n"); + sb.append("use framerorks: " + String.valueOf(useFrameworks) + "\n"); return sb.toString(); } } diff --git a/server/src/main/java/com/defold/extender/services/cocoapods/PodBuildSpec.java b/server/src/main/java/com/defold/extender/services/cocoapods/PodBuildSpec.java new file mode 100644 index 00000000..9a26e77f --- /dev/null +++ b/server/src/main/java/com/defold/extender/services/cocoapods/PodBuildSpec.java @@ -0,0 +1,387 @@ +package com.defold.extender.services.cocoapods; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.text.StringEscapeUtils; + +// similar to PodSpec but contains some runtime information that used during the build +public class PodBuildSpec { + public String name = ""; + public String moduleName = ""; + // https://github.com/CocoaPods/CocoaPods/blob/648ccdcaea2063fe63977a0146e1717aec3efa54/lib/cocoapods/target.rb#L157 + public String version = ""; + public String platformVersion = ""; + // The Swift source file header (ModuleName-Swift.h) + // This file is referenced from the modulemap and generated in + // Extender.java as part of the process when building .swift files + public File swiftModuleHeader = null; + public String swiftModuleDefinition = null; + public Set swiftSourceFilePaths = new LinkedHashSet<>(); // list of absolute file path which used to create temprorary file with paths to include into comile command as @tmp_path + public Set swiftSourceFiles = new LinkedHashSet<>(); + public Set sourceFiles = new LinkedHashSet<>(); + public Set publicHeaders = new LinkedHashSet<>(); + public Set privateHeaders = new LinkedHashSet<>(); + public Set includePaths = new LinkedHashSet<>(); + public Set frameworkSearchPaths; + public Map> resourceBundles; + + public LanguageSet flags = new LanguageSet(); + public Set defines = new HashSet<>(); + public List linkflags = new ArrayList<>(); + public Set vendoredFrameworks = new LinkedHashSet<>(); + public Set weakFrameworks = new HashSet<>(); + public Set resources = new HashSet<>(); + public Set frameworks = new HashSet<>(); + public Set libraries = new HashSet<>(); + public Map parsedXCConfig = null; + public File dir; + public File buildDir; + public File intermediatedDir; + public File headerMapFile; + public File vfsOverlay; + public Collection dependantSpecs = new HashSet<>(); + + PodBuildSpec() {} + + public PodBuildSpec(CreateBuildSpecArgs args, PodSpec mainSpec) throws IOException { + this.name = mainSpec.name; + this.moduleName = mainSpec.moduleName; + this.version = mainSpec.version; + this.platformVersion = mainSpec.platformVersion; + this.frameworkSearchPaths = new LinkedHashSet<>(mainSpec.frameworkSearchPaths); + this.resourceBundles = new HashMap<>(mainSpec.resourceBundles); + + this.flags = new LanguageSet(mainSpec.flags); + this.defines = new HashSet<>(mainSpec.defines); + this.linkflags = new ArrayList<>(mainSpec.linkflags); + this.vendoredFrameworks = new LinkedHashSet<>(mainSpec.vendoredFrameworks); + this.weakFrameworks = new HashSet<>(mainSpec.weakFrameworks); + this.resources = new HashSet<>(mainSpec.resources); + this.frameworks = new HashSet<>(mainSpec.frameworks); + this.libraries = new HashSet<>(mainSpec.libraries); + + + String platformVersionKey = null; + if (PodUtils.isIOS(args.selectedPlatform)) { + platformVersionKey = "env.IOS_VERSION_MIN"; + } else if (PodUtils.isMacOS(args.selectedPlatform)) { + platformVersionKey = "env.MACOS_VERSION_MIN"; + } + + this.platformVersion = !mainSpec.platformVersion.isEmpty() ? + mainSpec.platformVersion : + (String)args.jobEnvContext.get(platformVersionKey); + + this.dir = new File(args.podsDir, this.name); + + // generated files relating to the pod + // modulemap, swift header etc + this.buildDir = Path.of(args.buildDir.toString(), String.format("%s%s", args.configuration, args.selectedPlatform.toString().toLowerCase()), mainSpec.getPodName()).toFile(); + this.buildDir.mkdirs(); + + this.intermediatedDir = new File(this.buildDir, "intermediate"); + this.intermediatedDir.mkdirs(); + + collectSourceFilesFromSpec(mainSpec); + + // parse generated xcconfig + String configName = this.name; + this.parsedXCConfig = args.configParser.parse(this.moduleName, configName, Path.of(args.podsDir.toString(), "Target Support Files", configName, String.format("%s.%s.xcconfig", configName, args.configuration.toLowerCase())).toFile()); + updateFlagsFromConfig(parsedXCConfig); + + this.headerMapFile = new File(this.intermediatedDir, String.format("%s.hmap", mainSpec.getPodName())); + + this.flags.add(String.format("-iquote %s", this.headerMapFile.toString())); + this.flags.swift.add(String.format("-Xcc -iquote -Xcc %s", this.headerMapFile.toString())); + + this.vfsOverlay = new File(this.intermediatedDir, "all_files.yaml"); + this.flags.add(String.format("-ivfsoverlay %s", this.vfsOverlay.toString())); + this.flags.swift.add(String.format("-Xcc -ivfsoverlay -Xcc %s", this.vfsOverlay.toString())); + } + + public void addSubSpec(PodSpec subSpec) { + collectSourceFilesFromSpec(subSpec); + + frameworkSearchPaths.addAll(subSpec.frameworkSearchPaths); + resourceBundles.putAll(subSpec.resourceBundles); + + flags.addAll(subSpec.flags); + defines.addAll(subSpec.defines); + linkflags.addAll(subSpec.linkflags); + vendoredFrameworks.addAll(subSpec.vendoredFrameworks); + weakFrameworks.addAll(subSpec.weakFrameworks); + resources.addAll(subSpec.resources); + frameworks.addAll(subSpec.frameworks); + libraries.addAll(subSpec.libraries); + } + + void updateFlagsFromConfig(Map parsedConfig) { + // https://pewpewthespells.com/blog/buildsettings.html + // defines + List defs = argumentsAsList(parsedConfig.getOrDefault("GCC_PREPROCESSOR_DEFINITIONS", null)); + if (defs != null) { + this.defines.addAll(unescapeStrings(defs)); + } + // linker flags + // https://xcodebuildsettings.com/#other_ldflags + List ldFlags = argumentsAsList(parsedConfig.getOrDefault("OTHER_LDFLAGS", null)); + if (ldFlags != null) { + this.linkflags.addAll(ldFlags); + } + // compiler flags for c and objc files + // https://xcodebuildsettings.com/#other_cflags + List cFlags = argumentsAsList(parsedConfig.getOrDefault("OTHER_CFLAGS", null)); + if (cFlags != null) { + this.flags.c.addAll(cFlags); + this.flags.objc.addAll(cFlags); + } + // compiler flags + // https://developer.apple.com/documentation/xcode/build-settings-reference#C++-Language-Dialect + if (hasString(parsedConfig, "CLANG_CXX_LANGUAGE_STANDARD")) { + String cppStandard = parsedConfig.getOrDefault("CLANG_CXX_LANGUAGE_STANDARD", "compiler-default"); + String compilerFlag = ""; + switch (cppStandard) { + case "c++98": compilerFlag = "-std=c++98"; break; + case "c++11": + case "c++0x": compilerFlag = "-std=c++11"; break; + case "gnu++11": + case "gnu++0x": compilerFlag = "-std=gnu++11"; break; + case "c++14": compilerFlag = "-std=c++14"; break; + case "gnu++14": compilerFlag = "-std=gnu++17"; break; + case "c++17": compilerFlag = "-std=c++17"; break; + case "gnu++17": compilerFlag = "-std=gnu++17"; break; + case "c++20": compilerFlag = "-std=c++20"; break; + case "gnu++20": compilerFlag = "-std=gnu++20"; break; + case "gnu++98": + case "compiler-default": + default: compilerFlag = "-std=gnu++98"; break; + } + this.flags.cpp.add(compilerFlag); + this.flags.objcpp.add(compilerFlag); + } + if (hasString(parsedConfig, "GCC_C_LANGUAGE_STANDARD")) { + String cStandard = parsedConfig.getOrDefault("GCC_C_LANGUAGE_STANDARD", "compiler-default"); + String compilerFlag = ""; + switch (cStandard) { + case "ansi": compilerFlag = "-ansi"; break; + case "c89": compilerFlag = "-std=c89"; break; + case "gnu89": compilerFlag = "-std=gnu89"; break; + case "c99": compilerFlag = "-std=c99"; break; + case "c11": compilerFlag = "-std=c11"; break; + case "gnu11": compilerFlag = "-std=gnu11"; break; + case "gnu99": + case "compiler-default": + default: compilerFlag = "-std=gnu99"; break; + } + this.flags.c.add(compilerFlag); + } + if (hasString(parsedConfig, "CLANG_CXX_LIBRARY")) { + String stdLib = parsedConfig.getOrDefault("CLANG_CXX_LIBRARY", "compiler-default"); + String stdLibFlag = ""; + switch (stdLib) { + case "libc++": stdLibFlag = "-stdlib=libc++"; break; + case "libstdc++": + case "compiler-default": + default: stdLibFlag = "-stdlib=libstdlibc++"; break; + } + this.flags.cpp.add(stdLibFlag); + this.flags.objcpp.add(stdLibFlag); + } + if (compareString(parsedConfig, "GCC_ENABLE_CPP_EXCEPTIONS", "YES")) { + this.flags.cpp.add("-fcxx-exceptions"); + } + if (compareString(parsedConfig, "GCC_ENABLE_CPP_EXCEPTIONS", "NO")) { + this.flags.cpp.add("-fno-cxx-exceptions"); + } + if (compareString(parsedConfig, "GCC_ENABLE_EXCEPTIONS", "YES")) { + this.flags.add("-fexceptions"); + } + if (compareString(parsedConfig, "GCC_ENABLE_EXCEPTIONS", "NO")) { + this.flags.add("-fno-exceptions"); + } + if (compareString(parsedConfig, "GCC_ENABLE_OBJC_EXCEPTIONS", "YES")) { + this.flags.objc.add("-fobjc-exceptions"); + this.flags.objcpp.add("-fobjc-exceptions"); + } + if (compareString(parsedConfig, "GCC_ENABLE_OBJC_EXCEPTIONS", "NO")) { + this.flags.objc.add("-fno-objc-exceptions"); + this.flags.objcpp.add("-fno-objc-exceptions"); + } + if (compareString(parsedConfig, "GCC_ENABLE_CPP_RTTI", "YES")) { + this.flags.cpp.add("-frtti"); + } + if (compareString(parsedConfig, "GCC_ENABLE_CPP_RTTI", "NO")) { + this.flags.cpp.add("-fno-rtti"); + } + if (compareString(parsedConfig, "GCC_ENABLE_OBJC_GC", "supported")) { + this.flags.objc.add("-fobjc-gc"); + this.flags.objcpp.add("-fobjc-gc"); + } + if (compareString(parsedConfig, "GCC_ENABLE_OBJC_GC", "required")) { + this.flags.objc.add("-fobjc-gc-only"); + this.flags.objcpp.add("-fobjc-gc-only"); + } + if (compareString(parsedConfig, "GCC_ENABLE_ASM_KEYWORD", "YES")) { + this.flags.add("-fasm"); + } + if (compareString(parsedConfig, "GCC_ENABLE_ASM_KEYWORD", "NO")) { + this.flags.add("-fno-asm"); + } + if (compareString(parsedConfig, "APPLICATION_EXTENSION_API_ONLY", "YES")) { + this.flags.add("-fapplication-extension"); + this.flags.swift.add("-application-extension"); + } + if (hasString(parsedConfig, "SWIFT_INCLUDE_PATHS")) { + List l = argumentsAsList(parsedConfig.getOrDefault("SWIFT_INCLUDE_PATHS", null)); + if (l != null) { + for (String path : l) { + this.flags.swift.add(String.format("-I%s", path)); + this.flags.swift.add(String.format("-Xcc -I%s", path)); + } + } + } + if (hasString(parsedConfig, "OTHER_SWIFT_FLAGS")) { + List l = argumentsAsList(parsedConfig.getOrDefault("OTHER_SWIFT_FLAGS", null)); + if (l != null) { + this.flags.swift.addAll(l); + } + } + if (hasString(parsedConfig, "HEADER_SEARCH_PATHS")) { + List l = argumentsAsList(parsedConfig.getOrDefault("HEADER_SEARCH_PATHS", null)); + if (l != null) { + for (String path : l) { + this.includePaths.add(new File(path)); + } + } + } + if (hasString(parsedConfig, "FRAMEWORK_SEARCH_PATHS")) { + List l = argumentsAsList(parsedConfig.getOrDefault("FRAMEWORK_SEARCH_PATHS", null)); + if (l != null) { + for (String path : l) { + this.frameworkSearchPaths.add(new File(path)); + } + } + } + String compileModuleName = parsedConfig.getOrDefault("PRODUCT_MODULE_NAME", this.name); + this.flags.add(String.format("-fmodule-name=%s", compileModuleName)); + } + + // check if a string value exists + // will return false if the value doesn't exist or is an empty string + static boolean hasString(Map parsedConfig, String key) { + String value = parsedConfig.get(key); + return value != null && !value.trim().isEmpty(); + } + + static List unescapeStrings(List strings) { + List unescapedStrings = new ArrayList<>(); + for (String s : strings) { + unescapedStrings.add(StringEscapeUtils.unescapeJava(s)); + } + return unescapedStrings; + } + + // check if the value for a specific key matches an expected value + static boolean compareString(Map config, String key, String expected) { + String value = config.get(key); + if (value == null || value.trim().isEmpty()) { + return false; + } + return value.equals(expected); + } + + static List argumentsAsList(String arguments) { + arguments = arguments != null ? arguments.trim() : null; + if (arguments == null || arguments.isEmpty()) { + return null; + } + return new ArrayList<>(Arrays.asList(arguments.split(" "))); + } + + /** + * Add source files matching a pattern to a pod + * @param pattern Source file pattern (glob format) + */ + void addPodSourceFiles(String pattern) { + List podSrcFiles = PodUtils.listFilesGlob(this.dir, pattern); + for (File podSrcFile : podSrcFiles) { + final String filename = podSrcFile.getName(); + if (filename.endsWith(".swift")) { + this.swiftSourceFiles.add(podSrcFile); + this.swiftSourceFilePaths.add(podSrcFile.getAbsolutePath()); + } + else if (!PodUtils.isHeaderFile(filename)) { + this.sourceFiles.add(podSrcFile); + } + } + } + + /** + * Add a list of include paths matching a pattern to a pod + * @param pattern Source file pattern (glob format) + */ + void addPodIncludePaths(String pattern) { + List podSrcFiles = PodUtils.listFilesGlob(this.dir, pattern); + for (File podSrcFile : podSrcFiles) { + final String filename = podSrcFile.getName(); + if (PodUtils.isHeaderFile(filename)) { + if (!this.publicHeaders.contains(podSrcFile)) { + this.privateHeaders.add(podSrcFile); + } + } + } + } + + void collectSourceFilesFromSpec(PodSpec spec) { + for (String pattern : spec.publicHeadersPatterns) { + this.publicHeaders.addAll(PodUtils.listFilesGlob(this.dir, pattern)); + } + + Iterator it = spec.sourceFilesPatterns.iterator(); + while (it.hasNext()) { + String path = it.next(); + // don't copy header (and source) files from paths in xcframeworks + // framework headers are copied in a separate step in copyPodFrameworks() + if (!path.contains(".xcframework/")) { + addPodSourceFiles(path); + addPodIncludePaths(path); + } + } + + // add swift libs to the runtime search path + if (!this.swiftSourceFiles.isEmpty()) { + this.linkflags.add("-Wl,-rpath,/usr/lib/swift"); +// ******************************** Added for backward comapatibility ***************************************** +// ******************************** Remove after 6 month ****************************************************** + this.flags.swift.add("-import-underlying-module"); +// ************************************************************************************************************ + } + + // add ObjC link flag if pod contains Objective-C code + for (File sourceFile : this.sourceFiles) { + String filename = sourceFile.getName(); + if (filename.endsWith(".m") || filename.endsWith(".mm")) { + this.linkflags.add("-ObjC"); + break; + } + } + + + // swift compatability header (just path where to store header) + if (this.swiftModuleHeader == null && !this.swiftSourceFiles.isEmpty()) { + this.swiftModuleHeader = Path.of(this.buildDir.toString(), "SwiftCompatibilityHeader", this.moduleName + "-Swift.h").toFile(); + } + } +} diff --git a/server/src/main/java/com/defold/extender/services/cocoapods/PodSpec.java b/server/src/main/java/com/defold/extender/services/cocoapods/PodSpec.java index 231b8518..c484988b 100644 --- a/server/src/main/java/com/defold/extender/services/cocoapods/PodSpec.java +++ b/server/src/main/java/com/defold/extender/services/cocoapods/PodSpec.java @@ -12,19 +12,11 @@ public class PodSpec { public String name = ""; public String moduleName = ""; - // https://github.com/CocoaPods/CocoaPods/blob/648ccdcaea2063fe63977a0146e1717aec3efa54/lib/cocoapods/target.rb#L157 public String version = ""; public String platformVersion = ""; - // The Swift source file header (ModuleName-Swift.h) - // This file is referenced from the modulemap and generated in - // Extender.java as part of the process when building .swift files - public File swiftModuleHeader = null; - public String swiftModuleDefinition = null; - public Set swiftSourceFilePaths = new LinkedHashSet<>(); - public Set swiftSourceFiles = new LinkedHashSet<>(); - public Set sourceFiles = new LinkedHashSet<>(); - public Set headerFiles = new LinkedHashSet<>(); - public Set includePaths = new LinkedHashSet<>(); + public Map platforms = new HashMap<>(); + public Set publicHeadersPatterns = new LinkedHashSet<>(); + public Set sourceFilesPatterns = new LinkedHashSet<>(); public Set frameworkSearchPaths = new LinkedHashSet<>(); public PodSpec parentSpec = null; public List defaultSubspecs = new ArrayList<>(); @@ -34,15 +26,15 @@ public class PodSpec { public LanguageSet flags = new LanguageSet(); public Set defines = new HashSet<>(); - public Set linkflags = new HashSet<>(); + public List linkflags = new ArrayList<>(); public Set vendoredFrameworks = new LinkedHashSet<>(); public Set weakFrameworks = new HashSet<>(); public Set resources = new HashSet<>(); public Set frameworks = new HashSet<>(); public Set libraries = new HashSet<>(); public Map parsedXCConfig = null; - public File dir; - public File buildDir; + + public PodSpec() {} public PodSpec getSubspec(String name) { for (PodSpec spec : subspecs) { @@ -64,13 +56,8 @@ public String getPodName() { public String toString() { StringBuilder sb = new StringBuilder(); sb.append(name + ":" + version + "\n"); - sb.append(" dir: " + dir + "\n"); sb.append(" moduleName: " + moduleName + "\n"); - sb.append(" generated dir: " + buildDir + "\n"); - sb.append(" src: " + sourceFiles + "\n"); - sb.append(" swift src: " + swiftSourceFiles + "\n"); - sb.append(" swift module header: " + swiftModuleHeader + "\n"); - sb.append(" includes: " + includePaths + "\n"); + sb.append(" src: " + sourceFilesPatterns + "\n"); sb.append(" defines: " + defines + "\n"); sb.append(" flags: " + flags + "\n"); sb.append(" linkflags: " + linkflags + "\n"); diff --git a/server/src/main/java/com/defold/extender/services/cocoapods/PodSpecParser.java b/server/src/main/java/com/defold/extender/services/cocoapods/PodSpecParser.java index b1364406..9ce82ed4 100644 --- a/server/src/main/java/com/defold/extender/services/cocoapods/PodSpecParser.java +++ b/server/src/main/java/com/defold/extender/services/cocoapods/PodSpecParser.java @@ -1,8 +1,6 @@ package com.defold.extender.services.cocoapods; -import java.io.File; import java.io.IOException; -import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -12,7 +10,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.commons.text.StringEscapeUtils; import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; @@ -22,148 +19,27 @@ public final class PodSpecParser { private static final Pattern BRACE_PATTERN = Pattern.compile("\\{([^{}]*)\\}"); - // private static final String SWIFT_PARALLEL_FLAG = String.format("-j%d", Runtime.getRuntime().availableProcessors()); - - public enum Platform { - IPHONEOS, - IPHONESIMULATOR, - MACOSX, - UNKNOWN - } - - static boolean isIOS(Platform platform) { - return platform == Platform.IPHONEOS || platform == Platform.IPHONESIMULATOR; - } - - static boolean isMacOS(Platform platform) { - return platform == Platform.MACOSX; - } - - public static class CreatePodSpecArgs { - public static class Builder { - private JSONObject specJson; - private File podsDir; - private File buildDir; - private PodSpec parentSpec; - private Platform selectedPlatform; - private String configuration; - private Map jobEnvContext; - private IConfigParser configParser; - - public Builder() { } - - public Builder setSpecJson(String specJson) throws ExtenderException { - this.specJson = PodSpecParser.parseJson(specJson); - return this; - } - - public Builder setPodsDir(File podsDir) { - this.podsDir = podsDir; - return this; - } - - public Builder setBuildDir(File buildDir) { - this.buildDir = buildDir; - return this; - } - - public Builder setParentSpec(PodSpec parent) { - this.parentSpec = parent; - return this; - } - - public Builder setJobContext(Map context) { - this.jobEnvContext = context; - return this; - } - - public Builder setSelectedPlatform(Platform platform) { - this.selectedPlatform = platform; - return this; - } - - public Builder setConfigParser(IConfigParser parser) { - this.configParser = parser; - return this; - } - - public Builder setConfiguration(String configuration) { - this.configuration = configuration; - return this; - } - - public CreatePodSpecArgs build() { - return new CreatePodSpecArgs(this); - } - } - - private CreatePodSpecArgs(Builder builder) { - this.specJson = builder.specJson; - this.podsDir = builder.podsDir; - this.buildDir = builder.buildDir; - this.parentSpec = builder.parentSpec; - this.jobEnvContext = builder.jobEnvContext; - this.selectedPlatform = builder.selectedPlatform; - this.configuration = builder.configuration; - this.configParser = builder.configParser; - } - - private CreatePodSpecArgs(CreatePodSpecArgs copy) { - this.specJson = copy.specJson; - this.podsDir = copy.podsDir; - this.buildDir = copy.buildDir; - this.parentSpec = copy.parentSpec; - this.jobEnvContext = copy.jobEnvContext; - this.selectedPlatform = copy.selectedPlatform; - this.configuration = copy.configuration; - this.configParser = copy.configParser; - } - - protected JSONObject specJson; - protected File podsDir; - protected File buildDir; - protected PodSpec parentSpec; - protected Platform selectedPlatform; - protected String configuration; - protected Map jobEnvContext; - protected IConfigParser configParser; - } // https://guides.cocoapods.org/syntax/podspec.html - public static PodSpec createPodSpec(CreatePodSpecArgs args) throws ExtenderException, IOException { + public static PodSpec createPodSpec(JSONObject specJson, PodUtils.Platform selectedPlatform, PodSpec parentSpec) throws ExtenderException, IOException { PodSpec spec = new PodSpec(); - JSONObject specJson = args.specJson; spec.name = (String)specJson.get("name"); - spec.moduleName = getModuleName(specJson, args.parentSpec); - spec.version = (args.parentSpec == null) ? (String)specJson.get("version") : args.parentSpec.version; - spec.dir = (args.parentSpec == null) ? new File(args.podsDir, spec.name) : args.parentSpec.dir; - spec.parentSpec = args.parentSpec; - - // generated files relating to the pod - // modulemap, swift header etc - spec.buildDir = Path.of(args.buildDir.toString(), String.format("%s%s", args.configuration, args.selectedPlatform.toString().toLowerCase()), spec.name).toFile(); - spec.buildDir.mkdirs(); - - // inherit flags and defines from the parent - if (args.parentSpec != null) { - spec.flags.addAll(args.parentSpec.flags); - spec.defines.addAll(args.parentSpec.defines); - spec.linkflags.addAll(args.parentSpec.linkflags); - spec.flags.remove("-fmodule-name=" + args.parentSpec.moduleName); - } + spec.moduleName = getModuleName(specJson, parentSpec); + spec.version = (parentSpec == null) ? (String)specJson.get("version") : parentSpec.version; + spec.parentSpec = parentSpec; // platform versions JSONObject platforms = (JSONObject)specJson.get("platforms"); if (platforms != null) { - if (isIOS(args.selectedPlatform)) { - spec.platformVersion = (String)platforms.getOrDefault("ios", args.jobEnvContext.get("env.IOS_VERSION_MIN")); - } else if (isMacOS(args.selectedPlatform)) { - spec.platformVersion = (String)platforms.getOrDefault("osx", args.jobEnvContext.get("env.MACOS_VERSION_MIN")); + if (PodUtils.isIOS(selectedPlatform)) { + spec.platformVersion = (String)platforms.getOrDefault("ios", ""); + } else if (PodUtils.isMacOS(selectedPlatform)) { + spec.platformVersion = (String)platforms.getOrDefault("osx", ""); } } // for multi platform settings - JSONObject platformSettings = (JSONObject)specJson.get(isIOS(args.selectedPlatform) ? "ios" : "osx"); + JSONObject platformSettings = (JSONObject)specJson.get(PodUtils.isIOS(selectedPlatform) ? "ios" : "osx"); // requires_arc flag // The 'requires_arc' option can also be a file pattern string or array @@ -246,10 +122,7 @@ public static PodSpec createPodSpec(CreatePodSpecArgs args) throws ExtenderExcep Iterator it = subspecs.iterator(); while (it.hasNext()) { JSONObject o = it.next(); - CreatePodSpecArgs innerSpecArgs = new CreatePodSpecArgs(args); - innerSpecArgs.specJson = o; - innerSpecArgs.parentSpec = spec; - PodSpec subSpec = createPodSpec(innerSpecArgs); + PodSpec subSpec = createPodSpec(o, selectedPlatform, spec); spec.subspecs.add(subSpec); } } @@ -262,45 +135,12 @@ public static PodSpec createPodSpec(CreatePodSpecArgs args) throws ExtenderExcep } } + // collect public headers for case when need to build framework + spec.publicHeadersPatterns.addAll(getAsList(specJson, "public_header_files")); + // find source and header files // https://guides.cocoapods.org/syntax/podspec.html#source_files - List sourceFiles = getAsList(specJson, "source_files"); - if (sourceFiles != null) { - Iterator it = sourceFiles.iterator(); - while (it.hasNext()) { - String path = it.next(); - // don't copy header (and source) files from paths in xcframeworks - // framework headers are copied in a separate step in copyPodFrameworks() - if (!path.contains(".xcframework/")) { - addPodSourceFiles(spec, path); - addPodIncludePaths(spec, path); - } - } - } - - // add swift libs to the runtime search path - if (!spec.swiftSourceFiles.isEmpty()) { - spec.linkflags.add("-Wl,-rpath,/usr/lib/swift"); - } - - // add ObjC link flag if pod contains Objective-C code - for (File sourceFile : spec.sourceFiles) { - String filename = sourceFile.getName(); - if (filename.endsWith(".m") || filename.endsWith(".mm")) { - spec.linkflags.add("-ObjC"); - break; - } - } - - // parse generated xcconfig - String configName = (args.parentSpec == null) ? spec.name : spec.parentSpec.name; - spec.parsedXCConfig = args.configParser.parse(spec.moduleName, configName, Path.of(args.podsDir.toString(), "Target Support Files", configName, String.format("%s.%s.xcconfig", configName, args.configuration.toLowerCase())).toFile()); - updateFlagsFromConfig(spec, spec.parsedXCConfig); - - // swift compatability header (just path where to store header) - if (!spec.swiftSourceFiles.isEmpty()) { - spec.swiftModuleHeader = Path.of(spec.buildDir.toString(), "SwiftCompatibilityHeader", spec.moduleName + "-Swift.h").toFile(); - } + spec.sourceFilesPatterns.addAll(getAsList(specJson, "source_files")); return spec; } @@ -342,163 +182,6 @@ static String toC99extIdentifier(String s) { .replaceAll("_+", "_"); // Foo__Bar -> Foo_Bar } - static void updateFlagsFromConfig(PodSpec spec, Map parsedConfig) { - // https://pewpewthespells.com/blog/buildsettings.html - // defines - List defs = argumentsAsList(parsedConfig.getOrDefault("GCC_PREPROCESSOR_DEFINITIONS", null)); - if (defs != null) { - spec.defines.addAll(unescapeStrings(defs)); - } - // linker flags - // https://xcodebuildsettings.com/#other_ldflags - List ldFlags = argumentsAsList(parsedConfig.getOrDefault("OTHER_LDFLAGS", null)); - if (ldFlags != null) { - spec.linkflags.addAll(ldFlags); - } - // compiler flags for c and objc files - // https://xcodebuildsettings.com/#other_cflags - List cFlags = argumentsAsList(parsedConfig.getOrDefault("OTHER_CFLAGS", null)); - if (cFlags != null) { - spec.flags.c.addAll(cFlags); - spec.flags.objc.addAll(cFlags); - } - // compiler flags - // https://developer.apple.com/documentation/xcode/build-settings-reference#C++-Language-Dialect - if (hasString(parsedConfig, "CLANG_CXX_LANGUAGE_STANDARD")) { - String cppStandard = parsedConfig.getOrDefault("CLANG_CXX_LANGUAGE_STANDARD", "compiler-default"); - String compilerFlag = ""; - switch (cppStandard) { - case "c++98": compilerFlag = "-std=c++98"; break; - case "c++11": - case "c++0x": compilerFlag = "-std=c++11"; break; - case "gnu++11": - case "gnu++0x": compilerFlag = "-std=gnu++11"; break; - case "c++14": compilerFlag = "-std=c++14"; break; - case "gnu++14": compilerFlag = "-std=gnu++17"; break; - case "c++17": compilerFlag = "-std=c++17"; break; - case "gnu++17": compilerFlag = "-std=gnu++17"; break; - case "c++20": compilerFlag = "-std=c++20"; break; - case "gnu++20": compilerFlag = "-std=gnu++20"; break; - case "gnu++98": - case "compiler-default": - default: compilerFlag = "-std=gnu++98"; break; - } - spec.flags.cpp.add(compilerFlag); - spec.flags.objcpp.add(compilerFlag); - } - if (hasString(parsedConfig, "GCC_C_LANGUAGE_STANDARD")) { - String cStandard = parsedConfig.getOrDefault("GCC_C_LANGUAGE_STANDARD", "compiler-default"); - String compilerFlag = ""; - switch (cStandard) { - case "ansi": compilerFlag = "-ansi"; break; - case "c89": compilerFlag = "-std=c89"; break; - case "gnu89": compilerFlag = "-std=gnu89"; break; - case "c99": compilerFlag = "-std=c99"; break; - case "c11": compilerFlag = "-std=c11"; break; - case "gnu11": compilerFlag = "-std=gnu11"; break; - case "gnu99": - case "compiler-default": - default: compilerFlag = "-std=gnu99"; break; - } - spec.flags.c.add(compilerFlag); - } - if (hasString(parsedConfig, "CLANG_CXX_LIBRARY")) { - String stdLib = parsedConfig.getOrDefault("CLANG_CXX_LIBRARY", "compiler-default"); - String stdLibFlag = ""; - switch (stdLib) { - case "libc++": stdLibFlag = "-stdlib=libc++"; break; - case "libstdc++": - case "compiler-default": - default: stdLibFlag = "-stdlib=libstdlibc++"; break; - } - spec.flags.cpp.add(stdLibFlag); - spec.flags.objcpp.add(stdLibFlag); - } - if (compareString(parsedConfig, "GCC_ENABLE_CPP_EXCEPTIONS", "YES")) { - spec.flags.cpp.add("-fcxx-exceptions"); - } - if (compareString(parsedConfig, "GCC_ENABLE_CPP_EXCEPTIONS", "NO")) { - spec.flags.cpp.add("-fno-cxx-exceptions"); - } - if (compareString(parsedConfig, "GCC_ENABLE_EXCEPTIONS", "YES")) { - spec.flags.add("-fexceptions"); - } - if (compareString(parsedConfig, "GCC_ENABLE_EXCEPTIONS", "NO")) { - spec.flags.add("-fno-exceptions"); - } - if (compareString(parsedConfig, "GCC_ENABLE_OBJC_EXCEPTIONS", "YES")) { - spec.flags.objc.add("-fobjc-exceptions"); - spec.flags.objcpp.add("-fobjc-exceptions"); - } - if (compareString(parsedConfig, "GCC_ENABLE_OBJC_EXCEPTIONS", "NO")) { - spec.flags.objc.add("-fno-objc-exceptions"); - spec.flags.objcpp.add("-fno-objc-exceptions"); - } - if (compareString(parsedConfig, "GCC_ENABLE_CPP_RTTI", "YES")) { - spec.flags.cpp.add("-frtti"); - } - if (compareString(parsedConfig, "GCC_ENABLE_CPP_RTTI", "NO")) { - spec.flags.cpp.add("-fno-rtti"); - } - if (compareString(parsedConfig, "GCC_ENABLE_OBJC_GC", "supported")) { - spec.flags.objc.add("-fobjc-gc"); - spec.flags.objcpp.add("-fobjc-gc"); - } - if (compareString(parsedConfig, "GCC_ENABLE_OBJC_GC", "required")) { - spec.flags.objc.add("-fobjc-gc-only"); - spec.flags.objcpp.add("-fobjc-gc-only"); - } - if (compareString(parsedConfig, "GCC_ENABLE_ASM_KEYWORD", "YES")) { - spec.flags.add("-fasm"); - } - if (compareString(parsedConfig, "GCC_ENABLE_ASM_KEYWORD", "NO")) { - spec.flags.add("-fno-asm"); - } - if (compareString(parsedConfig, "APPLICATION_EXTENSION_API_ONLY", "YES")) { - spec.flags.add("-fapplication-extension"); - spec.flags.swift.add("-application-extension"); - } - if (hasString(parsedConfig, "SWIFT_INCLUDE_PATHS")) { - List l = argumentsAsList(parsedConfig.getOrDefault("SWIFT_INCLUDE_PATHS", null)); - if (l != null) { - for (String path : l) { - spec.flags.swift.add(String.format("-I%s", path)); - spec.flags.swift.add(String.format("-Xcc -I%s", path)); - } - } - } - if (hasString(parsedConfig, "OTHER_SWIFT_FLAGS")) { - List l = argumentsAsList(parsedConfig.getOrDefault("OTHER_SWIFT_FLAGS", null)); - if (l != null) { - spec.flags.swift.addAll(l); - } - } - if (hasString(parsedConfig, "HEADER_SEARCH_PATHS")) { - List l = argumentsAsList(parsedConfig.getOrDefault("HEADER_SEARCH_PATHS", null)); - if (l != null) { - for (String path : l) { - spec.includePaths.add(new File(path)); - } - } - } - if (hasString(parsedConfig, "FRAMEWORK_SEARCH_PATHS")) { - List l = argumentsAsList(parsedConfig.getOrDefault("FRAMEWORK_SEARCH_PATHS", null)); - if (l != null) { - for (String path : l) { - spec.frameworkSearchPaths.add(new File(path)); - } - } - } - } - - static List argumentsAsList(String arguments) { - arguments = arguments != null ? arguments.trim() : null; - if (arguments == null || arguments.isEmpty()) { - return null; - } - return new ArrayList<>(Arrays.asList(arguments.split(" "))); - } - // get values as List in case if value is List or String with ' ' delimeter static List getStringListValues(JSONObject o, String key) { if (o.containsKey(key)) { @@ -531,13 +214,6 @@ static List getStringListValues(JSONObject o, String key) { return null; } - // check if a string value exists - // will return false if the value doesn't exist or is an empty string - static boolean hasString(Map parsedConfig, String key) { - String value = parsedConfig.get(key); - return value != null && !value.trim().isEmpty(); - } - // get a string value from a JSON object and split it into a list using space character as delimiter // will return an empty list if the value does not exist static List getAsSplitString(JSONObject o, String key) { @@ -548,14 +224,6 @@ static List getAsSplitString(JSONObject o, String key) { return new ArrayList<>(Arrays.asList(value.split(" "))); } - static List unescapeStrings(List strings) { - List unescapedStrings = new ArrayList<>(); - for (String s : strings) { - unescapedStrings.add(StringEscapeUtils.unescapeJava(s)); - } - return unescapedStrings; - } - // get a string value from a JSON object // will return a default value if the value doesn't exist or is an empty string static String getAsString(JSONObject o, String key, String defaultValue) { @@ -566,15 +234,6 @@ static String getAsString(JSONObject o, String key, String defaultValue) { return value; } - // check if the value for a specific key matches an expected value - static boolean compareString(Map config, String key, String expected) { - String value = config.get(key); - if (value == null || value.trim().isEmpty()) { - return false; - } - return value.equals(expected); - } - // get the value of a key as a JSONArray even it is a single value static JSONArray getAsJSONArray(JSONObject o, String key) { Object value = o.get(key); @@ -642,50 +301,4 @@ static Map> getAsMapList(JSONObject o, String key) { } return result; } - - /** - * Add source files matching a pattern to a pod - * @param pod - * @param pattern Source file pattern (glob format) - */ - static void addPodSourceFiles(PodSpec pod, String pattern) { - List podSrcFiles = PodUtils.listFilesGlob(pod.dir, pattern); - for (File podSrcFile : podSrcFiles) { - final String filename = podSrcFile.getName(); - if (filename.endsWith(".swift")) { - pod.swiftSourceFiles.add(podSrcFile); - pod.swiftSourceFilePaths.add(podSrcFile.getAbsolutePath()); - } - else { - if (!PodUtils.isHeaderFile(filename)) { - pod.sourceFiles.add(podSrcFile); - } else { - pod.headerFiles.add(podSrcFile); - } - } - } - } - - - /** - * Add a list of include paths matching a pattern to a pod - * @param pod - * @param pattern Source file pattern (glob format) - */ - static void addPodIncludePaths(PodSpec pod, String pattern) { - List podSrcFiles = PodUtils.listFilesGlob(pod.dir, pattern); - for (File podSrcFile : podSrcFiles) { - final String filename = podSrcFile.getName(); - if (PodUtils.isHeaderFile(filename)) { - File podIncludeDir = podSrcFile.getParentFile(); - if (podIncludeDir != null) { - pod.includePaths.add(podIncludeDir); - File podIncludeParentDir = podIncludeDir.getParentFile(); - if (podIncludeParentDir != null) { - pod.includePaths.add(podIncludeParentDir); - } - } - } - } - } } diff --git a/server/src/main/java/com/defold/extender/services/cocoapods/PodUtils.java b/server/src/main/java/com/defold/extender/services/cocoapods/PodUtils.java index 2a39ed24..68dc8f7e 100644 --- a/server/src/main/java/com/defold/extender/services/cocoapods/PodUtils.java +++ b/server/src/main/java/com/defold/extender/services/cocoapods/PodUtils.java @@ -4,9 +4,25 @@ import java.nio.file.Path; import java.util.List; import java.util.function.BiFunction; + +import com.defold.extender.ExtenderException; import com.defold.extender.ExtenderUtil; public class PodUtils { + public enum Platform { + IPHONEOS, + IPHONESIMULATOR, + MACOSX, + UNKNOWN + } + + static boolean isIOS(Platform platform) { + return platform == Platform.IPHONEOS || platform == Platform.IPHONESIMULATOR; + } + + static boolean isMacOS(Platform platform) { + return platform == Platform.MACOSX; + } static List parametrizedListFileGlob(File dir, String pattern, BiFunction> listFunction) { String absPathPattern = Path.of(dir.getAbsolutePath(), pattern).toString(); @@ -78,4 +94,19 @@ static String getPodName(String podlockRecord) { String podnameparts[] = PodUtils.splitPodname(PodUtils.getSpecName(podlockRecord)); return PodUtils.sanitizePodName(podnameparts[0]); } + + public static boolean hasSourceFiles(PodBuildSpec spec) { + return !spec.sourceFiles.isEmpty() || !spec.swiftSourceFiles.isEmpty(); + } + + public static String swiftModuleNameFromPlatform(String extenderTargetPlatform) throws ExtenderException { + switch(extenderTargetPlatform) { + case "arm64-ios": return "arm64-apple-ios"; + case "x86_64-ios": return "x86_64-apple-ios-simulator"; + case "arm64-macos": return "arm64-apple-macos"; + case "x86_64-macos": return "x86_64-apple-macos"; + default: + throw new ExtenderException(String.format("Invalid platform input for swift module name %s", extenderTargetPlatform)); + } + } } diff --git a/server/src/main/java/com/defold/extender/services/cocoapods/PodfileParser.java b/server/src/main/java/com/defold/extender/services/cocoapods/PodfileParser.java new file mode 100644 index 00000000..f7d51118 --- /dev/null +++ b/server/src/main/java/com/defold/extender/services/cocoapods/PodfileParser.java @@ -0,0 +1,106 @@ +package com.defold.extender.services.cocoapods; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class PodfileParser { + public static class ParseResult { + public String minVersion; + public String platform; + public boolean useFrameworks = true; + public Set podDefinitions = new HashSet<>(); + public Set podNames = new HashSet<>(); + + public ParseResult() { } + + public ParseResult(String platform, String minVersion) { + this.platform = platform; + this.minVersion = minVersion; + } + + public ParseResult mergeWith(ParseResult other) throws PodfileParsingException { + if (minVersion == null) { + minVersion = other.minVersion; + } else if (other.minVersion != null && compareVersions(minVersion, other.minVersion) < 0) { + minVersion = other.minVersion; + } + + if (platform == null) { + platform = other.platform; + } else if (other.platform != null && !platform.equals(other.platform)) { + throw new PodfileParsingException(String.format("Mismatch 'platform': %s!=%s", platform, other.platform)); + } + + useFrameworks = useFrameworks || other.useFrameworks; + + podDefinitions.addAll(other.podDefinitions); + podNames.addAll(other.podNames); + return this; + } + } + + // https://www.baeldung.com/java-comparing-versions#customSolution + static int compareVersions(String version1, String version2) { + int result = 0; + String[] parts1 = version1.split("\\."); + String[] parts2 = version2.split("\\."); + int length = Math.max(parts1.length, parts2.length); + for (int i = 0; i < length; i++) { + Integer v1 = i < parts1.length ? Integer.parseInt(parts1[i]) : 0; + Integer v2 = i < parts2.length ? Integer.parseInt(parts2[i]) : 0; + int compare = v1.compareTo(v2); + if (compare != 0) { + result = compare; + break; + } + } + return result; + } + + public static ParseResult parsePodfile(File podFile) throws IOException, PodfileParsingException { + ParseResult res = new ParseResult(); + // Load all Podfiles + Pattern podPattern = Pattern.compile("pod '([\\w|-]+)'.*"); + // Split each file into lines and go through them one by one + // Search for a Podfile platform and version configuration, examples: + // platform :ios, '9.0' + // platform :osx, '10.2' + // Get the version and figure out which is the highest version defined. This + // version will be used in the combined Podfile created by this function. + // Treat everything else as pods + List lines = Files.readAllLines(podFile.toPath()); + for (String line : lines) { + if (line.isEmpty()) { + continue; + } + if (line.startsWith("platform :")) { + if (res.platform != null) { + throw new PodfileParsingException("'platform' is already defined."); + } + if (!line.contains(":ios") && !line.contains(":osx")) { + throw new PodfileParsingException("Unsupported 'platform'"); + } + res.platform = line.contains(":ios") ? "ios" : "osx"; + String version = line.replaceFirst("platform :ios|platform :osx", "").replace(",", "").replace("'", "").trim(); + if (!version.isEmpty()) { + res.minVersion = version; + } + } else if (line.startsWith("use_frameworks!")) { + res.useFrameworks = true; + } else { + Matcher matcher = podPattern.matcher(line); + if (matcher.matches()) { + res.podDefinitions.add(line); + res.podNames.add(matcher.group(1)); + } + } + } + return res; + } +} diff --git a/server/src/main/java/com/defold/extender/services/cocoapods/PodfileParsingException.java b/server/src/main/java/com/defold/extender/services/cocoapods/PodfileParsingException.java new file mode 100644 index 00000000..6b57e82f --- /dev/null +++ b/server/src/main/java/com/defold/extender/services/cocoapods/PodfileParsingException.java @@ -0,0 +1,9 @@ +package com.defold.extender.services.cocoapods; + +import com.defold.extender.ExtenderException; + +public class PodfileParsingException extends ExtenderException { + public PodfileParsingException(String reason) { + super(reason); + } +} diff --git a/server/src/main/java/com/defold/extender/services/cocoapods/ResolvedPods.java b/server/src/main/java/com/defold/extender/services/cocoapods/ResolvedPods.java index 3e9689a0..1992dd7d 100644 --- a/server/src/main/java/com/defold/extender/services/cocoapods/ResolvedPods.java +++ b/server/src/main/java/com/defold/extender/services/cocoapods/ResolvedPods.java @@ -18,15 +18,16 @@ import com.defold.extender.ExtenderException; import com.defold.extender.ExtenderUtil; -import com.defold.extender.FrameworkUtil; import com.defold.extender.services.cocoapods.PlistBuddyWrapper.CreateBundlePlistArgs; +import com.defold.extender.utils.FrameworkUtil; public class ResolvedPods { private static final Logger LOGGER = LoggerFactory.getLogger(ResolvedPods.class); static final String FRAMEWORK_RE = "(.+)\\.framework"; - private List pods = new ArrayList<>(); + private List pods = new ArrayList<>(); private File podsDir; private File frameworksDir; + private File targetSupportFilesDir; private String platformMinVersion; private File podFileLock; private List additionIncludePaths; @@ -36,12 +37,15 @@ public class ResolvedPods { private List frameworks; private List dynamicFrameworks; private List weakFrameworks; + private boolean useFrameworks = false; - public ResolvedPods(File podsDir, File frameworksDir, List specs, File podfileLock, String minVersion) throws IOException { - platformMinVersion = minVersion; - this.podsDir = podsDir; - this.frameworksDir = frameworksDir; + public ResolvedPods(CocoaPodsServiceBuildState cocoapodsBuildState, List specs, File podfileLock, MainPodfile mainPodfile) throws IOException { + this.platformMinVersion = mainPodfile.platformMinVersion; + this.podsDir = cocoapodsBuildState.getPodsDir(); + this.targetSupportFilesDir = new File(this.podsDir, "Target Support Files"); + this.frameworksDir = cocoapodsBuildState.getUnpackedFrameworksDir(); this.podFileLock = podfileLock; + this.useFrameworks = mainPodfile.useFrameworks; setPodsSpecs(specs); } @@ -52,62 +56,61 @@ public ResolvedPods(File podsDir, File frameworksDir, List specs, File // parent // https://guides.cocoapods.org/syntax/podspec.html#subspec - void addPodLibs(PodSpec pod, Set libs) { + void addPodLibs(PodBuildSpec pod, Set libs) { libs.addAll(pod.libraries); - if (pod.parentSpec != null) addPodLibs(pod.parentSpec, libs); } List collectPodLibs() throws IOException { Set libs = new LinkedHashSet<>(); - for (PodSpec pod : pods) { + for (PodBuildSpec pod : pods) { addPodLibs(pod, libs); } - libs.addAll(ExtenderUtil.collectStaticLibsByName(frameworksDir)); + if (!useFrameworks) { + libs.addAll(ExtenderUtil.collectStaticLibsByName(frameworksDir)); + } return new ArrayList<>(libs); } - void addPodLinkFlags(PodSpec pod, Set flags) { + void addPodLinkFlags(PodBuildSpec pod, List flags) { flags.addAll(pod.linkflags); - if (pod.parentSpec != null) addPodLinkFlags(pod.parentSpec, flags); } public List getAllPodLinkFlags() { - Set flags = new LinkedHashSet<>(); - for (PodSpec pod : pods) { + List flags = new ArrayList<>(); + for (PodBuildSpec pod : pods) { addPodLinkFlags(pod, flags); } return new ArrayList(flags); } - void addPodResources(PodSpec pod, Set resources) { + public static void addPodResources(PodBuildSpec pod, Set resources) { File podDir = pod.dir; for (String resource : pod.resources) { resources.addAll(PodUtils.listFilesAndDirsGlob(podDir, resource)); } - if (pod.parentSpec != null) { - addPodResources(pod.parentSpec, resources); - } } public List getAllPodResources() { Set resources = new LinkedHashSet<>(); - for (PodSpec pod : pods) { + for (PodBuildSpec pod : pods) { addPodResources(pod, resources); } return new ArrayList(resources); } - void addPodFrameworks(PodSpec pod, Set frameworks) { + void addPodFrameworks(PodBuildSpec pod, Set frameworks) { frameworks.addAll(pod.frameworks); - if (pod.parentSpec != null) addPodFrameworks(pod.parentSpec, frameworks); } - List collectFrameworkPaths() throws IOException { + List collectFrameworkPaths() { Set frameworkPaths = new HashSet<>(); - for (PodSpec spec : pods) { + for (PodBuildSpec spec : pods) { for (File f : spec.frameworkSearchPaths) { frameworkPaths.add(f.toString()); } + if (useFrameworks) { + frameworkPaths.add(spec.buildDir.toString()); + } } return new ArrayList<>(frameworkPaths); } @@ -119,7 +122,7 @@ List collectFrameworkStaticLibPaths() throws IOException { List collectAllPodFrameworks() throws IOException { Set frameworks = new LinkedHashSet<>(); - for (PodSpec pod : pods) { + for (PodBuildSpec pod : pods) { addPodFrameworks(pod, frameworks); } @@ -160,30 +163,49 @@ List collectAllPodsDynamicFrameworks() throws IOException { return new ArrayList(dynamicFrameworks); } - void addPodWeakFrameworks(PodSpec pod, Set weakFrameworks) { + List collectAdditionalIncludePaths() { + Set includePaths = new HashSet<>(); + for (PodBuildSpec spec: pods) { + for (File f : spec.includePaths) { + includePaths.add(f.toString()); + } + includePaths.add(spec.headerMapFile.toString()); + if (spec.swiftModuleHeader != null) { + includePaths.add(spec.swiftModuleHeader.toPath().getParent().toString()); + } + } + return new ArrayList(includePaths); + } + + void addPodWeakFrameworks(PodBuildSpec pod, Set weakFrameworks) { weakFrameworks.addAll(pod.weakFrameworks); - if (pod.parentSpec != null) addPodWeakFrameworks(pod.parentSpec, weakFrameworks); } List collectPodWeakFrameworks() { Set weakFrameworks = new LinkedHashSet<>(); - for (PodSpec pod : pods) { + for (PodBuildSpec pod : pods) { addPodWeakFrameworks(pod, weakFrameworks); } return new ArrayList(weakFrameworks); } + public static List createPodResourceBundles(PodBuildSpec spec, File targetDir, String platform) throws IOException, ExtenderException { + List result = new ArrayList<>(); + for (Map.Entry> entry : spec.resourceBundles.entrySet()) { + result.add(createResourceBundle(targetDir, platform, spec, entry.getKey(), entry.getValue())); + } + return result; + } + public List createResourceBundles(File targetDir, String platform) throws IOException, ExtenderException { List result = new ArrayList<>(); - for (PodSpec spec : pods) { - for (Map.Entry> entry : spec.resourceBundles.entrySet()) { - result.add(createResourceBundle(targetDir, platform, spec, entry.getKey(), entry.getValue())); - } + for (PodBuildSpec spec : pods) { + result.addAll(createPodResourceBundles(spec, targetDir, platform)); } return result; } - static File createResourceBundle(File targetDir, String platform, PodSpec pod, String bundleName, List content) throws IOException, ExtenderException { + static File createResourceBundle(File targetDir, String platform, PodBuildSpec pod, String bundleName, List content) throws IOException, ExtenderException { File resultFolder = new File(targetDir, bundleName + ".bundle"); resultFolder.mkdirs(); for (String contentElement : content) { @@ -205,20 +227,34 @@ static File createResourceBundle(File targetDir, String platform, PodSpec pod, S return resultFolder; } - public void setPodsSpecs(List specs) throws IOException { - pods = specs; + public void setPodsSpecs(List specs) throws IOException { + pods.addAll(specs); frameworkSearchPaths = collectFrameworkPaths(); librarySearchPaths = collectFrameworkStaticLibPaths(); staticLibraries = collectPodLibs(); frameworks = collectAllPodFrameworks(); dynamicFrameworks = collectAllPodsDynamicFrameworks(); weakFrameworks = collectPodWeakFrameworks(); + additionIncludePaths = collectAdditionalIncludePaths(); } public List getFrameworks() { return frameworks; } + public Set getBuiltFrameworks() { + if (!useFrameworks) { + return Set.of(); + } + Set result = new HashSet<>(); + for (PodBuildSpec spec : pods) { + if (PodUtils.hasSourceFiles(spec)) { + result.add(spec.moduleName); + } + } + return result; + } + public List getWeakFrameworks() { return weakFrameworks; } @@ -239,7 +275,7 @@ public List getStaticLibraries() { return staticLibraries; } - public List getPodSpecs() { + public List getPodSpecs() { return pods; } @@ -259,11 +295,19 @@ public File getCurrentPodsDirectory() { return podsDir; } + public boolean useFrameworks() { + return useFrameworks; + } + @Deprecated public List getPodsPrivacyManifests() { return ExtenderUtil.listFilesMatchingRecursive(podsDir, "PrivacyInfo.xcprivacy"); } + public File getTargetSupportFilesDir() { + return targetSupportFilesDir; + } + @Override public String toString() { StringBuilder sb = new StringBuilder(); diff --git a/server/src/main/java/com/defold/extender/services/cocoapods/XCConfigParser.java b/server/src/main/java/com/defold/extender/services/cocoapods/XCConfigParser.java index bb187b69..60af93b7 100644 --- a/server/src/main/java/com/defold/extender/services/cocoapods/XCConfigParser.java +++ b/server/src/main/java/com/defold/extender/services/cocoapods/XCConfigParser.java @@ -17,6 +17,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.defold.extender.ExtenderBuildState; + public class XCConfigParser implements IConfigParser { private static final Logger LOGGER = LoggerFactory.getLogger(XCConfigParser.class); private File buildDir; @@ -34,12 +36,17 @@ enum ParseMode { VALUE } - public XCConfigParser(File buildDir, File podsDir, String platform, String configuration, String arch) { + public XCConfigParser(ExtenderBuildState buildState, CocoaPodsServiceBuildState cocoapodsBuildState) { + this(buildState.getBuildDir(), cocoapodsBuildState.getPodsDir(), cocoapodsBuildState.getSelectedPlatform(), buildState.getBuildConfiguration(), buildState.getBuildArch()); + } + + XCConfigParser(File buildDir, File podsDir, PodUtils.Platform selectedPlatform, String buildConfiguration, String buildArch) { this.buildDir = buildDir; this.podsDir = podsDir; - this.platform = platform; - this.configuration = configuration; - this.arch = arch; + this.platform = selectedPlatform.toString().toLowerCase(); + + this.configuration = buildConfiguration; + this.arch = buildArch; } Map calculateBaseVariables(String moduleName, String podName) { diff --git a/server/src/main/java/com/defold/extender/FrameworkUtil.java b/server/src/main/java/com/defold/extender/utils/FrameworkUtil.java similarity index 93% rename from server/src/main/java/com/defold/extender/FrameworkUtil.java rename to server/src/main/java/com/defold/extender/utils/FrameworkUtil.java index 93af11be..e1f58ed0 100644 --- a/server/src/main/java/com/defold/extender/FrameworkUtil.java +++ b/server/src/main/java/com/defold/extender/utils/FrameworkUtil.java @@ -1,8 +1,9 @@ -package com.defold.extender; +package com.defold.extender.utils; import java.io.File; import java.util.List; +import com.defold.extender.ExtenderException; import com.defold.extender.process.ProcessUtils; public class FrameworkUtil { diff --git a/server/src/main/java/com/defold/extender/utils/PodBuildUtil.java b/server/src/main/java/com/defold/extender/utils/PodBuildUtil.java new file mode 100644 index 00000000..21efb654 --- /dev/null +++ b/server/src/main/java/com/defold/extender/utils/PodBuildUtil.java @@ -0,0 +1,113 @@ +package com.defold.extender.utils; + +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.StandardOpenOption; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +import org.apache.commons.text.StringSubstitutor; +import org.json.simple.JSONArray; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; + +import com.defold.extender.ExtenderException; +import com.defold.extender.process.ProcessUtils; +import com.defold.extender.services.cocoapods.PodBuildSpec; + +public class PodBuildUtil { + public static void putFileNameIntoVFS(Map> vfsMap, String section, File file) { + if (!vfsMap.containsKey(section)) { + vfsMap.put(section, new HashSet<>()); + } + vfsMap.get(section).add(file); + } + + public static File generateHeaderMap(PodBuildSpec spec) throws IOException, ExtenderException { + JSONObject root = new JSONObject(); + String moduleName = spec.moduleName; + for (File header : spec.privateHeaders) { + String filename = header.getName(); + String directory = header.getParent(); + Map data = Map.of("suffix", filename, "prefix", String.format("%s/", directory)); + root.putAll(Map.of(filename, data)); + root.putAll(Map.of(String.format("%s/%s", moduleName, filename), data)); + } + for (File header : spec.publicHeaders) { + String filename = header.getName(); + Map data = Map.of("suffix", filename, "prefix", String.format("%s/", moduleName)); + root.putAll(Map.of(filename, data)); + } + String serialized = root.toJSONString(); + File jsonHeaderMap = new File(spec.headerMapFile.getParentFile(), String.format("%s.json", spec.name)); + Files.writeString(jsonHeaderMap.toPath(), serialized, StandardCharsets.UTF_8); + ProcessUtils.execCommand(List.of( + "hmap", + "convert", + jsonHeaderMap.toString(), + spec.headerMapFile.toString() + ), null, Map.of()); + return spec.headerMapFile; + } + + public static File generateVFSOverlay(PodBuildSpec spec, Map> vfsInfo) throws IOException { + JSONArray rootArray = new JSONArray(); + + for (Map.Entry> entry : vfsInfo.entrySet()) { + JSONArray content = new JSONArray(); + for (File contentPath : entry.getValue()) { + content.add(Map.of( + "external-contents", contentPath.toString(), + "name", contentPath.getName(), + "type", "file" + )); + } + rootArray.add(Map.of( + "contents", content, + "name", entry.getKey(), + "type", "directory" + )); + } + JSONObject resultDocument = new JSONObject(); + resultDocument.put("roots", rootArray); + resultDocument.put("case-sensitive", "false"); // false as string, not boolean + resultDocument.put("version", 0); + + Files.writeString(spec.vfsOverlay.toPath(), resultDocument.toJSONString(), StandardCharsets.UTF_8); + + // 2. Get all dependencies from spec + for (PodBuildSpec depSpec : spec.dependantSpecs) { + mergeVFSOverlays(spec.vfsOverlay, depSpec.vfsOverlay); + } + + return spec.vfsOverlay; + } + + public static File mergeVFSOverlays(File overlayA, File overlayB) { + JSONParser parser = new JSONParser(); + try { + JSONObject parsedOverlayA = (JSONObject)parser.parse(new FileReader(overlayA)); + JSONObject parsedOverlayB = (JSONObject)parser.parse(new FileReader(overlayB)); + + JSONArray roots = (JSONArray)parsedOverlayA.get("roots"); + roots.addAll((JSONArray)parsedOverlayB.get("roots")); + + Files.writeString(overlayA.toPath(), parsedOverlayA.toJSONString(), StandardCharsets.UTF_8); + } catch (IOException | ParseException e) { + + } + return overlayA; + } + + public static void generatedInfoPlistFromTemplate(File sourceTemplate, Map data, File targetFile) throws IOException { + StringSubstitutor substitutor = new StringSubstitutor(data); + String template = Files.readString(sourceTemplate.toPath(), StandardCharsets.UTF_8); + Files.writeString(targetFile.toPath(), substitutor.replace(template), StandardOpenOption.CREATE, StandardOpenOption.WRITE); + } +} diff --git a/server/src/main/resources/template.podfile b/server/src/main/resources/template.podfile index 9dbb1748..54317235 100644 --- a/server/src/main/resources/template.podfile +++ b/server/src/main/resources/template.podfile @@ -1,6 +1,6 @@ platform :{{PLATFORM}}, '{{PLATFORM_VERSION}}' install! 'cocoapods', integrate_targets: false, skip_pods_project_generation: false -#use_frameworks! +{{#USE_FRAMEWORKS}}use_frameworks!{{/USE_FRAMEWORKS}} {{#PODS}} {{{.}}} {{/PODS}} diff --git a/server/src/test/java/com/defold/extender/ExtenderTest.java b/server/src/test/java/com/defold/extender/ExtenderTest.java index 3d465802..beafe880 100644 --- a/server/src/test/java/com/defold/extender/ExtenderTest.java +++ b/server/src/test/java/com/defold/extender/ExtenderTest.java @@ -462,7 +462,7 @@ static File setupJobFolder(String variant) throws IOException Map platforms = new HashMap<>(); map.put("context", context); map.put("platforms", platforms); - context.put(Extender.APPMANIFEST_BASE_VARIANT_KEYWORD, variant); + context.put(ExtenderBuildState.APPMANIFEST_BASE_VARIANT_KEYWORD, variant); writeYaml(new File(appDir, Extender.APPMANIFEST_FILENAME), map); return jobDir; diff --git a/server/src/test/java/com/defold/extender/ExtenderUtilTest.java b/server/src/test/java/com/defold/extender/ExtenderUtilTest.java index 6421e901..1e37a605 100644 --- a/server/src/test/java/com/defold/extender/ExtenderUtilTest.java +++ b/server/src/test/java/com/defold/extender/ExtenderUtilTest.java @@ -301,6 +301,7 @@ public void testSHA256Checksum() throws NoSuchAlgorithmException, FileNotFoundEx @Test public void testWriteSourceListToTempFile() throws IOException { + File tmpDir = Files.createTempDirectory("test-source-list").toFile(); Set sources = Set.of("/path/path1/path2/path3/source1.swift", "/path/path4/path2/path3/source2.swift", "/path/path1 space/path8/path3 space/source3.swift", @@ -312,7 +313,7 @@ public void testWriteSourceListToTempFile() throws IOException { "/path/path1\\ space/path8/path3\\ space/source3.swift", "/path/path4/path2/path/source4\\ space.swift", "/path/path6/path7\\ space/path3/source5.swift"); - File resFile = ExtenderUtil.writeSourceFilesListToTmpFile(sources); + File resFile = ExtenderUtil.writeSourceFilesListToTmpFile(tmpDir, sources); assertTrue(resFile.exists()); List writtenLines = FileUtils.readLines(resFile, StandardCharsets.UTF_8); assertEquals(expected.size(), writtenLines.size()); diff --git a/server/src/test/java/com/defold/extender/IntegrationTest.java b/server/src/test/java/com/defold/extender/IntegrationTest.java index e1cf49d7..f46d6d72 100644 --- a/server/src/test/java/com/defold/extender/IntegrationTest.java +++ b/server/src/test/java/com/defold/extender/IntegrationTest.java @@ -222,7 +222,7 @@ private String getLibName(String platform, String lib) { private String getDynamicLibName(String platform, String lib) { if (platform.endsWith("win32")) { return String.format("%s.dll", lib); - } else if (platform.endsWith("osx") || platform.endsWith("ios")) { + } else if (ExtenderUtil.isAppleTarget(platform)) { return String.format("%s.dylib", lib); } return String.format("lib%s.so", lib); diff --git a/server/src/test/java/com/defold/extender/services/cocoapods/CocoaPodsServiceTest.java b/server/src/test/java/com/defold/extender/services/cocoapods/CocoaPodsServiceTest.java deleted file mode 100644 index 9b3b7898..00000000 --- a/server/src/test/java/com/defold/extender/services/cocoapods/CocoaPodsServiceTest.java +++ /dev/null @@ -1,434 +0,0 @@ -package com.defold.extender.services.cocoapods; - -import static org.junit.jupiter.api.Assertions.assertArrayEquals; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardCopyOption; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Stream; - -import javax.naming.InvalidNameException; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInfo; -import org.junit.jupiter.api.condition.EnabledOnOs; -import org.junit.jupiter.api.condition.OS; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; - -import com.defold.extender.ExtenderException; -import com.defold.extender.TestUtils; - - -public class CocoaPodsServiceTest { - private static final String INHERITED_VALUE = "$(inherited)"; - private File emptyPodfile; - private File withCommentPodfile; - private File regularPodfile; - private File wrongPodfile; - - private Map jobContext; - - private File workingDir; - private File buildDir; - private File podsDir; - private File frameworksDir; - - class EmptyConfigParser implements IConfigParser { - @Override - public Map parse(String moduleName, String podName, File xcconfig) throws IOException { - return Map.of(); - } - - } - - private static Stream specData() { - return Stream.of( - Arguments.of("PNChartboostSDKAdapter", Path.of("test-data/pod_specs/PNChartboostSDKAdapter.json")), - Arguments.of("UnityAds", Path.of("test-data/pod_specs/UnityAds.json")), - Arguments.of("Cuckoo", Path.of("test-data/pod_specs/Cuckoo.json")), - Arguments.of("Rtc555Sdk", Path.of("test-data/pod_specs/Rtc555Sdk.json")), - Arguments.of("streethawk", Path.of("test-data/pod_specs/streethawk.json")), - Arguments.of("AXPracticalHUD", "test-data/pod_specs/AXPracticalHUD.json"), - Arguments.of("PubNub", "test-data/pod_specs/PubNub.json"), - Arguments.of("TPNiOS", "test-data/pod_specs/TPNiOS.json"), - Arguments.of("Wilddog", "test-data/pod_specs/Wilddog.json") - ); - } - - private static File createEmptyFiles(File podsDir, String podName, String[] filesToCreate) throws IOException { - File resultFolder = new File(podsDir, podName); - for (String f : filesToCreate) { - File emptyF = new File(resultFolder, f); - emptyF.getParentFile().mkdirs(); - emptyF.createNewFile(); - } - return resultFolder; - } - - @BeforeEach - public void setUp(TestInfo testInfo) throws IOException, InvalidNameException { - this.emptyPodfile = new File("test-data/podfiles/empty.Podfile"); - this.regularPodfile = new File("test-data/podfiles/regular.Podfile"); - this.wrongPodfile = new File("test-data/podfiles/wrong.Podfile"); - this.withCommentPodfile = new File("test-data/podfiles/with_comments.Podfile"); - - this.jobContext = new HashMap<>(); - this.jobContext.putAll(TestUtils.envFileToMap(new File("envs/.env"))); - this.jobContext.putAll(TestUtils.envFileToMap(new File("envs/macos.env"))); - - if (testInfo.getDisplayName().contains(" ")) { - // display name is used as folder prefix so if it contains spaces - ask to remove spaces )) - throw new InvalidNameException("Test's display name shouldn't contain spaces"); - } - this.workingDir = Files.createTempDirectory(testInfo.getDisplayName()).toFile(); - this.buildDir = new File(this.workingDir, "build"); - this.buildDir.mkdir(); - this.podsDir = new File(this.workingDir, "pods"); - this.podsDir.mkdir(); - this.frameworksDir = Path.of(this.buildDir.toString(), "Debugiphoneos", "XCFrameworkIntermediates").toFile(); - this.frameworksDir.mkdirs(); - this.workingDir.deleteOnExit(); - } - - @Test - public void testPodfileParserRegularSyntax() throws IOException { - MainPodfile mainPodfile = CocoaPodsService.createMainPodfile(); - CocoaPodsService.parsePodfiles(mainPodfile, List.of(this.regularPodfile)); - String[] expected = {"IronSourcePangleAdapter", "IronSourceSmaatoAdapter", "IronSourceSuperAwesomeAdapter", "IronSourceTencentAdapter", "IronSourceUnityAdsAdapter", "IronSourceYandexAdapter"}; - assertArrayEquals(expected, mainPodfile.podnames.toArray()); - } - - @Test - public void testPodfileParserComments() throws IOException { - MainPodfile mainPodfile = CocoaPodsService.createMainPodfile(); - mainPodfile.platformMinVersion = "10.3"; - CocoaPodsService.parsePodfiles(mainPodfile, List.of(this.withCommentPodfile)); - assertEquals(mainPodfile.platformMinVersion, "12.0"); - String[] expected = {"OneTrust-CMP-XCFramework"}; - assertArrayEquals(expected, mainPodfile.podnames.toArray()); - } - - @Test - public void testPodfileParserWrongSyntax() throws IOException { - MainPodfile mainPodfile = CocoaPodsService.createMainPodfile(); - CocoaPodsService.parsePodfiles(mainPodfile, List.of(this.wrongPodfile)); - String[] expected = {"IronSourceSuperAwesomeAdapter"}; - assertArrayEquals(expected, mainPodfile.podnames.toArray()); - } - - @Test - public void testPodfileParserEmptyFile() throws IOException { - MainPodfile mainPodfile = CocoaPodsService.createMainPodfile(); - CocoaPodsService.parsePodfiles(mainPodfile, List.of(this.emptyPodfile)); - assertTrue(mainPodfile.podnames.isEmpty()); - } - - @Test - public void testPodfileParserAll() throws IOException { - MainPodfile mainPodfile = CocoaPodsService.createMainPodfile(); - mainPodfile.platformMinVersion = "9.1"; - CocoaPodsService.parsePodfiles(mainPodfile, List.of(this.emptyPodfile, this.wrongPodfile, this.regularPodfile, this.withCommentPodfile)); - String[] expected = {"IronSourceSuperAwesomeAdapter", "IronSourcePangleAdapter", "IronSourceSmaatoAdapter", "IronSourceSuperAwesomeAdapter", "IronSourceTencentAdapter", "IronSourceUnityAdsAdapter", "IronSourceYandexAdapter", "OneTrust-CMP-XCFramework"}; - assertArrayEquals(expected, mainPodfile.podnames.toArray()); - } - - @ParameterizedTest(name = "{index}_testParsePodSpecs_{0}") - @MethodSource("specData") - public void testParsePodSpecs(String alias, Path spec) throws IOException, ExtenderException { - String jsonSpec = Files.readString(spec); - PodSpecParser.CreatePodSpecArgs args = new PodSpecParser.CreatePodSpecArgs.Builder() - .setBuildDir(this.buildDir) - .setPodsDir(this.podsDir) - .setSpecJson(jsonSpec) - .setJobContext(this.jobContext) - .setSelectedPlatform(PodSpecParser.Platform.IPHONEOS) - .setConfiguration("Debug") - .setConfigParser(new EmptyConfigParser()) - .build(); - assertDoesNotThrow(() -> PodSpecParser.createPodSpec(args)); - } - - @ParameterizedTest(name = "{index}_testPodSpecsNoInherited_{0}") - @MethodSource("specData") - public void testPodSpecsNoInherited(String alias, Path spec) throws IOException, ExtenderException { - String jsonSpec = Files.readString(spec); - PodSpecParser.CreatePodSpecArgs args = new PodSpecParser.CreatePodSpecArgs.Builder() - .setBuildDir(this.buildDir) - .setPodsDir(this.podsDir) - .setSpecJson(jsonSpec) - .setJobContext(this.jobContext) - .setSelectedPlatform(PodSpecParser.Platform.IPHONEOS) - .setConfiguration("Debug") - .setConfigParser(new EmptyConfigParser()) - .build(); - PodSpec podSpec = PodSpecParser.createPodSpec(args); - assertFalse(podSpec.linkflags.contains(INHERITED_VALUE)); - assertFalse(podSpec.defines.contains(INHERITED_VALUE)); - assertFalse(podSpec.flags.c.contains(INHERITED_VALUE)); - assertFalse(podSpec.flags.cpp.contains(INHERITED_VALUE)); - assertFalse(podSpec.flags.objc.contains(INHERITED_VALUE)); - assertFalse(podSpec.flags.objcpp.contains(INHERITED_VALUE)); - - for (PodSpec subspec : podSpec.subspecs) { - assertFalse(subspec.linkflags.contains(INHERITED_VALUE)); - assertFalse(subspec.defines.contains(INHERITED_VALUE)); - assertFalse(subspec.flags.c.contains(INHERITED_VALUE)); - assertFalse(subspec.flags.cpp.contains(INHERITED_VALUE)); - assertFalse(subspec.flags.objc.contains(INHERITED_VALUE)); - assertFalse(subspec.flags.objcpp.contains(INHERITED_VALUE)); - } - } - - @Test - public void testSpecBraceExpanderVendoredFrameworks() throws ExtenderException, IOException { - String jsonSpec = Files.readString(Path.of("test-data/pod_specs/TPNiOS.json")); - PodSpecParser.CreatePodSpecArgs args = new PodSpecParser.CreatePodSpecArgs.Builder() - .setBuildDir(this.buildDir) - .setPodsDir(this.podsDir) - .setSpecJson(jsonSpec) - .setJobContext(this.jobContext) - .setSelectedPlatform(PodSpecParser.Platform.IPHONEOS) - .setConfiguration("Debug") - .setConfigParser(new EmptyConfigParser()) - .build(); - PodSpec podSpec = PodSpecParser.createPodSpec(args); - String[] expectedValues = new String[]{ - "core/AnyThinkBanner.xcframework", - "core/AnyThinkSplash.xcframework", - "core/AnyThinkRewardedVideo.xcframework", - "core/AnyThinkInterstitial.xcframework", - "core/AnyThinkNative.xcframework", - "core/AnyThinkMediaVideo.xcframework", - "core/AnyThinkSDK.xcframework" - }; - assertArrayEquals(expectedValues, podSpec.subspecs.get(0).vendoredFrameworks.toArray()); - } - - @Test - public void testSpecBraceExpanderSourceFiles() throws ExtenderException, IOException { - String jsonSpec = Files.readString(Path.of("test-data/pod_specs/PubNub.json")); - PodSpecParser.CreatePodSpecArgs args = new PodSpecParser.CreatePodSpecArgs.Builder() - .setBuildDir(this.buildDir) - .setPodsDir(this.podsDir) - .setSpecJson(jsonSpec) - .setJobContext(this.jobContext) - .setSelectedPlatform(PodSpecParser.Platform.IPHONEOS) - .setConfiguration("Debug") - .setConfigParser(new EmptyConfigParser()) - .build(); - // create empty files to simulate sources - String[] simulatedFiles = new String[]{ - "PubNub/Core/core1/core11.cpp", - "PubNub/Core/core1/core12.cpp", - "PubNub/Core/core1/core1.h", - "PubNub/Core/core2/core21.cpp", - "PubNub/Core/core2/core22.cpp", - "PubNub/Core/core2/core2.h", - - "PubNub/Data/data1/data11.cpp", - "PubNub/Data/data1/data12.cpp", - "PubNub/Data/data1/data1.h", - "PubNub/Data/data2/data21.cpp", - "PubNub/Data/data2/data22.cpp", - "PubNub/Data/data2/data2.h", - - "PubNub/Misc/misc1/misc11.cpp", - "PubNub/Misc/misc1/misc12.cpp", - "PubNub/Misc/misc1/misc1.h", - "PubNub/Misc/misc2/misc21.cpp", - "PubNub/Misc/misc2/misc22.cpp", - "PubNub/Misc/misc2/misc2.h", - - "PubNub/Network/network1/network11.cpp", - "PubNub/Network/network1/network12.cpp", - "PubNub/Network/network1/network1.h", - "PubNub/Network/network2/network21.cpp", - "PubNub/Network/network2/network22.cpp", - "PubNub/Network/network2/network2.h" - }; - File pubNubFolder = createEmptyFiles(this.podsDir, "PubNub", simulatedFiles); - - PodSpec podSpec = PodSpecParser.createPodSpec(args); - File[] expectedValues = new File[]{ - new File(pubNubFolder, "PubNub/Core/core1/core11.cpp"), - new File(pubNubFolder, "PubNub/Core/core1/core12.cpp"), - new File(pubNubFolder, "PubNub/Core/core2/core21.cpp"), - new File(pubNubFolder, "PubNub/Core/core2/core22.cpp"), - - new File(pubNubFolder, "PubNub/Data/data1/data11.cpp"), - new File(pubNubFolder, "PubNub/Data/data1/data12.cpp"), - new File(pubNubFolder, "PubNub/Data/data2/data21.cpp"), - new File(pubNubFolder, "PubNub/Data/data2/data22.cpp"), - - new File(pubNubFolder, "PubNub/Misc/misc1/misc11.cpp"), - new File(pubNubFolder, "PubNub/Misc/misc1/misc12.cpp"), - new File(pubNubFolder, "PubNub/Misc/misc2/misc21.cpp"), - new File(pubNubFolder, "PubNub/Misc/misc2/misc22.cpp"), - - new File(pubNubFolder, "PubNub/Network/network1/network11.cpp"), - new File(pubNubFolder, "PubNub/Network/network1/network12.cpp"), - new File(pubNubFolder, "PubNub/Network/network2/network21.cpp"), - new File(pubNubFolder, "PubNub/Network/network2/network22.cpp") - }; - assertArrayEquals(expectedValues, podSpec.subspecs.get(0).sourceFiles.toArray()); - } - - @Test - public void testDefaultSubspecs() throws ExtenderException, IOException { - String jsonSpec = Files.readString(Path.of("test-data/pod_specs/Wilddog.json")); - PodSpecParser.CreatePodSpecArgs args = new PodSpecParser.CreatePodSpecArgs.Builder() - .setBuildDir(this.buildDir) - .setPodsDir(this.podsDir) - .setSpecJson(jsonSpec) - .setJobContext(this.jobContext) - .setSelectedPlatform(PodSpecParser.Platform.IPHONEOS) - .setConfiguration("Debug") - .setConfigParser(new EmptyConfigParser()) - .build(); - PodSpec podSpec = PodSpecParser.createPodSpec(args); - String[] expectedValues = new String[]{ - "Public", - "Sync", - "Auth", - "Core" - }; - assertArrayEquals(expectedValues, podSpec.defaultSubspecs.toArray()); - } - - @Test - public void testPodResourceFromBundle() throws IOException, ExtenderException { - String jsonSpec = Files.readString(Path.of("test-data/pod_specs/AXPracticalHUD.json")); - PodSpecParser.CreatePodSpecArgs args = new PodSpecParser.CreatePodSpecArgs.Builder() - .setBuildDir(this.buildDir) - .setPodsDir(this.podsDir) - .setSpecJson(jsonSpec) - .setJobContext(this.jobContext) - .setSelectedPlatform(PodSpecParser.Platform.IPHONEOS) - .setConfiguration("Debug") - .setConfigParser(new EmptyConfigParser()) - .build(); - String[] simulatedFiles = new String[]{ - "AXPracticalHUD/AXPracticalHUD/AXPracticalHUD.bundle/ax_hud_error@2x.png", - "AXPracticalHUD/AXPracticalHUD/AXPracticalHUD.bundle/ax_hud_error@3x.png", - "AXPracticalHUD/AXPracticalHUD/AXPracticalHUD.bundle/ax_hud_success@2x.png", - "AXPracticalHUD/AXPracticalHUD/AXPracticalHUD.bundle/ax_hud_success@3x.png" - }; - File podFolder = createEmptyFiles(this.podsDir, "AXPracticalHUD", simulatedFiles); - File[] expectedFiles = new File[]{ - new File(podFolder, "AXPracticalHUD/AXPracticalHUD/AXPracticalHUD.bundle/ax_hud_error@2x.png"), - new File(podFolder, "AXPracticalHUD/AXPracticalHUD/AXPracticalHUD.bundle/ax_hud_error@3x.png"), - new File(podFolder, "AXPracticalHUD/AXPracticalHUD/AXPracticalHUD.bundle/ax_hud_success@2x.png"), - new File(podFolder, "AXPracticalHUD/AXPracticalHUD/AXPracticalHUD.bundle/ax_hud_success@3x.png") - }; - PodSpec podSpec = PodSpecParser.createPodSpec(args); - ResolvedPods resolvedPods = new ResolvedPods(this.podsDir, this.frameworksDir, List.of(podSpec), new File(this.buildDir, "Podfile.lock"), "11.0"); - - List result = resolvedPods.getAllPodResources(); - assertArrayEquals(expectedFiles, result.toArray()); - } - - @Test - @EnabledOnOs({ OS.MAC }) - public void testResourceBundleParsing() throws IOException, ExtenderException { - String jsonSpec = Files.readString(Path.of("test-data/pod_specs/UnityAds.json")); - PodSpecParser.CreatePodSpecArgs args = new PodSpecParser.CreatePodSpecArgs.Builder() - .setBuildDir(this.buildDir) - .setPodsDir(this.podsDir) - .setSpecJson(jsonSpec) - .setJobContext(this.jobContext) - .setSelectedPlatform(PodSpecParser.Platform.IPHONEOS) - .setConfiguration("Debug") - .setConfigParser(new EmptyConfigParser()) - .build(); - - String[] simulatedFiles = new String[]{ - "UnityAds.xcframework/ios-arm64/UnityAds.framework/PrivacyInfo.xcprivacy", - "UnityAds.xcframework/ios-arm64/UnityAds.framework/omid-session-client-v1.js", - "UnityAds.xcframework/ios-arm64/UnityAds.framework/omsdk-v1.js", - "UnityAds.xcframework/ios-arm64/UnityAds.framework/Info.plist" - }; - File podFolder = createEmptyFiles(this.podsDir, "UnityAds", simulatedFiles); - - PodSpec unityAdsSpec = PodSpecParser.createPodSpec(args); - ResolvedPods resolvedPods = new ResolvedPods(this.podsDir, this.frameworksDir, List.of(unityAdsSpec), new File(this.buildDir, "Podfile.lock"), "11.0"); - - File targetDir = new File(this.workingDir, "result"); - List result = resolvedPods.createResourceBundles(targetDir, "arm64-ios"); - assertEquals(result.size(), 1); - assertEquals("UnityAdsResources.bundle", result.get(0).getName()); - List expectedFiles = List.of( - "PrivacyInfo.xcprivacy", - "omid-session-client-v1.js", - "omsdk-v1.js", - "Info.plist" - ); - - File bundleDir = new File(targetDir, "UnityAdsResources.bundle"); - assertTrue(bundleDir.exists()); - List resultContent = Files.list(bundleDir.toPath()).toList(); - assertEquals(expectedFiles.size(), resultContent.size()); - for (Path p : resultContent) { - assertTrue(expectedFiles.contains(p.getFileName().toString())); - } - } - - @Test - public void testValueSubstitution() throws IOException, ExtenderException { - String jsonSpec = Files.readString(Path.of("test-data/pod_specs/Sentry.json")); - XCConfigParser parser = new XCConfigParser(this.buildDir, this.podsDir, PodSpecParser.Platform.IPHONEOS.toString().toLowerCase(), "Debug", "arm64"); - File sentryTargetFolder = Path.of(this.podsDir.toString(), "Target Support Files", "Sentry").toFile(); - sentryTargetFolder.mkdirs(); - Files.copy(Path.of("test-data/xcconfigs/Sentry.xcconfig"), Path.of(sentryTargetFolder.toString(), "Sentry.debug.xcconfig"), StandardCopyOption.REPLACE_EXISTING); - - PodSpecParser.CreatePodSpecArgs args = new PodSpecParser.CreatePodSpecArgs.Builder() - .setBuildDir(this.buildDir) - .setPodsDir(this.podsDir) - .setSpecJson(jsonSpec) - .setJobContext(this.jobContext) - .setSelectedPlatform(PodSpecParser.Platform.IPHONEOS) - .setConfiguration("Debug") - .setConfigParser(parser) - .build(); - PodSpec podSpec = PodSpecParser.createPodSpec(args); - assertTrue(podSpec.flags.c.contains("-DAPPLICATION_EXTENSION_API_ONLY_YES")); - assertTrue(podSpec.flags.objc.contains("-DAPPLICATION_EXTENSION_API_ONLY_YES")); - } - - @Test - public void testCompilationFlags() throws IOException, ExtenderException { - String jsonSpec = Files.readString(Path.of("test-data/pod_specs/Sentry.json")); - XCConfigParser parser = new XCConfigParser(this.buildDir, this.podsDir, PodSpecParser.Platform.IPHONEOS.toString().toLowerCase(), "Debug", "arm64"); - File sentryTargetFolder = Path.of(this.podsDir.toString(), "Target Support Files", "Sentry").toFile(); - sentryTargetFolder.mkdirs(); - Files.copy(Path.of("test-data/xcconfigs/Sentry.xcconfig"), Path.of(sentryTargetFolder.toString(), "Sentry.debug.xcconfig"), StandardCopyOption.REPLACE_EXISTING); - - PodSpecParser.CreatePodSpecArgs args = new PodSpecParser.CreatePodSpecArgs.Builder() - .setBuildDir(this.buildDir) - .setPodsDir(this.podsDir) - .setSpecJson(jsonSpec) - .setJobContext(this.jobContext) - .setSelectedPlatform(PodSpecParser.Platform.IPHONEOS) - .setConfiguration("Debug") - .setConfigParser(parser) - .build(); - PodSpec podSpec = PodSpecParser.createPodSpec(args); - // check handle APPLICATION_EXTENSION_API_ONLY flag - assertTrue(podSpec.flags.c.contains("-fapplication-extension")); - assertTrue(podSpec.flags.objc.contains("-fapplication-extension")); - assertTrue(podSpec.flags.swift.contains("-application-extension")); - // check SWIFT_INCLUDE_PATHS - assertTrue(podSpec.flags.swift.contains(String.format("-I%s/Sentry/Sources/Sentry/include", this.podsDir.toString()))); - } -} diff --git a/server/src/test/java/com/defold/extender/services/cocoapods/PodBuildSpecTest.java b/server/src/test/java/com/defold/extender/services/cocoapods/PodBuildSpecTest.java new file mode 100644 index 00000000..63284ac5 --- /dev/null +++ b/server/src/test/java/com/defold/extender/services/cocoapods/PodBuildSpecTest.java @@ -0,0 +1,105 @@ +package com.defold.extender.services.cocoapods; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.HashMap; +import java.util.Map; + +import javax.naming.InvalidNameException; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; + +import com.defold.extender.ExtenderException; +import com.defold.extender.TestUtils; + +public class PodBuildSpecTest { + + private Map jobContext; + private File workingDir; + private File buildDir; + private File podsDir; + + @BeforeEach + public void setUp(TestInfo testInfo) throws IOException, InvalidNameException { + this.jobContext = new HashMap<>(); + this.jobContext.putAll(TestUtils.envFileToMap(new File("envs/.env"))); + this.jobContext.putAll(TestUtils.envFileToMap(new File("envs/macos.env"))); + + if (testInfo.getDisplayName().contains(" ")) { + // display name is used as folder prefix so if it contains spaces - ask to remove spaces )) + throw new InvalidNameException("Test's display name shouldn't contain spaces"); + } + this.workingDir = Files.createTempDirectory(testInfo.getDisplayName()).toFile(); + this.buildDir = new File(this.workingDir, "build"); + this.buildDir.mkdir(); + this.podsDir = new File(this.workingDir, "pods"); + this.podsDir.mkdir(); + this.workingDir.deleteOnExit(); + } + + @Test + public void testValueSubstitution() throws IOException, ExtenderException { + String jsonSpec = Files.readString(Path.of("test-data/pod_specs/Sentry.json")); + XCConfigParser parser = new XCConfigParser(this.buildDir, this.podsDir, PodUtils.Platform.IPHONEOS, "Debug", "arm64"); + File sentryTargetFolder = Path.of(this.podsDir.toString(), "Target Support Files", "Sentry").toFile(); + sentryTargetFolder.mkdirs(); + Files.copy(Path.of("test-data/xcconfigs/Sentry.xcconfig"), Path.of(sentryTargetFolder.toString(), "Sentry.debug.xcconfig"), StandardCopyOption.REPLACE_EXISTING); + + PodSpec podSpec = PodSpecParser.createPodSpec(PodSpecParser.parseJson(jsonSpec), PodUtils.Platform.IPHONEOS, null); + + CocoaPodsServiceBuildState cocoapodsState = new CocoaPodsServiceBuildState(); + cocoapodsState.workingDir = this.workingDir; + cocoapodsState.podsDir = this.podsDir; + cocoapodsState.selectedPlatform = PodUtils.Platform.IPHONEOS; + + CreateBuildSpecArgs args = new CreateBuildSpecArgs.Builder() + .setConfigParser(parser) + .setCocoapodsBuildState(cocoapodsState) + .setJobContext(this.jobContext) + .build(); + args.buildDir = this.buildDir; + args.configuration = "Debug"; + PodBuildSpec buildSpec = new PodBuildSpec(args, podSpec); + assertTrue(buildSpec.flags.c.contains("-DAPPLICATION_EXTENSION_API_ONLY_YES")); + assertTrue(buildSpec.flags.objc.contains("-DAPPLICATION_EXTENSION_API_ONLY_YES")); + } + + @Test + public void testCompilationFlags() throws IOException, ExtenderException { + String jsonSpec = Files.readString(Path.of("test-data/pod_specs/Sentry.json")); + XCConfigParser parser = new XCConfigParser(this.buildDir, this.podsDir, PodUtils.Platform.IPHONEOS, "Debug", "arm64"); + File sentryTargetFolder = Path.of(this.podsDir.toString(), "Target Support Files", "Sentry").toFile(); + sentryTargetFolder.mkdirs(); + Files.copy(Path.of("test-data/xcconfigs/Sentry.xcconfig"), Path.of(sentryTargetFolder.toString(), "Sentry.debug.xcconfig"), StandardCopyOption.REPLACE_EXISTING); + + PodSpec podSpec = PodSpecParser.createPodSpec(PodSpecParser.parseJson(jsonSpec), PodUtils.Platform.IPHONEOS, null); + CocoaPodsServiceBuildState cocoapodsState = new CocoaPodsServiceBuildState(); + cocoapodsState.workingDir = this.workingDir; + cocoapodsState.podsDir = this.podsDir; + cocoapodsState.selectedPlatform = PodUtils.Platform.IPHONEOS; + + CreateBuildSpecArgs args = new CreateBuildSpecArgs.Builder() + .setConfigParser(parser) + .setCocoapodsBuildState(cocoapodsState) + .setJobContext(this.jobContext) + .build(); + args.buildDir = this.buildDir; + args.configuration = "Debug"; + + PodBuildSpec buildSpec = new PodBuildSpec(args, podSpec); + // check handle APPLICATION_EXTENSION_API_ONLY flag + assertTrue(buildSpec.flags.c.contains("-fapplication-extension")); + assertTrue(buildSpec.flags.objc.contains("-fapplication-extension")); + assertTrue(buildSpec.flags.swift.contains("-application-extension")); + // check SWIFT_INCLUDE_PATHS + assertTrue(buildSpec.flags.swift.contains(String.format("-I%s/Sentry/Sources/Sentry/include", this.podsDir.toString()))); + } + +} diff --git a/server/src/test/java/com/defold/extender/services/cocoapods/PodSpecParserTest.java b/server/src/test/java/com/defold/extender/services/cocoapods/PodSpecParserTest.java new file mode 100644 index 00000000..0274b969 --- /dev/null +++ b/server/src/test/java/com/defold/extender/services/cocoapods/PodSpecParserTest.java @@ -0,0 +1,106 @@ +package com.defold.extender.services.cocoapods; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertFalse; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.stream.Stream; + +import javax.naming.InvalidNameException; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import com.defold.extender.ExtenderException; + + +public class PodSpecParserTest { + private static final String INHERITED_VALUE = "$(inherited)"; + + private static Stream specData() { + return Stream.of( + Arguments.of("PNChartboostSDKAdapter", Path.of("test-data/pod_specs/PNChartboostSDKAdapter.json")), + Arguments.of("UnityAds", Path.of("test-data/pod_specs/UnityAds.json")), + Arguments.of("Cuckoo", Path.of("test-data/pod_specs/Cuckoo.json")), + Arguments.of("Rtc555Sdk", Path.of("test-data/pod_specs/Rtc555Sdk.json")), + Arguments.of("streethawk", Path.of("test-data/pod_specs/streethawk.json")), + Arguments.of("AXPracticalHUD", "test-data/pod_specs/AXPracticalHUD.json"), + Arguments.of("PubNub", "test-data/pod_specs/PubNub.json"), + Arguments.of("TPNiOS", "test-data/pod_specs/TPNiOS.json"), + Arguments.of("Wilddog", "test-data/pod_specs/Wilddog.json") + ); + } + + @BeforeEach + public void setUp(TestInfo testInfo) throws IOException, InvalidNameException { + if (testInfo.getDisplayName().contains(" ")) { + // display name is used as folder prefix so if it contains spaces - ask to remove spaces )) + throw new InvalidNameException("Test's display name shouldn't contain spaces"); + } + } + + @ParameterizedTest(name = "{index}_testParsePodSpecs_{0}") + @MethodSource("specData") + public void testParsePodSpecs(String alias, Path spec) throws IOException, ExtenderException { + String jsonSpec = Files.readString(spec); + assertDoesNotThrow(() -> PodSpecParser.createPodSpec(PodSpecParser.parseJson(jsonSpec), PodUtils.Platform.IPHONEOS, null)); + } + + @ParameterizedTest(name = "{index}_testPodSpecsNoInherited_{0}") + @MethodSource("specData") + public void testPodSpecsNoInherited(String alias, Path spec) throws IOException, ExtenderException { + String jsonSpec = Files.readString(spec); + PodSpec podSpec = PodSpecParser.createPodSpec(PodSpecParser.parseJson(jsonSpec), PodUtils.Platform.IPHONEOS, null); + assertFalse(podSpec.linkflags.contains(INHERITED_VALUE)); + assertFalse(podSpec.defines.contains(INHERITED_VALUE)); + assertFalse(podSpec.flags.c.contains(INHERITED_VALUE)); + assertFalse(podSpec.flags.cpp.contains(INHERITED_VALUE)); + assertFalse(podSpec.flags.objc.contains(INHERITED_VALUE)); + assertFalse(podSpec.flags.objcpp.contains(INHERITED_VALUE)); + + for (PodSpec subspec : podSpec.subspecs) { + assertFalse(subspec.linkflags.contains(INHERITED_VALUE)); + assertFalse(subspec.defines.contains(INHERITED_VALUE)); + assertFalse(subspec.flags.c.contains(INHERITED_VALUE)); + assertFalse(subspec.flags.cpp.contains(INHERITED_VALUE)); + assertFalse(subspec.flags.objc.contains(INHERITED_VALUE)); + assertFalse(subspec.flags.objcpp.contains(INHERITED_VALUE)); + } + } + + @Test + public void testSpecBraceExpanderVendoredFrameworks() throws ExtenderException, IOException { + String jsonSpec = Files.readString(Path.of("test-data/pod_specs/TPNiOS.json")); + PodSpec podSpec = PodSpecParser.createPodSpec(PodSpecParser.parseJson(jsonSpec), PodUtils.Platform.IPHONEOS, null); + String[] expectedValues = new String[]{ + "core/AnyThinkBanner.xcframework", + "core/AnyThinkSplash.xcframework", + "core/AnyThinkRewardedVideo.xcframework", + "core/AnyThinkInterstitial.xcframework", + "core/AnyThinkNative.xcframework", + "core/AnyThinkMediaVideo.xcframework", + "core/AnyThinkSDK.xcframework" + }; + assertArrayEquals(expectedValues, podSpec.subspecs.get(0).vendoredFrameworks.toArray()); + } + + @Test + public void testDefaultSubspecs() throws ExtenderException, IOException { + String jsonSpec = Files.readString(Path.of("test-data/pod_specs/Wilddog.json")); + PodSpec podSpec = PodSpecParser.createPodSpec(PodSpecParser.parseJson(jsonSpec), PodUtils.Platform.IPHONEOS, null); + String[] expectedValues = new String[]{ + "Public", + "Sync", + "Auth", + "Core" + }; + assertArrayEquals(expectedValues, podSpec.defaultSubspecs.toArray()); + } +} diff --git a/server/src/test/java/com/defold/extender/services/cocoapods/PodUtilsTest.java b/server/src/test/java/com/defold/extender/services/cocoapods/PodUtilsTest.java index d11bd452..f20bf77e 100644 --- a/server/src/test/java/com/defold/extender/services/cocoapods/PodUtilsTest.java +++ b/server/src/test/java/com/defold/extender/services/cocoapods/PodUtilsTest.java @@ -1,6 +1,9 @@ package com.defold.extender.services.cocoapods; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.File; import java.io.IOException; @@ -10,6 +13,8 @@ import org.junit.jupiter.api.Test; +import com.defold.extender.ExtenderException; + public class PodUtilsTest { @Test @@ -52,4 +57,26 @@ public void testListFilesByPattern() throws IOException { files = PodUtils.listFilesAndDirsGlob(workDir, "inner_folder1/inner_folder2/PodTestBundleDir.bundle"); assertEquals(1, files.size()); } + + @Test + public void testHasSource() { + PodBuildSpec spec = new PodBuildSpec(); + assertFalse(PodUtils.hasSourceFiles(spec)); + spec.sourceFiles.addAll(List.of(new File("source/source1.cpp"), new File("source/source2.cpp"))); + assertTrue(PodUtils.hasSourceFiles(spec)); + + PodBuildSpec spec1 = new PodBuildSpec(); + spec1.swiftSourceFiles.addAll(List.of(new File("swift_code/swift1.swift"))); + assertTrue(PodUtils.hasSourceFiles(spec1)); + } + + @Test + public void testSwiftModuleName() throws ExtenderException { + assertEquals("arm64-apple-ios", PodUtils.swiftModuleNameFromPlatform("arm64-ios")); + assertEquals("x86_64-apple-ios-simulator", PodUtils.swiftModuleNameFromPlatform("x86_64-ios")); + assertEquals("arm64-apple-macos", PodUtils.swiftModuleNameFromPlatform("arm64-macos")); + assertEquals("x86_64-apple-macos", PodUtils.swiftModuleNameFromPlatform("x86_64-macos")); + assertThrows(ExtenderException.class, () -> { PodUtils.swiftModuleNameFromPlatform("x86_64-linux"); }); + + } } diff --git a/server/src/test/java/com/defold/extender/services/cocoapods/PodfileParserTest.java b/server/src/test/java/com/defold/extender/services/cocoapods/PodfileParserTest.java new file mode 100644 index 00000000..0500141d --- /dev/null +++ b/server/src/test/java/com/defold/extender/services/cocoapods/PodfileParserTest.java @@ -0,0 +1,224 @@ +package com.defold.extender.services.cocoapods; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.util.List; +import java.util.stream.Stream; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +public class PodfileParserTest { + private File emptyPodfile; + private File withCommentPodfile; + private File regularPodfile; + private File wrongPodfile; + + @BeforeEach + public void setUp(TestInfo testInfo) { + this.emptyPodfile = new File("test-data/podfiles/empty.Podfile"); + this.regularPodfile = new File("test-data/podfiles/regular.Podfile"); + this.wrongPodfile = new File("test-data/podfiles/wrong.Podfile"); + this.withCommentPodfile = new File("test-data/podfiles/with_comments.Podfile"); + } + + @Test + public void testPodfileParserRegularSyntax() throws IOException, PodfileParsingException { + PodfileParser.ParseResult res = CocoaPodsService.parsePodfiles(List.of(this.regularPodfile), "ios", "11.0"); + List expected = List.of("IronSourcePangleAdapter", "IronSourceSmaatoAdapter", "IronSourceSuperAwesomeAdapter", "IronSourceTencentAdapter", "IronSourceUnityAdsAdapter", "IronSourceYandexAdapter"); + assertTrue(res.podNames.containsAll(expected) && expected.containsAll(res.podNames)); + } + + @Test + public void testPodfileParserComments() throws IOException, PodfileParsingException { + PodfileParser.ParseResult res = CocoaPodsService.parsePodfiles(List.of(this.withCommentPodfile), "ios", "10.3"); + assertEquals(res.minVersion, "12.0"); + List expected = List.of("OneTrust-CMP-XCFramework"); + assertTrue(expected.containsAll(res.podNames) && res.podNames.containsAll(expected)); + } + + @Test + public void testPodfileParserWrongSyntax() throws IOException, PodfileParsingException { + PodfileParser.ParseResult res = CocoaPodsService.parsePodfiles(List.of(this.wrongPodfile), "ios", "11.0"); + List expected = List.of("IronSourceSuperAwesomeAdapter"); + assertTrue(expected.containsAll(res.podNames) && res.podNames.containsAll(expected)); + } + + @Test + public void testPodfileParserEmptyFile() throws IOException, PodfileParsingException { + PodfileParser.ParseResult res = CocoaPodsService.parsePodfiles(List.of(this.emptyPodfile), "ios", "11.0"); + assertTrue(res.podNames.isEmpty()); + } + + @Test + public void testPodfileParserAll() throws IOException, PodfileParsingException { + PodfileParser.ParseResult result = CocoaPodsService.parsePodfiles(List.of(this.emptyPodfile, this.wrongPodfile, this.regularPodfile, this.withCommentPodfile), "ios", "9.1"); + List expected = List.of("IronSourceSuperAwesomeAdapter", "IronSourcePangleAdapter", "IronSourceSmaatoAdapter", "IronSourceSuperAwesomeAdapter", "IronSourceTencentAdapter", "IronSourceUnityAdsAdapter", "IronSourceYandexAdapter", "OneTrust-CMP-XCFramework"); + assertTrue(expected.containsAll(result.podNames) && result.podNames.containsAll(expected)); + } + + @Test + public void testCompareVersions() { + assertThrows(NullPointerException.class, () -> { PodfileParser.compareVersions(null, null); }); + assertEquals(0, PodfileParser.compareVersions("12.0", "12.0")); + assertEquals(0, PodfileParser.compareVersions("12.0", "12.0.")); + assertTrue(PodfileParser.compareVersions("13.0.1", "13.0") > 0); + assertTrue(PodfileParser.compareVersions("9.3", "12.6.1") < 0); + assertThrows(NumberFormatException.class, () -> { PodfileParser.compareVersions("9.3", "unknown"); }); + assertThrows(NullPointerException.class, () -> { PodfileParser.compareVersions("9.3", null); }); + } + + private static Stream mergeVersionsData() { + return Stream.of( + Arguments.of("tvos", "3.0"), + Arguments.of("ios", "11.0"), + Arguments.of("macos", "10.15") + ); + } + + @Test + public void testParseResultMergeEmpty() { + PodfileParser.ParseResult emptyResult = new PodfileParser.ParseResult(); + + PodfileParser.ParseResult test = new PodfileParser.ParseResult(); + assertDoesNotThrow(() -> { test.mergeWith(emptyResult); }); + assertTrue(test.minVersion == null); + assertTrue(test.platform == null); + assertTrue(test.useFrameworks); + assertTrue(test.podDefinitions.isEmpty()); + assertTrue(test.podNames.isEmpty()); + } + + @ParameterizedTest + @MethodSource("mergeVersionsData") + public void testParseResultMergeNonEmpty(String platform, String version) { + PodfileParser.ParseResult nonEmpty = new PodfileParser.ParseResult(platform, version); + + PodfileParser.ParseResult test = new PodfileParser.ParseResult(); + assertDoesNotThrow(() -> { test.mergeWith(nonEmpty); }); + assertEquals(version, test.minVersion); + assertEquals(platform, test.platform); + assertTrue(test.useFrameworks); + assertTrue(test.podDefinitions.isEmpty()); + assertTrue(test.podNames.isEmpty()); + } + + @Test + public void testParseResultMismatchPlatform() { + PodfileParser.ParseResult baseResult = new PodfileParser.ParseResult("ios", "13.0"); + + PodfileParser.ParseResult unsupportedPlatform = new PodfileParser.ParseResult("tvos", "3.0"); + assertThrows(PodfileParsingException.class, () -> { baseResult.mergeWith(unsupportedPlatform); }); + assertEquals("13.0", baseResult.minVersion); + assertEquals("ios", baseResult.platform); + assertTrue(baseResult.useFrameworks); + assertTrue(baseResult.podDefinitions.isEmpty()); + assertTrue(baseResult.podNames.isEmpty()); + } + + @Test + public void testParseResultMergeDiffVersions() { + PodfileParser.ParseResult lessVersion = new PodfileParser.ParseResult("macos", "10.21"); + PodfileParser.ParseResult greaterVersion = new PodfileParser.ParseResult("macos", "11.3"); + PodfileParser.ParseResult sameVersion = new PodfileParser.ParseResult("macos", "11.0"); + + PodfileParser.ParseResult test1 = new PodfileParser.ParseResult("macos", "11.0"); + assertDoesNotThrow(() -> { test1.mergeWith(lessVersion); }); + assertEquals("11.0", test1.minVersion); + + PodfileParser.ParseResult test2 = new PodfileParser.ParseResult("macos", "11.0"); + assertDoesNotThrow(() -> { test2.mergeWith(greaterVersion); }); + assertEquals("11.3", test2.minVersion); + + PodfileParser.ParseResult test3 = new PodfileParser.ParseResult("macos", "11.0"); + assertDoesNotThrow(() -> { test3.mergeWith(sameVersion); }); + assertEquals("11.0", test3.minVersion); + } + + @Test + public void testParseResultUseFrameworkFlag() { + PodfileParser.ParseResult falseFlag = new PodfileParser.ParseResult("ios", "11.0"); + falseFlag.useFrameworks = false; + PodfileParser.ParseResult trueFlag = new PodfileParser.ParseResult("ios", "11.0"); + trueFlag.useFrameworks = true; + + PodfileParser.ParseResult baseFalseTest1 = new PodfileParser.ParseResult("ios", "11.0"); + baseFalseTest1.useFrameworks = false; + assertDoesNotThrow(() -> { baseFalseTest1.mergeWith(falseFlag); }); + assertFalse(baseFalseTest1.useFrameworks); + + PodfileParser.ParseResult baseFalseTest2 = new PodfileParser.ParseResult("ios", "11.0"); + baseFalseTest2.useFrameworks = false; + assertDoesNotThrow(() -> { baseFalseTest2.mergeWith(trueFlag); }); + assertTrue(baseFalseTest2.useFrameworks); + + PodfileParser.ParseResult baseTrueTest1 = new PodfileParser.ParseResult("ios", "11.0"); + baseTrueTest1.useFrameworks = true; + assertDoesNotThrow(() -> { baseTrueTest1.mergeWith(falseFlag); }); + assertTrue(baseTrueTest1.useFrameworks); + + PodfileParser.ParseResult baseTrueTest2 = new PodfileParser.ParseResult("ios", "11.0"); + baseTrueTest2.useFrameworks = true; + assertDoesNotThrow(() -> { baseTrueTest2.mergeWith(trueFlag); }); + assertTrue(baseTrueTest2.useFrameworks); + } + + @Test + public void testParseResultPodDefinitions() { + PodfileParser.ParseResult result1 = new PodfileParser.ParseResult("ios", "12.0"); + result1.podDefinitions.addAll( + List.of( + "pod 'testPod1', '1.0.3'", + "pod 'testPod2', '~>0.0.4'", + "pod 'testPod3/subspec1'" + ) + ); + result1.podNames.addAll(List.of("testPod1", "testPod2", "testPod3")); + + PodfileParser.ParseResult result2 = new PodfileParser.ParseResult("ios", "12.0"); + result2.podDefinitions.addAll( + List.of( + "pod 'testPod5', '1.0.3'", + "pod 'testPod3/subspec2', '~>0.0.4'", + "pod 'testPod2'", + "pod 'testPod3/subspec1'" + ) + ); + result2.podNames.addAll(List.of("testPod5", "testPod3", "testPod2")); + + List expectedDefinitions1 = List.of( + "pod 'testPod1', '1.0.3'", + "pod 'testPod2', '~>0.0.4'", + "pod 'testPod3/subspec1'" + ); + List expectedNames1 = List.of("testPod1", "testPod2", "testPod3"); + List expectedDefinitions2 = List.of( + "pod 'testPod1', '1.0.3'", + "pod 'testPod2', '~>0.0.4'", + "pod 'testPod3/subspec1'", + "pod 'testPod5', '1.0.3'", + "pod 'testPod3/subspec2', '~>0.0.4'", + "pod 'testPod2'" + ); + List expectedNames2 = List.of("testPod1", "testPod2", "testPod3", "testPod5"); + + PodfileParser.ParseResult test = new PodfileParser.ParseResult("ios", "12.0"); + assertDoesNotThrow(() -> { test.mergeWith(result1); }); + assertTrue(test.podDefinitions.containsAll(expectedDefinitions1) && expectedDefinitions1.containsAll(test.podDefinitions)); + assertTrue(test.podNames.containsAll(expectedNames1) && expectedNames1.containsAll(test.podNames)); + + assertDoesNotThrow(() -> { test.mergeWith(result2); }); + assertTrue(test.podDefinitions.containsAll(expectedDefinitions2) && expectedDefinitions2.containsAll(test.podDefinitions)); + assertTrue(test.podNames.containsAll(expectedNames2) && expectedNames2.containsAll(test.podNames)); + } +} diff --git a/server/src/test/java/com/defold/extender/services/cocoapods/ResolvedPodsTest.java b/server/src/test/java/com/defold/extender/services/cocoapods/ResolvedPodsTest.java new file mode 100644 index 00000000..c3471b9f --- /dev/null +++ b/server/src/test/java/com/defold/extender/services/cocoapods/ResolvedPodsTest.java @@ -0,0 +1,220 @@ +package com.defold.extender.services.cocoapods; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.naming.InvalidNameException; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.junit.jupiter.api.condition.EnabledOnOs; +import org.junit.jupiter.api.condition.OS; + +import com.defold.extender.ExtenderException; +import com.defold.extender.TestUtils; + +public class ResolvedPodsTest { + File emptyPodfile; + File regularPodfile; + File wrongPodfile; + File withCommentPodfile; + + private Map jobContext; + private File workingDir; + private File buildDir; + private File podsDir; + private File frameworksDir; + + private CocoaPodsServiceBuildState cocoapodsState; + CreateBuildSpecArgs args; + + class EmptyConfigParser implements IConfigParser { + @Override + public Map parse(String moduleName, String podName, File xcconfig) throws IOException { + return Map.of(); + } + + } + + private static File createEmptyFiles(File podsDir, String podName, String[] filesToCreate) throws IOException { + File resultFolder = new File(podsDir, podName); + for (String f : filesToCreate) { + File emptyF = new File(resultFolder, f); + emptyF.getParentFile().mkdirs(); + emptyF.createNewFile(); + } + return resultFolder; + } + + @BeforeEach + public void setUp(TestInfo testInfo) throws IOException, InvalidNameException { + this.emptyPodfile = new File("test-data/podfiles/empty.Podfile"); + this.regularPodfile = new File("test-data/podfiles/regular.Podfile"); + this.wrongPodfile = new File("test-data/podfiles/wrong.Podfile"); + this.withCommentPodfile = new File("test-data/podfiles/with_comments.Podfile"); + + this.jobContext = new HashMap<>(); + this.jobContext.putAll(TestUtils.envFileToMap(new File("envs/.env"))); + this.jobContext.putAll(TestUtils.envFileToMap(new File("envs/macos.env"))); + + if (testInfo.getDisplayName().contains(" ")) { + // display name is used as folder prefix so if it contains spaces - ask to remove spaces )) + throw new InvalidNameException("Test's display name shouldn't contain spaces"); + } + this.workingDir = Files.createTempDirectory(testInfo.getDisplayName()).toFile(); + this.buildDir = new File(this.workingDir, "build"); + this.buildDir.mkdir(); + this.podsDir = new File(this.workingDir, "pods"); + this.podsDir.mkdir(); + this.frameworksDir = Path.of(this.buildDir.toString(), "Debugiphoneos", "XCFrameworkIntermediates").toFile(); + this.frameworksDir.mkdirs(); + this.workingDir.deleteOnExit(); + + this.cocoapodsState = new CocoaPodsServiceBuildState(); + this.cocoapodsState.podsDir = this.podsDir; + this.cocoapodsState.selectedPlatform = PodUtils.Platform.IPHONEOS; + this.cocoapodsState.workingDir = this.workingDir; + this.cocoapodsState.unpackedFrameworksDir = this.frameworksDir; + this.args = new CreateBuildSpecArgs.Builder() + .setJobContext(jobContext) + .setCocoapodsBuildState(this.cocoapodsState) + .setConfigParser(new EmptyConfigParser()) + .build(); + this.args.buildDir = this.buildDir; + this.args.configuration = "Debug"; + + } + + @Test + @EnabledOnOs({ OS.MAC }) + public void testResourceBundleParsing() throws IOException, ExtenderException { + String jsonSpec = Files.readString(Path.of("test-data/pod_specs/UnityAds.json")); + + String[] simulatedFiles = new String[]{ + "UnityAds.xcframework/ios-arm64/UnityAds.framework/PrivacyInfo.xcprivacy", + "UnityAds.xcframework/ios-arm64/UnityAds.framework/omid-session-client-v1.js", + "UnityAds.xcframework/ios-arm64/UnityAds.framework/omsdk-v1.js", + "UnityAds.xcframework/ios-arm64/UnityAds.framework/Info.plist" + }; + createEmptyFiles(this.podsDir, "UnityAds", simulatedFiles); + + PodSpec unityAdsSpec = PodSpecParser.createPodSpec(PodSpecParser.parseJson(jsonSpec), PodUtils.Platform.IPHONEOS, null); + PodBuildSpec buildSpec = new PodBuildSpec(this.args, unityAdsSpec); + ResolvedPods resolvedPods = new ResolvedPods(this.cocoapodsState, List.of(buildSpec), new File(this.buildDir, "Podfile.lock"), new MainPodfile()); + + File targetDir = new File(this.workingDir, "result"); + List result = resolvedPods.createResourceBundles(targetDir, "arm64-ios"); + assertEquals(result.size(), 1); + assertEquals("UnityAdsResources.bundle", result.get(0).getName()); + List expectedFiles = List.of( + "PrivacyInfo.xcprivacy", + "omid-session-client-v1.js", + "omsdk-v1.js", + "Info.plist" + ); + + File bundleDir = new File(targetDir, "UnityAdsResources.bundle"); + assertTrue(bundleDir.exists()); + List resultContent = Files.list(bundleDir.toPath()).toList(); + assertEquals(expectedFiles.size(), resultContent.size()); + for (Path p : resultContent) { + assertTrue(expectedFiles.contains(p.getFileName().toString())); + } + } + + @Test + public void testPodResourceFromBundle() throws IOException, ExtenderException { + String jsonSpec = Files.readString(Path.of("test-data/pod_specs/AXPracticalHUD.json")); + String[] simulatedFiles = new String[]{ + "AXPracticalHUD/AXPracticalHUD/AXPracticalHUD.bundle/ax_hud_error@2x.png", + "AXPracticalHUD/AXPracticalHUD/AXPracticalHUD.bundle/ax_hud_error@3x.png", + "AXPracticalHUD/AXPracticalHUD/AXPracticalHUD.bundle/ax_hud_success@2x.png", + "AXPracticalHUD/AXPracticalHUD/AXPracticalHUD.bundle/ax_hud_success@3x.png" + }; + File podFolder = createEmptyFiles(this.podsDir, "AXPracticalHUD", simulatedFiles); + File[] expectedFiles = new File[]{ + new File(podFolder, "AXPracticalHUD/AXPracticalHUD/AXPracticalHUD.bundle/ax_hud_error@2x.png"), + new File(podFolder, "AXPracticalHUD/AXPracticalHUD/AXPracticalHUD.bundle/ax_hud_error@3x.png"), + new File(podFolder, "AXPracticalHUD/AXPracticalHUD/AXPracticalHUD.bundle/ax_hud_success@2x.png"), + new File(podFolder, "AXPracticalHUD/AXPracticalHUD/AXPracticalHUD.bundle/ax_hud_success@3x.png") + }; + PodSpec podSpec = PodSpecParser.createPodSpec(PodSpecParser.parseJson(jsonSpec), PodUtils.Platform.IPHONEOS, null); + PodBuildSpec buildSpec = new PodBuildSpec(this.args, podSpec); + ResolvedPods resolvedPods = new ResolvedPods(this.cocoapodsState, List.of(buildSpec), new File(this.buildDir, "Podfile.lock"), new MainPodfile()); + + List result = resolvedPods.getAllPodResources(); + assertArrayEquals(expectedFiles, result.toArray()); + } + + @Test + public void testSpecBraceExpanderSourceFiles() throws ExtenderException, IOException { + String jsonSpec = Files.readString(Path.of("test-data/pod_specs/PubNub.json")); + // create empty files to simulate sources + String[] simulatedFiles = new String[]{ + "PubNub/Core/core1/core11.cpp", + "PubNub/Core/core1/core12.cpp", + "PubNub/Core/core1/core1.h", + "PubNub/Core/core2/core21.cpp", + "PubNub/Core/core2/core22.cpp", + "PubNub/Core/core2/core2.h", + + "PubNub/Data/data1/data11.cpp", + "PubNub/Data/data1/data12.cpp", + "PubNub/Data/data1/data1.h", + "PubNub/Data/data2/data21.cpp", + "PubNub/Data/data2/data22.cpp", + "PubNub/Data/data2/data2.h", + + "PubNub/Misc/misc1/misc11.cpp", + "PubNub/Misc/misc1/misc12.cpp", + "PubNub/Misc/misc1/misc1.h", + "PubNub/Misc/misc2/misc21.cpp", + "PubNub/Misc/misc2/misc22.cpp", + "PubNub/Misc/misc2/misc2.h", + + "PubNub/Network/network1/network11.cpp", + "PubNub/Network/network1/network12.cpp", + "PubNub/Network/network1/network1.h", + "PubNub/Network/network2/network21.cpp", + "PubNub/Network/network2/network22.cpp", + "PubNub/Network/network2/network2.h" + }; + File pubNubFolder = createEmptyFiles(this.podsDir, "PubNub", simulatedFiles); + + PodSpec podSpec = PodSpecParser.createPodSpec(PodSpecParser.parseJson(jsonSpec), PodUtils.Platform.IPHONEOS, null); + File[] expectedValues = new File[]{ + new File(pubNubFolder, "PubNub/Core/core1/core11.cpp"), + new File(pubNubFolder, "PubNub/Core/core1/core12.cpp"), + new File(pubNubFolder, "PubNub/Core/core2/core21.cpp"), + new File(pubNubFolder, "PubNub/Core/core2/core22.cpp"), + + new File(pubNubFolder, "PubNub/Data/data1/data11.cpp"), + new File(pubNubFolder, "PubNub/Data/data1/data12.cpp"), + new File(pubNubFolder, "PubNub/Data/data2/data21.cpp"), + new File(pubNubFolder, "PubNub/Data/data2/data22.cpp"), + + new File(pubNubFolder, "PubNub/Misc/misc1/misc11.cpp"), + new File(pubNubFolder, "PubNub/Misc/misc1/misc12.cpp"), + new File(pubNubFolder, "PubNub/Misc/misc2/misc21.cpp"), + new File(pubNubFolder, "PubNub/Misc/misc2/misc22.cpp"), + + new File(pubNubFolder, "PubNub/Network/network1/network11.cpp"), + new File(pubNubFolder, "PubNub/Network/network1/network12.cpp"), + new File(pubNubFolder, "PubNub/Network/network2/network21.cpp"), + new File(pubNubFolder, "PubNub/Network/network2/network22.cpp") + }; + PodBuildSpec buildSpec = new PodBuildSpec(this.args, podSpec); + buildSpec.addSubSpec(podSpec.subspecs.get(0)); + assertArrayEquals(expectedValues, buildSpec.sourceFiles.toArray()); + } +} diff --git a/server/src/test/java/com/defold/extender/services/cocoapods/XCConfigParserTest.java b/server/src/test/java/com/defold/extender/services/cocoapods/XCConfigParserTest.java index 4e75dfd8..96f35295 100644 --- a/server/src/test/java/com/defold/extender/services/cocoapods/XCConfigParserTest.java +++ b/server/src/test/java/com/defold/extender/services/cocoapods/XCConfigParserTest.java @@ -24,7 +24,6 @@ public class XCConfigParserTest { private static final String PODS_BUILD_DIR = "/var/tmp/tmp-dir/build"; private static final String PODS_DIR = "/var/tmp/tmp-dir/pods-dir"; - // private static final String POD_CONFIGURATION_DIR = String.format("%s/Target Support Files", PODS_DIR); XCConfigParser parser; Map baseVars; @@ -35,7 +34,7 @@ public void beforeAll() throws IOException { podsDir.deleteOnExit(); File workingDir = Files.createTempDirectory("pods-working-dir").toFile(); workingDir.deleteOnExit(); - this.parser = new XCConfigParser(workingDir, podsDir, PodSpecParser.Platform.IPHONEOS.toString().toLowerCase(), "Debug", "arm64"); + this.parser = new XCConfigParser(workingDir, podsDir, PodUtils.Platform.IPHONEOS, "Debug", "arm64"); this.baseVars = createMockBaseVars(); } @@ -44,7 +43,8 @@ private Map createMockBaseVars() { "PODS_ROOT", "/Users/test-pod/Pods", "PODS_BUILD_DIR", "/var/tmp/tmp-dir/pod-build-dir", "CONFIGURATION", "debug", - "EFFECTIVE_PLATFORM_NAME","iphones" + "EFFECTIVE_PLATFORM_NAME","iphoneos", + "TOOLCHAIN_DIR", "/opt/platformsdk/XcodeDefaults16.0.toolchain" ); } @@ -63,16 +63,16 @@ private static Stream postProcessData() { return Stream.of( Arguments.of("INHERITED_VALUE = $(inherited) additional_value other_value ", "additional_value other_value"), Arguments.of("VALUE_SUBSTITUTION1=$(inherited) ${PODS_ROOT}/Headers/Private ${PODS_ROOT}/Headers/Private/KSCrash ${PODS_ROOT}/Headers/Public ${PODS_ROOT}/Headers/Public/KSCrash", "/Users/test-pod/Pods/Headers/Private /Users/test-pod/Pods/Headers/Private/KSCrash /Users/test-pod/Pods/Headers/Public /Users/test-pod/Pods/Headers/Public/KSCrash"), - Arguments.of("VALUE_SUBSTITUTION2=${PODS_BUILD_DIR}/$(CONFIGURATION)/$(EFFECTIVE_PLATFORM_NAME)", "/var/tmp/tmp-dir/pod-build-dir/debug/iphones"), - Arguments.of("VALUE_SUBSTITUTION3=${PODS_BUILD_DIR}/${CONFIGURATION}/$(EFFECTIVE_PLATFORM_NAME)", "/var/tmp/tmp-dir/pod-build-dir/debug/iphones"), + Arguments.of("VALUE_SUBSTITUTION2=${PODS_BUILD_DIR}/$(CONFIGURATION)/$(EFFECTIVE_PLATFORM_NAME)", "/var/tmp/tmp-dir/pod-build-dir/debug/iphoneos"), + Arguments.of("VALUE_SUBSTITUTION3=${PODS_BUILD_DIR}/${CONFIGURATION}/$(EFFECTIVE_PLATFORM_NAME)", "/var/tmp/tmp-dir/pod-build-dir/debug/iphoneos"), Arguments.of("POD_VERSION=$(POD_VERSION)", "$(POD_VERSION)"), Arguments.of("POD_VERSION=${POD_VERSION}", "${POD_VERSION}") ); } private static Stream parsingData() { - String podsConfigurationBuildDir = String.format("%s/%s%s", PODS_BUILD_DIR, "Debug", "iphones"); - String podsXCFrameworksBuildDir = String.format("%s/XCFrameworkIntermediates", podsConfigurationBuildDir, "Debug", "iphones"); + String podsConfigurationBuildDir = String.format("%s/%s%s", PODS_BUILD_DIR, "Debug", "iphoneos"); + String podsXCFrameworksBuildDir = String.format("%s/XCFrameworkIntermediates", podsConfigurationBuildDir, "Debug", "iphoneos"); // Arguments struct // * Pod name // * path to xcconfig @@ -133,7 +133,14 @@ private static Stream parsingData() { @Test public void testBaseVariables() { - Map baseVars = parser.calculateBaseVariables(null, "testPodSDK"); + Map vars = parser.calculateBaseVariables("testPodSDKModule", "testPodSDK"); + assertEquals(vars, vars); + assertEquals("iphoneos", vars.get("EFFECTIVE_PLATFORM_NAME")); + assertEquals("Debug", vars.get("CONFIGURATION")); + assertEquals("Headers/Public/testPodSDKModule/testPodSDK.modulemap", vars.get("MODULEMAP_FILE")); + assertEquals("en", vars.get("DEVELOPMENT_LANGUAGE")); + assertEquals("iphoneos", vars.get("PLATFORM_NAME")); + assertEquals("arm64", vars.get("ARCHS")); } @ParameterizedTest @@ -154,7 +161,7 @@ public void testPostProcessLine(String inputLine, String expectedResult) { @ParameterizedTest(name = "{index}_testParsing_{0}") @MethodSource("parsingData") public void testParsing(String podName, File inputSource, String moduleName, Map expectedSubset) throws IOException { - XCConfigParser parser = new XCConfigParser(new File(PODS_BUILD_DIR), new File(PODS_DIR), "iphones", "Debug", "arm64"); + XCConfigParser parser = new XCConfigParser(new File(PODS_BUILD_DIR), new File(PODS_DIR), PodUtils.Platform.IPHONEOS, "Debug", "arm64"); Map result = parser.parse(moduleName, podName, inputSource); for (Map.Entry entry : expectedSubset.entrySet()) { String key = entry.getKey(); @@ -165,7 +172,7 @@ public void testParsing(String podName, File inputSource, String moduleName, Map @Test public void testIncludeParsing() { - XCConfigParser parser = new XCConfigParser(new File(PODS_BUILD_DIR), new File(PODS_DIR), "iphones", "Debug", "arm64"); + XCConfigParser parser = new XCConfigParser(new File(PODS_BUILD_DIR), new File(PODS_DIR), PodUtils.Platform.IPHONEOS, "Debug", "arm64"); File testFile = new File("test-data/xcconfigs/include_test.xcconfig"); assertThrows(UnsupportedOperationException.class, () -> { parser.parse("other", "other", testFile); }); }