From 69f5878ad7335a3ca81f7572093d36c8a7c34529 Mon Sep 17 00:00:00 2001 From: Kharkunov Eugene Date: Wed, 12 Nov 2025 10:11:14 +0300 Subject: [PATCH 1/4] Tests --- server/envs/android.env | 44 +++++++++++++++++++ server/envs/generate_user_env.sh | 4 ++ .../com/defold/extender/ExtenderUtil.java | 19 ++++---- .../com/defold/extender/ExtenderUtilTest.java | 23 ++++++++++ server/test-data/appmanifests/app.appmanifest | 6 +++ server/test-data/appmanifests/ext1.manifest | 5 +++ server/test-data/appmanifests/ext2.manifest | 10 +++++ 7 files changed, 102 insertions(+), 9 deletions(-) create mode 100644 server/envs/android.env create mode 100644 server/test-data/appmanifests/app.appmanifest create mode 100644 server/test-data/appmanifests/ext1.manifest create mode 100644 server/test-data/appmanifests/ext2.manifest diff --git a/server/envs/android.env b/server/envs/android.env new file mode 100644 index 00000000..e67f9d93 --- /dev/null +++ b/server/envs/android.env @@ -0,0 +1,44 @@ +GRADLE_USER_HOME=/tmp/.gradle +GRADLE_VERSION=8.7 +GRADLE_PLUGIN_VERSION=8.6.1 + +ANDROID_ROOT=/Users/yauheni/Library/Android/sdk +ANDROID_HOME=/Users/yauheni/Library/Android/sdk +ANDROID_SDK_ROOT=/Users/yauheni/Library/Android/sdk +ANDROID_SDK_HOME=/Users/yauheni/Library/Android/sdk + +# +# SDK 35 (Defold 1.9.8): +# + +ANDROID_BUILD_TOOLS_VERSION_35=35.0.0 +ANDROID_SDK_VERSION_35=35 + +ANDROID_SDK_BUILD_TOOLS_PATH_35=/Users/yauheni/Library/Android/sdk/build-tools/35.0.0 +ENV ANDROID_LIBRARYJAR_35=/Users/yauheni/Library/Android/sdk/platforms/android-35/android.jar + +# We specify it in build_input.yml by setting it the first in PATH for new SDK +# But at least one SDK should be specified by default in Dockerfile +ANDROID_SDK_VERSION=35 + +ENV ANDROID_NDK25_VERSION=25b +ENV ANDROID_NDK25_API_VERSION=19 +ENV ANDROID_64_NDK25_API_VERSION=21 + +# These paths are the same for both the 32 and 64 bit toolchains +ANDROID_NDK25_PATH=/Users/yauheni/Library/Android/sdk/ndk/25.1.8937393 +ANDROID_NDK25_BIN_PATH=/Users/yauheni/Library/Android/sdk/ndk/25.1.8937393/toolchains/llvm/prebuilt/darwin-x86_64/bin +ANDROID_NDK25_SYSROOT=/Users/yauheni/Library/Android/sdk/ndk/25.1.8937393/toolchains/llvm/prebuilt/darwin-x86_64/sysroot + +# We specify it in build_input.yml by setting it the first in PATH + +--------ENV ANDROID_PROGUARD=/usr/share/java/proguard.jar + +# download and install R8 utility separatly. It's done in that way because +# R8/D8 received fixes from time to time which not populated with build tools +# It's ok that R8.jar downloaded but saved to D8.jar. Can't find D8 as separate utility. +# R8 contains all classes of D8. +ENV R8_VERSION=8.7.0-dev +RUN \ + wget -O ${ANDROID_SDK_BUILD_TOOLS_PATH_34}/lib/d8.jar https://storage.googleapis.com/r8-releases/raw/${R8_VERSION}/r8.jar && \ + wget -O ${ANDROID_SDK_BUILD_TOOLS_PATH_35}/lib/d8.jar https://storage.googleapis.com/r8-releases/raw/${R8_VERSION}/r8.jar diff --git a/server/envs/generate_user_env.sh b/server/envs/generate_user_env.sh index be844d9a..de89f16b 100755 --- a/server/envs/generate_user_env.sh +++ b/server/envs/generate_user_env.sh @@ -67,4 +67,8 @@ echo "XCTOOLCHAIN_PATH=${PLATFORMSDK_DIR}/XcodeDefault${XCODE_16_VERSION}.xctool echo "PATH=\"${APPENDED_PATH}\"" >> $OUTPUT_FILE +echo ";DM_DEBUG_COMMANDS=1" >> $OUTPUT_FILE +echo ";DYNAMO_HOME=" >> $OUTPUT_FILE +echo ";DM_DEBUG_KEEP_JOB_FOLDER=1" >> $OUTPUT_FILE + echo "Generation completed." diff --git a/server/src/main/java/com/defold/extender/ExtenderUtil.java b/server/src/main/java/com/defold/extender/ExtenderUtil.java index 50dfa8f4..d1cb20fa 100644 --- a/server/src/main/java/com/defold/extender/ExtenderUtil.java +++ b/server/src/main/java/com/defold/extender/ExtenderUtil.java @@ -560,19 +560,20 @@ private static class PruneMapping { } }; + private static List MAPPINGS = List.of( + new PruneMapping("libs", "includeLibs", "excludeLibs"), + new PruneMapping("engineLibs", "includeLibs", "excludeLibs"), + new PruneMapping("engineJsLibs", "includeJsLibs", "excludeJsLibs"), + new PruneMapping("objectFiles", "includeObjectFiles", "excludeObjectFiles"), + new PruneMapping("dynamicLibs", "includeDynamicLibs", "excludeDynamicLibs"), + new PruneMapping("symbols", "includeSymbols", "excludeSymbols") + ); + // Copies the original context, and appends the extra context's elements, if the keys and types are valid static public Map mergeContexts(Map a, Map b) throws ExtenderException { Map context = mergeMaps(a, b); - List mappings = new ArrayList<>(); - mappings.add(new PruneMapping("libs", "includeLibs", "excludeLibs")); - mappings.add(new PruneMapping("engineLibs", "includeLibs", "excludeLibs")); - mappings.add(new PruneMapping("engineJsLibs", "includeJsLibs", "excludeJsLibs")); - mappings.add(new PruneMapping("objectFiles", "includeObjectFiles", "excludeObjectFiles")); - mappings.add(new PruneMapping("dynamicLibs", "includeDynamicLibs", "excludeDynamicLibs")); - mappings.add(new PruneMapping("symbols", "includeSymbols", "excludeSymbols")); - - for (PruneMapping mapping : mappings) { + for (PruneMapping mapping : MAPPINGS) { List srcList = ExtenderUtil.getStringList(context, mapping.targetName); if (srcList.isEmpty()) continue; diff --git a/server/src/test/java/com/defold/extender/ExtenderUtilTest.java b/server/src/test/java/com/defold/extender/ExtenderUtilTest.java index 1e37a605..58b60906 100644 --- a/server/src/test/java/com/defold/extender/ExtenderUtilTest.java +++ b/server/src/test/java/com/defold/extender/ExtenderUtilTest.java @@ -7,6 +7,7 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -320,4 +321,26 @@ public void testWriteSourceListToTempFile() throws IOException { assertTrue(expected.containsAll(writtenLines)); assertTrue(writtenLines.containsAll(expected)); } + + @Test + public void testMergeManifests() throws IOException, ExtenderException { + AppManifestConfiguration appManifest = Extender.loadYaml(null, new File("test-data/appmanifests/app.appmanifest"), AppManifestConfiguration.class); + ManifestConfiguration ext1 = Extender.loadYaml(null, new File("test-data/appmanifests/ext1.manifest"), ManifestConfiguration.class); + ManifestConfiguration ext2 = Extender.loadYaml(null, new File("test-data/appmanifests/ext2.manifest"), ManifestConfiguration.class); + Map res = ExtenderUtil.mergeContexts(ext1.platforms.get("linux").context, ext2.platforms.get("linux").context); + res = ExtenderUtil.mergeContexts(res, appManifest.platforms.get("linux").context); + List libs = (List)res.get("libs"); + List symbols = (List)res.get("symbols"); + assertTrue(libs.contains("profile")); + assertTrue(libs.contains("profilerext")); + assertTrue(libs.contains("profiler_remotery")); + + assertFalse(libs.contains("profile_null")); + assertFalse(libs.contains("profilerext_null")); + assertFalse(libs.contains("record_null")); + + assertTrue(symbols.contains("ProfilerExt")); + assertTrue(symbols.contains("ProfilerBasic")); + assertTrue(symbols.contains("ProfilerRemotery")); + } } diff --git a/server/test-data/appmanifests/app.appmanifest b/server/test-data/appmanifests/app.appmanifest new file mode 100644 index 00000000..c5f728f7 --- /dev/null +++ b/server/test-data/appmanifests/app.appmanifest @@ -0,0 +1,6 @@ +platforms: + linux: + context: + excludeLibs: ["profile_null", "profilerext_null", "record_null"] + libs: ["profile", "profilerext", "profiler_remotery"] + symbols: ['ProfilerExt', 'ProfilerBasic', 'ProfilerRemotery'] diff --git a/server/test-data/appmanifests/ext1.manifest b/server/test-data/appmanifests/ext1.manifest new file mode 100644 index 00000000..12cb63e4 --- /dev/null +++ b/server/test-data/appmanifests/ext1.manifest @@ -0,0 +1,5 @@ +platforms: + linux: + context: + libs: ["profile", "profilerext", "profiler_remotery"] + symbols: ['ProfilerExt', 'ProfilerBasic', 'ProfilerRemotery'] diff --git a/server/test-data/appmanifests/ext2.manifest b/server/test-data/appmanifests/ext2.manifest new file mode 100644 index 00000000..35adab5c --- /dev/null +++ b/server/test-data/appmanifests/ext2.manifest @@ -0,0 +1,10 @@ +platforms: + common: + context: + defines: ["DM_RELEASE"] + + linux: + context: + excludeLibs: ["engine", "engine_service", "profile", "profilerext", "profiler_remotery", "record", "vpx"] + libs: ["engine_release", "engine_service_null", "profile_null", "profilerext_null", "record_null"] + excludeSymbols: ['ProfilerExt', 'ProfilerBasic', 'ProfilerRemotery'] From aaddc4267f0d607c7e53f013799c728fde284c43 Mon Sep 17 00:00:00 2001 From: Kharkunov Eugene Date: Wed, 12 Nov 2025 18:59:57 +0300 Subject: [PATCH 2/4] Change how contexts are merged --- server/envs/android.env | 44 ------------------- .../com/defold/extender/ExtenderUtil.java | 10 +++-- 2 files changed, 7 insertions(+), 47 deletions(-) delete mode 100644 server/envs/android.env diff --git a/server/envs/android.env b/server/envs/android.env deleted file mode 100644 index e67f9d93..00000000 --- a/server/envs/android.env +++ /dev/null @@ -1,44 +0,0 @@ -GRADLE_USER_HOME=/tmp/.gradle -GRADLE_VERSION=8.7 -GRADLE_PLUGIN_VERSION=8.6.1 - -ANDROID_ROOT=/Users/yauheni/Library/Android/sdk -ANDROID_HOME=/Users/yauheni/Library/Android/sdk -ANDROID_SDK_ROOT=/Users/yauheni/Library/Android/sdk -ANDROID_SDK_HOME=/Users/yauheni/Library/Android/sdk - -# -# SDK 35 (Defold 1.9.8): -# - -ANDROID_BUILD_TOOLS_VERSION_35=35.0.0 -ANDROID_SDK_VERSION_35=35 - -ANDROID_SDK_BUILD_TOOLS_PATH_35=/Users/yauheni/Library/Android/sdk/build-tools/35.0.0 -ENV ANDROID_LIBRARYJAR_35=/Users/yauheni/Library/Android/sdk/platforms/android-35/android.jar - -# We specify it in build_input.yml by setting it the first in PATH for new SDK -# But at least one SDK should be specified by default in Dockerfile -ANDROID_SDK_VERSION=35 - -ENV ANDROID_NDK25_VERSION=25b -ENV ANDROID_NDK25_API_VERSION=19 -ENV ANDROID_64_NDK25_API_VERSION=21 - -# These paths are the same for both the 32 and 64 bit toolchains -ANDROID_NDK25_PATH=/Users/yauheni/Library/Android/sdk/ndk/25.1.8937393 -ANDROID_NDK25_BIN_PATH=/Users/yauheni/Library/Android/sdk/ndk/25.1.8937393/toolchains/llvm/prebuilt/darwin-x86_64/bin -ANDROID_NDK25_SYSROOT=/Users/yauheni/Library/Android/sdk/ndk/25.1.8937393/toolchains/llvm/prebuilt/darwin-x86_64/sysroot - -# We specify it in build_input.yml by setting it the first in PATH - ---------ENV ANDROID_PROGUARD=/usr/share/java/proguard.jar - -# download and install R8 utility separatly. It's done in that way because -# R8/D8 received fixes from time to time which not populated with build tools -# It's ok that R8.jar downloaded but saved to D8.jar. Can't find D8 as separate utility. -# R8 contains all classes of D8. -ENV R8_VERSION=8.7.0-dev -RUN \ - wget -O ${ANDROID_SDK_BUILD_TOOLS_PATH_34}/lib/d8.jar https://storage.googleapis.com/r8-releases/raw/${R8_VERSION}/r8.jar && \ - wget -O ${ANDROID_SDK_BUILD_TOOLS_PATH_35}/lib/d8.jar https://storage.googleapis.com/r8-releases/raw/${R8_VERSION}/r8.jar diff --git a/server/src/main/java/com/defold/extender/ExtenderUtil.java b/server/src/main/java/com/defold/extender/ExtenderUtil.java index d1cb20fa..74140c72 100644 --- a/server/src/main/java/com/defold/extender/ExtenderUtil.java +++ b/server/src/main/java/com/defold/extender/ExtenderUtil.java @@ -566,7 +566,9 @@ private static class PruneMapping { new PruneMapping("engineJsLibs", "includeJsLibs", "excludeJsLibs"), new PruneMapping("objectFiles", "includeObjectFiles", "excludeObjectFiles"), new PruneMapping("dynamicLibs", "includeDynamicLibs", "excludeDynamicLibs"), - new PruneMapping("symbols", "includeSymbols", "excludeSymbols") + new PruneMapping("symbols", "includeSymbols", "excludeSymbols"), + new PruneMapping("jars", "includeJars", "excludeJars"), + new PruneMapping("frameworks", "includeFrameworks", "excludeFrameworks") ); // Copies the original context, and appends the extra context's elements, if the keys and types are valid @@ -574,13 +576,15 @@ static public Map mergeContexts(Map a, Map context = mergeMaps(a, b); for (PruneMapping mapping : MAPPINGS) { + context.remove(mapping.includeName); + context.remove(mapping.excludeName); List srcList = ExtenderUtil.getStringList(context, mapping.targetName); if (srcList.isEmpty()) continue; context.put(mapping.targetName, ExtenderUtil.pruneItems(srcList, - ExtenderUtil.getStringList(context, mapping.includeName), - ExtenderUtil.getStringList(context, mapping.excludeName)) ); + ExtenderUtil.getStringList(b, mapping.includeName), + ExtenderUtil.getStringList(b, mapping.excludeName)) ); } return context; } From 298db7cad031482b616338b022c352e0b14657e4 Mon Sep 17 00:00:00 2001 From: Kharkunov Eugene Date: Wed, 12 Nov 2025 20:14:26 +0300 Subject: [PATCH 3/4] Fix tests --- server/src/main/java/com/defold/extender/ExtenderUtil.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/src/main/java/com/defold/extender/ExtenderUtil.java b/server/src/main/java/com/defold/extender/ExtenderUtil.java index 74140c72..3ff52afe 100644 --- a/server/src/main/java/com/defold/extender/ExtenderUtil.java +++ b/server/src/main/java/com/defold/extender/ExtenderUtil.java @@ -576,8 +576,6 @@ static public Map mergeContexts(Map a, Map context = mergeMaps(a, b); for (PruneMapping mapping : MAPPINGS) { - context.remove(mapping.includeName); - context.remove(mapping.excludeName); List srcList = ExtenderUtil.getStringList(context, mapping.targetName); if (srcList.isEmpty()) continue; From 9e575de6b371b4b15a313caeeaba628454beb2f0 Mon Sep 17 00:00:00 2001 From: Kharkunov Eugene Date: Wed, 12 Nov 2025 20:33:32 +0300 Subject: [PATCH 4/4] Update test case --- server/src/main/java/com/defold/extender/ExtenderUtil.java | 5 ++++- .../src/test/java/com/defold/extender/ExtenderUtilTest.java | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/com/defold/extender/ExtenderUtil.java b/server/src/main/java/com/defold/extender/ExtenderUtil.java index 3ff52afe..7d7ea5f9 100644 --- a/server/src/main/java/com/defold/extender/ExtenderUtil.java +++ b/server/src/main/java/com/defold/extender/ExtenderUtil.java @@ -567,7 +567,10 @@ private static class PruneMapping { new PruneMapping("objectFiles", "includeObjectFiles", "excludeObjectFiles"), new PruneMapping("dynamicLibs", "includeDynamicLibs", "excludeDynamicLibs"), new PruneMapping("symbols", "includeSymbols", "excludeSymbols"), - new PruneMapping("jars", "includeJars", "excludeJars"), + // if applied manifest contains 'symbols' which were excluded before - need update 'excludeSymbols' list + // because depends on that list we defined which extension to build and which symbols should be included into + // result binary + new PruneMapping("excludeSymbols", "", "symbols"), new PruneMapping("frameworks", "includeFrameworks", "excludeFrameworks") ); diff --git a/server/src/test/java/com/defold/extender/ExtenderUtilTest.java b/server/src/test/java/com/defold/extender/ExtenderUtilTest.java index 58b60906..a547270a 100644 --- a/server/src/test/java/com/defold/extender/ExtenderUtilTest.java +++ b/server/src/test/java/com/defold/extender/ExtenderUtilTest.java @@ -331,6 +331,7 @@ public void testMergeManifests() throws IOException, ExtenderException { res = ExtenderUtil.mergeContexts(res, appManifest.platforms.get("linux").context); List libs = (List)res.get("libs"); List symbols = (List)res.get("symbols"); + List excludeSymbols = (List)res.get("excludeSymbols"); assertTrue(libs.contains("profile")); assertTrue(libs.contains("profilerext")); assertTrue(libs.contains("profiler_remotery")); @@ -342,5 +343,9 @@ public void testMergeManifests() throws IOException, ExtenderException { assertTrue(symbols.contains("ProfilerExt")); assertTrue(symbols.contains("ProfilerBasic")); assertTrue(symbols.contains("ProfilerRemotery")); + + assertFalse(excludeSymbols.contains("ProfilerExt")); + assertFalse(excludeSymbols.contains("ProfilerBasic")); + assertFalse(excludeSymbols.contains("ProfilerRemotery")); } }