Skip to content

Commit 142e9bd

Browse files
ekharkunovekharkunov
committed
Implement source code as archive uploading for remote builder. Move some constants to separate class
1 parent e85a268 commit 142e9bd

8 files changed

Lines changed: 67 additions & 43 deletions

File tree

‎server/envs/macos.env‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
SWIFT_5_5_VERSION=5.5
22
IOS_VERSION_MIN=11.0
3-
MACOS_VERSION_MIN=10.13
3+
MACOS_VERSION_MIN=10.15
44
XCODE_16_VERSION=16.2
55
XCODE_16_CLANG_VERSION=16.0.0
66
MACOS_15_VERSION=15.2

‎server/src/main/java/com/defold/extender/Extender.java‎

Lines changed: 10 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -83,11 +83,6 @@ class Extender {
8383
static final String FOLDER_COMMON_SRC = "commonsrc";// common source shared between both types
8484

8585
static final String APPMANIFEST_FILENAME = "app.manifest";
86-
static final String FRAMEWORK_RE = "(.+)\\.framework";
87-
static final String JAR_RE = "(.+\\.jar)";
88-
static final String JS_RE = "(.+\\.js)";
89-
static final String PROTO_RE = "(?i).*(\\.proto)";
90-
static final String ENGINE_JAR_RE = "(?:.*)\\/share\\/java\\/[\\w\\-\\.]*\\.jar$";
9186

9287
private static final String MANIFEST_IOS = "Info.plist";
9388
private static final String MANIFEST_OSX = "Info.plist";
@@ -409,9 +404,9 @@ private Map<String, Object> createContext(Map<String, Object> src) throws Extend
409404
private List<String> getFrameworks(File dir) {
410405
List<String> frameworks = new ArrayList<>();
411406
final String[] platformParts = buildState.fullPlatform.split("-");
412-
frameworks.addAll(ExtenderUtil.collectDirsByName(new File(dir, "lib" + File.separator + buildState.fullPlatform), FRAMEWORK_RE)); // e.g. armv64-ios
407+
frameworks.addAll(ExtenderUtil.collectDirsByName(new File(dir, "lib" + File.separator + buildState.fullPlatform), ExtenderConst.FRAMEWORK_RE)); // e.g. armv64-ios
413408
if (platformParts.length == 2) {
414-
frameworks.addAll(ExtenderUtil.collectDirsByName(new File(dir, "lib" + File.separator + platformParts[1]), FRAMEWORK_RE)); // e.g. "ios"
409+
frameworks.addAll(ExtenderUtil.collectDirsByName(new File(dir, "lib" + File.separator + platformParts[1]), ExtenderConst.FRAMEWORK_RE)); // e.g. "ios"
415410
}
416411
return frameworks;
417412
}
@@ -439,10 +434,10 @@ private List<String> getFrameworkPaths(File dir) {
439434

440435
private List<String> getExtensionLibJars(File extDir) {
441436
List<String> jars = new ArrayList<>();
442-
jars.addAll(ExtenderUtil.collectFilesByPath(new File(extDir, "lib" + File.separator + buildState.fullPlatform), JAR_RE)); // e.g. armv7-android
437+
jars.addAll(ExtenderUtil.collectFilesByPath(new File(extDir, "lib" + File.separator + buildState.fullPlatform), ExtenderConst.JAR_RE)); // e.g. armv7-android
443438
String[] platformParts = buildState.fullPlatform.split("-");
444439
if (platformParts.length == 2) {
445-
jars.addAll(ExtenderUtil.collectFilesByPath(new File(extDir, "lib" + File.separator + platformParts[1]), JAR_RE)); // e.g. "android"
440+
jars.addAll(ExtenderUtil.collectFilesByPath(new File(extDir, "lib" + File.separator + platformParts[1]), ExtenderConst.JAR_RE)); // e.g. "android"
446441
}
447442
return jars;
448443
}
@@ -1229,7 +1224,7 @@ private List<File> buildExtensionInternal(File manifest, Map<String, Object> man
12291224
srcFiles.addAll(ExtenderUtil.listFiles(srcDirs, platformConfig.zigSourceRe));
12301225

12311226
// Generate C++ files first (output into the source folder)
1232-
List<File> protoFiles = ExtenderUtil.listFiles(srcDirs, PROTO_RE);
1227+
List<File> protoFiles = ExtenderUtil.listFiles(srcDirs, ExtenderConst.PROTO_RE);
12331228
List<File> generated = generateProtoCxxForEngine(extDir, manifestContext, protoFiles);
12341229
if (!protoFiles.isEmpty() && generated.isEmpty()) {
12351230
throw new ExtenderException(String.format("%s:1: error: Protofiles didn't generate any output engine cpp files!", ExtenderUtil.getRelativePath(buildState.uploadDir, protoFiles.get(0)) ));
@@ -1285,7 +1280,7 @@ private List<File> buildPipelineExtension(File manifest, Map<String, Object> man
12851280
extBuildDir.mkdir();
12861281
File[] srcDirs = { new File(extDir, FOLDER_COMMON_SRC), new File(extDir, FOLDER_PLUGIN_SRC) };
12871282

1288-
List<File> protoFiles = ExtenderUtil.listFiles(srcDirs, PROTO_RE);
1283+
List<File> protoFiles = ExtenderUtil.listFiles(srcDirs, ExtenderConst.PROTO_RE);
12891284

12901285
List<File> outputFiles = new ArrayList<>();
12911286

@@ -1425,7 +1420,7 @@ private void getProjectPaths(Map<String, Object> mainContext, Map<String, Object
14251420

14261421
extShLibs.addAll(ExtenderUtil.collectFilesByName(libDir, platformConfig.shlibRe));
14271422
extLibs.addAll(ExtenderUtil.collectFilesByName(libDir, platformConfig.stlibRe));
1428-
extJsLibs.addAll(ExtenderUtil.collectFilesByPath(libDir, JS_RE));
1423+
extJsLibs.addAll(ExtenderUtil.collectFilesByPath(libDir, ExtenderConst.JS_RE));
14291424

14301425
extFrameworks.addAll(getFrameworks(extDir));
14311426

@@ -1440,7 +1435,7 @@ private void getProjectPaths(Map<String, Object> mainContext, Map<String, Object
14401435

14411436
extShLibs.addAll(ExtenderUtil.collectFilesByName(libCommonDir, platformConfig.shlibRe));
14421437
extLibs.addAll(ExtenderUtil.collectFilesByName(libCommonDir, platformConfig.stlibRe));
1443-
extJsLibs.addAll(ExtenderUtil.collectFilesByPath(libCommonDir, JS_RE));
1438+
extJsLibs.addAll(ExtenderUtil.collectFilesByPath(libCommonDir, ExtenderConst.JS_RE));
14441439
extFrameworkPaths.addAll(getFrameworkPaths(extDir));
14451440
}
14461441
}
@@ -2140,10 +2135,10 @@ public static Map<String, Object> getManifestContext(String platform, ManifestCo
21402135
private File buildMainDexList(List<String> jars) throws ExtenderException {
21412136

21422137
// Find the engine libraries (**/share/java/*.jar)
2143-
List<String> mainListJars = ExtenderUtil.filterStrings(jars, ENGINE_JAR_RE);
2138+
List<String> mainListJars = ExtenderUtil.filterStrings(jars, ExtenderConst.ENGINE_JAR_RE);
21442139

21452140
if (mainListJars.isEmpty()) {
2146-
throw new ExtenderException("Regex failed to find any engine jars: " + ENGINE_JAR_RE);
2141+
throw new ExtenderException("Regex failed to find any engine jars: " + ExtenderConst.ENGINE_JAR_RE);
21472142
}
21482143

21492144
List<String> mainClassNames = new ArrayList<String>();

‎server/src/main/java/com/defold/extender/ExtenderConst.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
package com.defold.extender;
2+
3+
public class ExtenderConst {
4+
public static final String SOURCE_CODE_ARCHIVE_MAGIC_NAME = "__source_code__.zip";
5+
public static final String FRAMEWORK_RE = "(.+)\\.framework";
6+
public static final String JAR_RE = "(.+\\.jar)";
7+
public static final String JS_RE = "(.+\\.js)";
8+
public static final String PROTO_RE = "(?i).*(\\.proto)";
9+
public static final String ENGINE_JAR_RE = "(?:.*)\\/share\\/java\\/[\\w\\-\\.]*\\.jar$";
10+
11+
}

‎server/src/main/java/com/defold/extender/ExtenderController.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,6 @@ public class ExtenderController {
5656
// Used to verify the uploaded filenames
5757
private static final Pattern FILENAME_RE = Pattern.compile("^([\\w ](?:[\\w+\\-\\/ @]|(?:\\.[\\w+\\-\\/ ]*))+)$");
5858

59-
private static final String SOURCE_CODE_ARCHIVE_MAGIC_NAME = "__source_code__.zip";
60-
6159
private final DefoldSdkService defoldSdkService;
6260
private final DataCacheService dataCacheService;
6361
private final MeterRegistry meterRegistry;
@@ -388,7 +386,7 @@ static void receiveUpload(MultipartHttpServletRequest request, File uploadDirect
388386
}
389387

390388
// check if source code archive presented
391-
File sourceCodeArchive = new File(uploadDirectory, SOURCE_CODE_ARCHIVE_MAGIC_NAME);
389+
File sourceCodeArchive = new File(uploadDirectory, ExtenderConst.SOURCE_CODE_ARCHIVE_MAGIC_NAME);
392390
if (sourceCodeArchive.exists()) {
393391
LOGGER.debug("Source code archive found. Unarchiving...");
394392
ZipUtils.unzip(new FileInputStream(sourceCodeArchive), uploadDirectory.toPath());

‎server/src/main/java/com/defold/extender/remote/RemoteEngineBuilder.java‎

Lines changed: 37 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
package com.defold.extender.remote;
22

33
import com.defold.extender.BuilderConstants;
4+
import com.defold.extender.ExtenderConst;
5+
import com.defold.extender.ExtenderException;
46
import com.defold.extender.metrics.MetricsWriter;
7+
import com.defold.extender.services.DataCacheService;
58
import com.defold.extender.services.GCPInstanceService;
69
import com.defold.extender.tracing.ExtenderTracerInterceptor;
710

@@ -19,10 +22,8 @@
1922
import org.apache.http.client.HttpClient;
2023
import org.apache.http.client.methods.HttpGet;
2124
import org.apache.http.client.methods.HttpPost;
22-
import org.apache.http.entity.ContentType;
2325
import org.apache.http.entity.mime.MultipartEntityBuilder;
24-
import org.apache.http.entity.mime.content.FileBody;
25-
import org.apache.http.entity.mime.content.AbstractContentBody;
26+
import org.apache.http.entity.mime.content.ByteArrayBody;
2627
import org.apache.http.impl.client.HttpClientBuilder;
2728
import org.apache.http.util.EntityUtils;
2829
import org.slf4j.Logger;
@@ -34,13 +35,17 @@
3435

3536
import java.io.OutputStream;
3637
import java.io.PrintWriter;
38+
import java.io.ByteArrayOutputStream;
3739
import java.io.File;
3840
import java.io.FileNotFoundException;
3941
import java.io.FileOutputStream;
4042
import java.io.IOException;
4143
import java.nio.file.Files;
44+
import java.nio.file.Path;
4245
import java.nio.file.StandardCopyOption;
4346
import java.util.Optional;
47+
import java.util.zip.ZipEntry;
48+
import java.util.zip.ZipOutputStream;
4449
import java.nio.charset.Charset;
4550

4651
@Service
@@ -91,7 +96,7 @@ public void buildAsync(final RemoteInstanceConfig remoteInstanceConfig,
9196

9297
try {
9398
httpEntity = buildHttpEntity(projectDirectory);
94-
} catch(IllegalStateException|IOException e) {
99+
} catch(IllegalStateException | IOException | ExtenderException e) {
95100
throw new RemoteBuildException("Failed to add files to multipart request", e);
96101
}
97102

@@ -176,19 +181,34 @@ public void buildAsync(final RemoteInstanceConfig remoteInstanceConfig,
176181
}
177182
}
178183

179-
HttpEntity buildHttpEntity(final File projectDirectory) throws IOException {
180-
final MultipartEntityBuilder builder = MultipartEntityBuilder.create()
181-
.setContentType(ContentType.MULTIPART_FORM_DATA);
182-
183-
Files.walk(projectDirectory.toPath())
184-
.filter(Files::isRegularFile)
185-
.forEach(path -> {
186-
String relativePath = path.toFile().getAbsolutePath().substring(projectDirectory.getAbsolutePath().length() + 1);
187-
AbstractContentBody body = new FileBody(path.toFile(), ContentType.DEFAULT_BINARY, relativePath);
188-
builder.addPart(relativePath, body);
189-
});
190-
191-
return builder.build();
184+
HttpEntity buildHttpEntity(final File projectDirectory) throws IOException, ExtenderException {
185+
MultipartEntityBuilder entityBuilder = MultipartEntityBuilder.create();
186+
entityBuilder.setStrictMode();
187+
188+
ByteArrayOutputStream byteStream = new ByteArrayOutputStream();
189+
try (ZipOutputStream zipStream = new ZipOutputStream(byteStream)) {
190+
Path projectDirectoryPath = projectDirectory.toPath();
191+
Files.walk(projectDirectoryPath)
192+
.filter(Files::isRegularFile)
193+
.filter(path -> { return !path.getFileName().toString().equals(ExtenderConst.SOURCE_CODE_ARCHIVE_MAGIC_NAME); })
194+
.filter(path -> { return !path.getFileName().toString().equals(DataCacheService.FILE_CACHE_INFO_FILE); })
195+
.forEach(res -> {
196+
Path path = projectDirectoryPath.relativize(res);
197+
LOGGER.warn(path.toString());
198+
try {
199+
ZipEntry entry = new ZipEntry(path.toString());
200+
zipStream.putNextEntry(entry);
201+
zipStream.write(Files.readAllBytes(res));
202+
zipStream.closeEntry();
203+
} catch (IOException e) {
204+
LOGGER.warn("Exception during add entry to source code archive", e);
205+
}
206+
});
207+
} catch (IOException exc) {
208+
throw new ExtenderException(exc, "Failed to create source code archive");
209+
}
210+
entityBuilder.addPart(ExtenderConst.SOURCE_CODE_ARCHIVE_MAGIC_NAME, new ByteArrayBody(byteStream.toByteArray(), ExtenderConst.SOURCE_CODE_ARCHIVE_MAGIC_NAME));
211+
return entityBuilder.build();
192212
}
193213

194214
private void touchInstance(String instanceId) {

‎server/src/main/java/com/defold/extender/services/DataCacheService.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ public class DataCacheService {
3030

3131
private static final Logger LOGGER = LoggerFactory.getLogger(DataCacheService.class);
3232

33-
static final String FILE_CACHE_INFO_FILE = "ne-cache-info.json";
33+
public static final String FILE_CACHE_INFO_FILE = "ne-cache-info.json";
3434
static final String FILE_CACHE_INFO_HASH_TYPE = "sha256";
3535
static final int FILE_CACHE_INFO_VERSION = 1;
3636

‎server/src/main/java/com/defold/extender/services/cocoapods/ResolvedPods.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,14 @@
1616
import org.slf4j.Logger;
1717
import org.slf4j.LoggerFactory;
1818

19+
import com.defold.extender.ExtenderConst;
1920
import com.defold.extender.ExtenderException;
2021
import com.defold.extender.ExtenderUtil;
2122
import com.defold.extender.services.cocoapods.PlistBuddyWrapper.CreateBundlePlistArgs;
2223
import com.defold.extender.utils.FrameworkUtil;
2324

2425
public class ResolvedPods {
2526
private static final Logger LOGGER = LoggerFactory.getLogger(ResolvedPods.class);
26-
static final String FRAMEWORK_RE = "(.+)\\.framework";
2727
private List<PodBuildSpec> pods = new ArrayList<>();
2828
private File podsDir;
2929
private File frameworksDir;
@@ -127,7 +127,7 @@ List<String> collectAllPodFrameworks() throws IOException {
127127
}
128128

129129
// collect unpacked xcframeworks
130-
Pattern pattern = Pattern.compile(FRAMEWORK_RE);
130+
Pattern pattern = Pattern.compile(ExtenderConst.FRAMEWORK_RE);
131131
Files.walk(frameworksDir.toPath())
132132
.filter(Files::isDirectory)
133133
.forEach(path -> {
@@ -143,7 +143,7 @@ List<String> collectAllPodFrameworks() throws IOException {
143143
List<File> collectAllPodsDynamicFrameworks() throws IOException {
144144
Set<File> dynamicFrameworks = new HashSet<>();
145145
// collect unpacked xcframeworks
146-
Pattern pattern = Pattern.compile(FRAMEWORK_RE);
146+
Pattern pattern = Pattern.compile(ExtenderConst.FRAMEWORK_RE);
147147
Files.walk(frameworksDir.toPath())
148148
.filter(Files::isDirectory)
149149
.forEach(path -> {

‎server/src/test/java/com/defold/extender/ExtenderTest.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -222,7 +222,7 @@ public void testCollectLibraries() {
222222
checkArray(expected, result);
223223
}
224224
{
225-
List<String> result = ExtenderUtil.collectDirsByName(new File("test-data/ext/lib/x86_64-osx"), Extender.FRAMEWORK_RE);
225+
List<String> result = ExtenderUtil.collectDirsByName(new File("test-data/ext/lib/x86_64-osx"), ExtenderConst.FRAMEWORK_RE);
226226
String[] expected = {"blib"};
227227
checkArray(expected, result);
228228
}
@@ -238,7 +238,7 @@ public void testCollectJars() {
238238
String[] endings = {"test-data/ext/lib/android/Dummy.jar", "test-data/ext/lib/android/JarDep.jar",
239239
"test-data/ext/lib/android/VeryLarge1.jar", "test-data/ext/lib/android/VeryLarge2.jar",
240240
"test-data/ext/lib/android/meta-inf.jar"};
241-
List<String> paths = ExtenderUtil.collectFilesByPath(new File("test-data/ext/lib/android"), Extender.JAR_RE);
241+
List<String> paths = ExtenderUtil.collectFilesByPath(new File("test-data/ext/lib/android"), ExtenderConst.JAR_RE);
242242
assertEquals(endings.length, paths.size());
243243

244244

@@ -256,7 +256,7 @@ public void testCollectJars() {
256256

257257
@Test
258258
public void testCollectJsFiles() {
259-
List<String> result = ExtenderUtil.collectFilesByPath(new File("test-data/ext/lib/js-web"), Extender.JS_RE);
259+
List<String> result = ExtenderUtil.collectFilesByPath(new File("test-data/ext/lib/js-web"), ExtenderConst.JS_RE);
260260
assertEquals(1, result.size());
261261
assertTrue(result.get(0).endsWith("test-data/ext/lib/js-web/library_dummy.js"));
262262
}

0 commit comments

Comments
 (0)