diff --git a/client/build.gradle b/client/build.gradle index 2474a0ac..f8179e3a 100644 --- a/client/build.gradle +++ b/client/build.gradle @@ -40,6 +40,8 @@ dependencies { testImplementation('commons-io:commons-io:2.18.0') testImplementation("org.mockito:mockito-core:5.+") testImplementation('org.junit.jupiter:junit-jupiter-engine:5.11.4') + testImplementation('org.junit.jupiter:junit-jupiter-params:5.11.4') + testImplementation('org.apache.james:apache-mime4j:0.8.13') } test { diff --git a/client/src/main/java/com/defold/extender/client/ExtenderClient.java b/client/src/main/java/com/defold/extender/client/ExtenderClient.java index 635e256b..5420732d 100644 --- a/client/src/main/java/com/defold/extender/client/ExtenderClient.java +++ b/client/src/main/java/com/defold/extender/client/ExtenderClient.java @@ -24,12 +24,15 @@ import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; import java.io.OutputStream; import java.io.UnsupportedEncodingException; import java.math.BigInteger; +import java.nio.file.Files; +import java.nio.file.Path; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.List; @@ -38,6 +41,8 @@ import java.util.Set; import java.util.concurrent.TimeoutException; import java.util.regex.Pattern; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; import java.util.Iterator; import java.util.Base64; import java.util.logging.Level; @@ -47,10 +52,11 @@ public class ExtenderClient { private static Logger logger = Logger.getLogger(ExtenderClient.class.getName()); private static final String TRACE_ID_HEADER_NAME = "X-TraceId"; + static final String SOURCE_CODE_ARCHIVE_MAGIC_NAME = "__source_code__.zip"; + static final String CACHE_INFO_NAME = "ne-cache-info.json"; private final String extenderBaseUrl; private ExtenderClientCache cache; - private CookieStore httpCookies; private long buildSleepTimeout; private long buildResultWaitTimeout; private List headers; @@ -67,18 +73,34 @@ public class ExtenderClient { * @param cacheDir A directory where the cache files are located (it must exist beforehand) */ public ExtenderClient(String extenderBaseUrl, File cacheDir) throws IOException { + this(new BasicCookieStore(), extenderBaseUrl, cacheDir); + } + + public ExtenderClient(CookieStore cookieStore, + String extenderBaseUrl, + File cacheDir) throws IOException { + this(new ExtenderClientCache(cacheDir), cookieStore, + HttpClientBuilder.create() + .setDefaultRequestConfig( + RequestConfig.custom() + .setExpectContinueEnabled(true) + .build() + ) + .setDefaultCookieStore(cookieStore) + .build() + , extenderBaseUrl); + } + + public ExtenderClient(ExtenderClientCache cache, + CookieStore cookieStore, + HttpClient httpClient, + String extenderBaseUrl) { this.extenderBaseUrl = extenderBaseUrl; - this.cache = new ExtenderClientCache(cacheDir); - this.httpCookies = new BasicCookieStore(); + this.cache = cache; this.buildSleepTimeout = Long.parseLong(System.getProperty("com.defold.extender.client.build-sleep-timeout", "5000")); this.buildResultWaitTimeout = Long.parseLong(System.getProperty("com.defold.extender.client.build-wait-timeout", "1200000")); this.headers = new ArrayList(); - RequestConfig requestConfig = RequestConfig.custom() - .setExpectContinueEnabled(true).build(); - HttpClientBuilder clientBuilder = HttpClientBuilder.create() - .setDefaultRequestConfig(requestConfig) - .setDefaultCookieStore(httpCookies); - this.httpClient = clientBuilder.build(); + this.httpClient = httpClient; } private void log(String s, Object... args) { @@ -174,14 +196,11 @@ String queryCache(List sourceResources) throws ExtenderClientE try { String url = String.format("%s/query", extenderBaseUrl); - HttpPost request = new HttpPost(url); + HttpPost request = createPostRequest(url); request.setEntity(new ByteArrayEntity(data.getBytes())); request.setHeader("Accept", "application/json"); request.setHeader("Content-type", "application/json"); - addAuthorizationHeader(request); - addHeaders(request); - HttpResponse response = httpClient.execute(request); if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { @@ -196,7 +215,7 @@ String queryCache(List sourceResources) throws ExtenderClientE } // Gets a set of files that are currently cached - private static Set getCachedFiles(String json) throws ExtenderClientException { + static Set getCachedFiles(String json) throws ExtenderClientException { Set cachedFiles = new HashSet<>(); JSONObject root = null; @@ -224,14 +243,11 @@ private static Set getCachedFiles(String json) throws ExtenderClientExce private void build_async(String platform, String sdkVersion, HttpEntity entity, File destination, File log) throws ExtenderClientException { try { String url = String.format("%s/build_async/%s/%s", extenderBaseUrl, platform, sdkVersion); - HttpPost request = new HttpPost(url); + HttpPost request = createPostRequest(url); request.setEntity(entity); log("Sending async build request to %s", url); - addAuthorizationHeader(request); - addHeaders(request); - HttpResponse response = httpClient.execute(request); StatusLine statusLine = response.getStatusLine(); @@ -284,7 +300,42 @@ private void build_async(String platform, String sdkVersion, HttpEntity entity, } catch (Exception e) { throw new ExtenderClientException("Failed to communicate with Extender service.", e); } + } + + HttpEntity createBuildRequestPayload(List sourceResources) throws ExtenderClientException { + MultipartEntityBuilder entityBuilder = MultipartEntityBuilder.create(); + entityBuilder.setStrictMode(); + // // Now, let's ask the server what files it already has + String cacheInfoJson = queryCache(sourceResources); + final Set cachedFiles = cacheInfoJson != null ? getCachedFiles(cacheInfoJson) : Set.of(); + if (cacheInfoJson != null) { + // add the updated info to the file + entityBuilder.addPart(CACHE_INFO_NAME, new ByteArrayBody(cacheInfoJson.getBytes(), CACHE_INFO_NAME)); // Same as specified in DataStoreService.java + } + + ByteArrayOutputStream byteStream = new ByteArrayOutputStream(); + try (ZipOutputStream zipStream = new ZipOutputStream(byteStream)) { + sourceResources.stream() + .filter(res -> { + return !cachedFiles.contains(res.getPath()); + }) + .forEach(res -> { + Path path = Path.of(res.getPath()); + try { + if (Files.isRegularFile(path)) { + ZipEntry entry = new ZipEntry(path.toString()); + zipStream.putNextEntry(entry); + Files.copy(path, zipStream); + zipStream.closeEntry(); + } + } catch (IOException e) { } + }); + } catch (IOException exc) { + throw new ExtenderClientException("Failed to create source code archive", exc); + } + entityBuilder.addPart(SOURCE_CODE_ARCHIVE_MAGIC_NAME, new ByteArrayBody(byteStream.toByteArray(), SOURCE_CODE_ARCHIVE_MAGIC_NAME)); + return entityBuilder.build(); } /** @@ -306,36 +357,8 @@ public void build(String platform, String sdkVersion, List sou return; } - MultipartEntityBuilder entityBuilder = MultipartEntityBuilder.create(); - entityBuilder.setStrictMode(); - - // Now, let's ask the server what files it already has - String cacheInfoName = "ne-cache-info.json"; - String cacheInfoJson = queryCache(sourceResources); - Set cachedFiles = new HashSet<>(); - if (cacheInfoJson != null) { - cachedFiles = getCachedFiles(cacheInfoJson); - - // add the updated info to the file - entityBuilder.addPart(cacheInfoName, new ByteArrayBody(cacheInfoJson.getBytes(), cacheInfoName)); // Same as specified in DataStoreService.java - } - - for (ExtenderResource s : sourceResources) { - // If the file was already cached, don't upload it - if (cachedFiles.contains(s.getPath())) { - continue; - } - - ByteArrayBody bin; - try { - bin = new ByteArrayBody(s.getContent(), s.getPath()); - } catch (IOException e) { - throw new ExtenderClientException("Error while getting content for " + s.getPath() + ": " + e.getMessage()); - } - entityBuilder.addPart(s.getPath(), bin); - } - - build_async(platform, sdkVersion, entityBuilder.build(), destination, log); + HttpEntity payload = createBuildRequestPayload(sourceResources); + build_async(platform, sdkVersion, payload, destination, log); // Store the new build cache.put(platform, cacheKey, destination); @@ -354,9 +377,7 @@ public String httpRequestToString(HttpRequestBase request) { } public boolean health() throws IOException { - HttpGet request = new HttpGet(extenderBaseUrl); - addAuthorizationHeader(request); - addHeaders(request); + HttpGet request = createGetRequest(extenderBaseUrl); HttpResponse response = httpClient.execute(request); EntityUtils.consumeQuietly(response.getEntity()); if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { @@ -364,4 +385,19 @@ public boolean health() throws IOException { } return false; } + + public HttpGet createGetRequest(String url) throws UnsupportedEncodingException { + HttpGet request = new HttpGet(extenderBaseUrl); + addAuthorizationHeader(request); + addHeaders(request); + return request; + } + + public HttpPost createPostRequest(String url) throws UnsupportedEncodingException { + HttpPost request = new HttpPost(url); + + addAuthorizationHeader(request); + addHeaders(request); + return request; + } } diff --git a/client/src/test/java/com/defold/extender/client/ExtenderClientCacheTest.java b/client/src/test/java/com/defold/extender/client/ExtenderClientCacheTest.java new file mode 100644 index 00000000..9cc71b2c --- /dev/null +++ b/client/src/test/java/com/defold/extender/client/ExtenderClientCacheTest.java @@ -0,0 +1,232 @@ +package com.defold.extender.client; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; + +import java.io.File; +import java.io.IOException; +import java.math.BigInteger; +import java.nio.file.Files; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Test; + +public class ExtenderClientCacheTest { + @Test + public void testClientCacheHash() throws IOException, InterruptedException, ExtenderClientException { + TestUtils.writeToFile("build/a", "a"); + TestUtils.writeToFile("build/b", "a"); + TestUtils.writeToFile("build/c", "b"); + + ExtenderClientCache cache = new ExtenderClientCache(new File(".")); + + { + File file1 = new File("build/a"); + File file2 = new File("build/b"); + File file3 = new File("build/c"); + FileExtenderResource file1Res = new FileExtenderResource(file1); + FileExtenderResource file2Res = new FileExtenderResource(file2); + FileExtenderResource file3Res = new FileExtenderResource(file3); + assertEquals(cache.getHash(file1Res), cache.getHash(file2Res)); + assertNotEquals(cache.getHash(file1Res), cache.getHash(file3Res)); + } + + Thread.sleep(1000); + + TestUtils.writeToFile("build/b", "b"); + + { + File file1 = new File("build/a"); + File file2 = new File("build/b"); + FileExtenderResource file1Res = new FileExtenderResource(file1); + FileExtenderResource file2Res = new FileExtenderResource(file2); + + assertNotEquals(cache.getHash(file1Res), cache.getHash(file2Res)); + } + + FileUtils.deleteQuietly(new File("build/a")); + FileUtils.deleteQuietly(new File("build/b")); + FileUtils.deleteQuietly(new File("build/c")); + } + + @Test + public void testClientCacheSignatureHash() throws IOException, ExtenderClientException { + File a = new File("build/a"); + File b = new File("build/b"); + FileExtenderResource aRes = new FileExtenderResource(a); + FileExtenderResource bRes = new FileExtenderResource(b); + + TestUtils.writeToFile("build/a", "a"); + TestUtils.writeToFile("build/b", "b"); + + List files1 = new ArrayList<>(); + files1.add(aRes); + files1.add(bRes); + + + List files2 = new ArrayList<>(); + files2.add(bRes); + files2.add(aRes); + + String platform = "osx"; + String sdkVersion = "abc456"; + ExtenderClientCache cache = new ExtenderClientCache(new File(".")); + + assertEquals(cache.getHash(files1), cache.getHash(files2)); + assertEquals(cache.calcKey(platform, sdkVersion, files1), cache.calcKey(platform, sdkVersion, files2)); + + files2.add(aRes); + + assertNotEquals(cache.getHash(files1), cache.getHash(files2)); + assertNotEquals(cache.calcKey(platform, sdkVersion, files1), cache.calcKey(platform, sdkVersion, files2)); + + FileUtils.deleteQuietly(new File("build/a")); + FileUtils.deleteQuietly(new File("build/b")); + } + + @Test + public void testClientCacheValidBuild() throws IOException, InterruptedException, ExtenderClientException { + File a = new File("build/a"); + File b = new File("build/b"); + File c = new File("build/c"); + FileExtenderResource aRes = new FileExtenderResource(a); + FileExtenderResource bRes = new FileExtenderResource(b); + FileExtenderResource cRes = new FileExtenderResource(c); + + a.deleteOnExit(); + b.deleteOnExit(); + c.deleteOnExit(); + + TestUtils.writeToFile("build/a", "a"); + TestUtils.writeToFile("build/b", "b"); + TestUtils.writeToFile("build/c", "c"); + + List files = new ArrayList<>(); + files.add(aRes); + files.add(bRes); + + String platform = "osx"; + String sdkVersion = "abc456"; + ExtenderClientCache cache = new ExtenderClientCache(new File(".")); + + if (cache.getCacheFile().exists()) { + cache.getCacheFile().delete(); + } + + String key = null; + // Is doesn't exist yet, so false + key = cache.calcKey(platform, sdkVersion, files); + assertEquals(false, cache.isCached(platform, key)); + + File build = cache.getCachedBuildFile(platform); + build.deleteOnExit(); + + TestUtils.writeToFile(build.getAbsolutePath(), (new Date()).toString()); + cache.put(platform, key, build); + + // It should exist now, so true + assertEquals(true, cache.isCached(platform, key)); + + // Changing a source file should invalidate the file + Thread.sleep(1000); + TestUtils.writeToFile("build/b", "bb"); + key = cache.calcKey(platform, sdkVersion, files); + + assertEquals(false, cache.isCached(platform, key)); + + // If we update the build, is should be cached + cache.put(platform, key, build); + assertEquals(true, cache.isCached(platform, key)); + + // Add a "new" file to the list, but let it have an old timestamp + files.add(cRes); + key = cache.calcKey(platform, sdkVersion, files); + + assertEquals(false, cache.isCached(platform, key)); + + // If we update the build, is should be cached + cache.put(platform, key, build); + assertEquals(true, cache.isCached(platform, key)); + + // Remove one file + files.remove(0); + key = cache.calcKey(platform, sdkVersion, files); + + assertEquals(false, cache.isCached(platform, key)); + + // If we update the build, is should be cached + cache.put(platform, key, build); + assertEquals(true, cache.isCached(platform, key)); + } + + private static String calcChecksum(File file) throws IOException, NoSuchAlgorithmException { + MessageDigest md = MessageDigest.getInstance("SHA-256"); + byte[] data = Files.readAllBytes(file.toPath()); + md.update(data); + byte[] digest = md.digest(); + return new BigInteger(1, digest).toString(16); + } + + @Test + public void testClientCachePersistence() throws IOException, ExtenderClientException, NoSuchAlgorithmException { + File a = new File("build/a"); + FileExtenderResource aRes = new FileExtenderResource(a); + a.deleteOnExit(); + TestUtils.writeToFile("build/a", "a"); + + List files = new ArrayList<>(); + files.add(aRes); + + String platform = "osx"; + String sdkVersion = "abc456"; + File cacheDir = new File("."); + + + String key; + { + ExtenderClientCache cache = new ExtenderClientCache(cacheDir); + key = cache.calcKey(platform, sdkVersion, files); + + if (cache.getCacheFile().exists()) { + cache.getCacheFile().delete(); + } + assertFalse(cache.getCacheFile().exists()); + } + + // Start with an empty cache + String checksum = null; + { + ExtenderClientCache cache = new ExtenderClientCache(cacheDir); + + assertEquals(false, cache.isCached(platform, key)); + + // Write the build, and update the cache + File build = File.createTempFile("test", "build"); + build.deleteOnExit(); + TestUtils.writeToFile(build.getAbsolutePath(), (new Date()).toString()); + cache.put(platform, key, build); + + checksum = calcChecksum(build); + } + + // Now, lets create another cache, and check that we get a cached version + { + ExtenderClientCache cache = new ExtenderClientCache(cacheDir); + + assertEquals(true, cache.isCached(platform, key)); + + File build = File.createTempFile("test2", "build"); + cache.get(platform, key, build); + + String checksum2 = calcChecksum(build); + + assertEquals(checksum, checksum2); + } + } +} diff --git a/client/src/test/java/com/defold/extender/client/ExtenderClientTest.java b/client/src/test/java/com/defold/extender/client/ExtenderClientTest.java index 554d8bc8..bfa674a5 100644 --- a/client/src/test/java/com/defold/extender/client/ExtenderClientTest.java +++ b/client/src/test/java/com/defold/extender/client/ExtenderClientTest.java @@ -1,6 +1,6 @@ package com.defold.extender.client; -import org.apache.commons.io.FileUtils; +import org.apache.http.HttpEntity; import org.apache.http.StatusLine; import org.apache.http.client.ClientProtocolException; import org.apache.http.client.entity.EntityBuilder; @@ -9,44 +9,40 @@ import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.util.EntityUtils; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.mockito.Mockito; + +import org.apache.james.mime4j.dom.BinaryBody; +import org.apache.james.mime4j.dom.Body; +import org.apache.james.mime4j.dom.Entity; +import org.apache.james.mime4j.dom.Message; +import org.apache.james.mime4j.dom.Multipart; +import org.apache.james.mime4j.message.DefaultMessageBuilder; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.ByteArrayInputStream; import java.io.File; -import java.io.FileWriter; import java.io.IOException; import java.lang.reflect.Field; import java.lang.reflect.InvocationTargetException; -import java.math.BigInteger; -import java.nio.file.Files; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.time.Instant; import java.util.ArrayList; -import java.util.Date; import java.util.List; -import java.util.regex.Matcher; +import java.util.stream.Stream; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; public class ExtenderClientTest extends Mockito { - - private void writeToFile(String path, String msg) throws IOException { - File f = new File(path); - FileWriter fwr = new FileWriter(f); - fwr.write(msg); - fwr.flush(); - fwr.close(); - f.setLastModified(Instant.now().toEpochMilli() + 23); - } - @BeforeAll public static void beforeClass() { File buildDir = new File("build"); @@ -59,331 +55,20 @@ public static void cleanup() { buildLog.delete(); } - @Test - public void testClientCacheHash() throws IOException, InterruptedException, ExtenderClientException { - writeToFile("build/a", "a"); - writeToFile("build/b", "a"); - writeToFile("build/c", "b"); - - ExtenderClientCache cache = new ExtenderClientCache(new File(".")); - - { - File file1 = new File("build/a"); - File file2 = new File("build/b"); - File file3 = new File("build/c"); - FileExtenderResource file1Res = new FileExtenderResource(file1); - FileExtenderResource file2Res = new FileExtenderResource(file2); - FileExtenderResource file3Res = new FileExtenderResource(file3); - assertEquals(cache.getHash(file1Res), cache.getHash(file2Res)); - assertNotEquals(cache.getHash(file1Res), cache.getHash(file3Res)); - } - - Thread.sleep(1000); - - writeToFile("build/b", "b"); - - { - File file1 = new File("build/a"); - File file2 = new File("build/b"); - FileExtenderResource file1Res = new FileExtenderResource(file1); - FileExtenderResource file2Res = new FileExtenderResource(file2); - - assertNotEquals(cache.getHash(file1Res), cache.getHash(file2Res)); - } - - FileUtils.deleteQuietly(new File("build/a")); - FileUtils.deleteQuietly(new File("build/b")); - FileUtils.deleteQuietly(new File("build/c")); - } - - @Test - public void testClientCacheSignatureHash() throws IOException, ExtenderClientException { - File a = new File("build/a"); - File b = new File("build/b"); - FileExtenderResource aRes = new FileExtenderResource(a); - FileExtenderResource bRes = new FileExtenderResource(b); - - writeToFile("build/a", "a"); - writeToFile("build/b", "b"); - - List files1 = new ArrayList<>(); - files1.add(aRes); - files1.add(bRes); - - - List files2 = new ArrayList<>(); - files2.add(bRes); - files2.add(aRes); - - String platform = "osx"; - String sdkVersion = "abc456"; - ExtenderClientCache cache = new ExtenderClientCache(new File(".")); - - assertEquals(cache.getHash(files1), cache.getHash(files2)); - assertEquals(cache.calcKey(platform, sdkVersion, files1), cache.calcKey(platform, sdkVersion, files2)); - - files2.add(aRes); - - assertNotEquals(cache.getHash(files1), cache.getHash(files2)); - assertNotEquals(cache.calcKey(platform, sdkVersion, files1), cache.calcKey(platform, sdkVersion, files2)); - - FileUtils.deleteQuietly(new File("build/a")); - FileUtils.deleteQuietly(new File("build/b")); - } - - @Test - public void testClientCacheValidBuild() throws IOException, InterruptedException, ExtenderClientException { - File a = new File("build/a"); - File b = new File("build/b"); - File c = new File("build/c"); - FileExtenderResource aRes = new FileExtenderResource(a); - FileExtenderResource bRes = new FileExtenderResource(b); - FileExtenderResource cRes = new FileExtenderResource(c); - - a.deleteOnExit(); - b.deleteOnExit(); - c.deleteOnExit(); - - writeToFile("build/a", "a"); - writeToFile("build/b", "b"); - writeToFile("build/c", "c"); - - List files = new ArrayList<>(); - files.add(aRes); - files.add(bRes); - - String platform = "osx"; - String sdkVersion = "abc456"; - ExtenderClientCache cache = new ExtenderClientCache(new File(".")); - - if (cache.getCacheFile().exists()) { - cache.getCacheFile().delete(); - } - - String key = null; - // Is doesn't exist yet, so false - key = cache.calcKey(platform, sdkVersion, files); - assertEquals(false, cache.isCached(platform, key)); - - File build = cache.getCachedBuildFile(platform); - build.deleteOnExit(); - - writeToFile(build.getAbsolutePath(), (new Date()).toString()); - cache.put(platform, key, build); - - // It should exist now, so true - assertEquals(true, cache.isCached(platform, key)); - - // Changing a source file should invalidate the file - Thread.sleep(1000); - writeToFile("build/b", "bb"); - key = cache.calcKey(platform, sdkVersion, files); - - assertEquals(false, cache.isCached(platform, key)); - - // If we update the build, is should be cached - cache.put(platform, key, build); - assertEquals(true, cache.isCached(platform, key)); - - // Add a "new" file to the list, but let it have an old timestamp - files.add(cRes); - key = cache.calcKey(platform, sdkVersion, files); - - assertEquals(false, cache.isCached(platform, key)); - - // If we update the build, is should be cached - cache.put(platform, key, build); - assertEquals(true, cache.isCached(platform, key)); - - // Remove one file - files.remove(0); - key = cache.calcKey(platform, sdkVersion, files); - - assertEquals(false, cache.isCached(platform, key)); - - // If we update the build, is should be cached - cache.put(platform, key, build); - assertEquals(true, cache.isCached(platform, key)); - } - - private static String calcChecksum(File file) throws IOException, NoSuchAlgorithmException { - MessageDigest md = MessageDigest.getInstance("SHA-256"); - byte[] data = Files.readAllBytes(file.toPath()); - md.update(data); - byte[] digest = md.digest(); - return new BigInteger(1, digest).toString(16); - } - - @Test - public void testClientCachePersistence() throws IOException, ExtenderClientException, NoSuchAlgorithmException { - File a = new File("build/a"); - FileExtenderResource aRes = new FileExtenderResource(a); - a.deleteOnExit(); - writeToFile("build/a", "a"); - - List files = new ArrayList<>(); - files.add(aRes); - - String platform = "osx"; - String sdkVersion = "abc456"; - File cacheDir = new File("."); - - - String key; - { - ExtenderClientCache cache = new ExtenderClientCache(cacheDir); - key = cache.calcKey(platform, sdkVersion, files); - - if (cache.getCacheFile().exists()) { - cache.getCacheFile().delete(); - } - assertFalse(cache.getCacheFile().exists()); - } - - // Start with an empty cache - String checksum = null; - { - ExtenderClientCache cache = new ExtenderClientCache(cacheDir); - - assertEquals(false, cache.isCached(platform, key)); - - // Write the build, and update the cache - File build = File.createTempFile("test", "build"); - build.deleteOnExit(); - writeToFile(build.getAbsolutePath(), (new Date()).toString()); - cache.put(platform, key, build); - - checksum = calcChecksum(build); - } - - // Now, lets create another cache, and check that we get a cached version - { - ExtenderClientCache cache = new ExtenderClientCache(cacheDir); - - assertEquals(true, cache.isCached(platform, key)); - - File build = File.createTempFile("test2", "build"); - cache.get(platform, key, build); - - String checksum2 = calcChecksum(build); - - assertEquals(checksum, checksum2); - } - } - - @Test - public void testClientGetSource() throws IOException { - List files = null; - - String platform = "x86-osx"; - files = getExtensionSource(new File("../server/test-data/testproject/a"), platform); - assertEquals(0, files.size()); - - files = getExtensionSource(new File("../server/test-data/testproject/b"), platform); - assertEquals(4, files.size()); - - files = getExtensionSource(new File("../server/test-data/testproject"), platform); - assertEquals(4, files.size()); - } - - private static List getExtensionSource(File root, String platform) throws IOException { - List source = new ArrayList<>(); - List extensions = listExtensionFolders(root); - - for (File f : extensions) { - - source.add(new FileExtenderResource(f.getAbsolutePath() + File.separator + ExtenderClient.extensionFilename)); - source.addAll(listFilesRecursive(new File(f.getAbsolutePath() + File.separator + "include"))); - source.addAll(listFilesRecursive(new File(f.getAbsolutePath() + File.separator + "src"))); - source.addAll(listFilesRecursive(new File(f.getAbsolutePath() + File.separator + "lib" + File.separator + platform))); - - String[] platformParts = platform.split("-"); - if (platformParts.length == 2) { - source.addAll(listFilesRecursive(new File(f.getAbsolutePath() + File.separator + "lib" + File.separator + platformParts[1]))); - } - } - return source; - } - - private static List listExtensionFolders(File dir) throws IOException { - if (!dir.isDirectory()) { - throw new IOException("Path is not a directory: " + dir.getAbsolutePath()); - } - - List folders = new ArrayList<>(); - - File[] files = dir.listFiles(); - for (File f : files) { - Matcher m = ExtenderClient.extensionPattern.matcher(f.getName()); - if (m.matches()) { - folders.add(dir); - return folders; - } - if (f.isDirectory()) { - folders.addAll(listExtensionFolders(f)); - } - } - return folders; - } - - private static List listFilesRecursive(File dir) { - List output = new ArrayList<>(); - if (!dir.isDirectory()) { - return output; // the extensions doesn't have to have all folders that we look for - } - - File[] files = dir.listFiles(); - for (File f : files) { - if (f.isFile()) { - output.add(new FileExtenderResource(f)); - } else { - output.addAll(listFilesRecursive(f)); - } - } - return output; - } - - @Test - public void testClientHasExtensions() { - assertFalse(hasExtensions(new File("../server/test-data/testproject/a"))); - assertTrue(hasExtensions(new File("../server/test-data/testproject/b"))); - assertTrue(hasExtensions(new File("../server/test-data/testproject"))); - } - @Test public void testClientHeaders() throws Exception { - class MockHttpClient extends DefaultHttpClient { - public HttpUriRequest request; - - @Override - public CloseableHttpResponse execute(HttpUriRequest request) { - this.request = request; - CloseableHttpResponse response = mock(CloseableHttpResponse.class); - StatusLine statusLine = mock(StatusLine.class); - when(response.getStatusLine()).thenReturn(statusLine); - when(statusLine.getStatusCode()).thenReturn(200); - return response; - } - }; - try { final String HDR_NAME_1 = "x-custom-defold-header1"; final String HDR_VALUE_1 = "my custom header1"; final String HDR_NAME_2 = "x-custom-defold-header2"; final String HDR_VALUE_2 = "my custom header2"; - MockHttpClient httpClient = new MockHttpClient(); - File cacheDir = new File("build"); - Class mockExtenderClientClass = Class.forName("com.defold.extender.client.ExtenderClient"); - ExtenderClient extenderClient = (ExtenderClient) mockExtenderClientClass.getDeclaredConstructor(String.class, File.class).newInstance("http://localhost", cacheDir); - Field field = mockExtenderClientClass.getDeclaredField("httpClient"); - field.setAccessible(true); - field.set(extenderClient, httpClient); + DefaultHttpClient httpClient = Mockito.mock(DefaultHttpClient.class); + ExtenderClient extenderClient = new ExtenderClient(null, null, httpClient, "http://localhost"); extenderClient.setHeader(HDR_NAME_1, HDR_VALUE_1); extenderClient.setHeader(HDR_NAME_2, HDR_VALUE_2); - extenderClient.health(); - HttpUriRequest request = httpClient.request; + HttpUriRequest request = extenderClient.createGetRequest("http://localhost/health"); assertEquals(HDR_VALUE_1, request.getFirstHeader(HDR_NAME_1).getValue()); assertEquals(HDR_VALUE_2, request.getFirstHeader(HDR_NAME_2).getValue()); } @@ -424,9 +109,9 @@ public void testClientHandleHTTPError() throws ClientProtocolException, IOExcept b.deleteOnExit(); c.deleteOnExit(); - writeToFile("build/a", "a"); - writeToFile("build/b", "b"); - writeToFile("build/c", "c"); + TestUtils.writeToFile("build/a", "a"); + TestUtils.writeToFile("build/b", "b"); + TestUtils.writeToFile("build/c", "c"); List inputFiles = new ArrayList<>(); inputFiles.add(aRes); @@ -445,26 +130,94 @@ public void testClientHandleHTTPError() throws ClientProtocolException, IOExcept } - /* - Scans a directory and returns true if there are extensions available - */ - private static boolean hasExtensions(File dir) { - File[] files = dir.listFiles(); - if (!dir.exists()) { - return false; - } - for (File f : files) { - Matcher m = ExtenderClient.extensionPattern.matcher(f.getName()); - if (m.matches()) { - return true; + private static Stream uploadData() { + return Stream.of( + Arguments.of("{\"files\":[{\"cached\":true,\"path\":\"build/a\"},{\"cached\":true,\"path\":\"build/b\"},{\"cached\":false,\"path\":\"build/c\"}]}", List.of("build/c")), + Arguments.of("{\"files\":[]}", List.of("build/a", "build/b", "build/c")), + Arguments.of(null, List.of("build/a", "build/b", "build/c")) + ); + } + + @ParameterizedTest() + @MethodSource("uploadData") + public void testUploadStructure(String cacheResponse, List expectedFilenames) throws Exception { + try { + ExtenderClient extenderClient = Mockito.mock(ExtenderClient.class); + when(extenderClient.queryCache(anyList())).thenReturn(cacheResponse); + when(extenderClient.createBuildRequestPayload(anyList())).thenCallRealMethod(); + + File a = new File("build/a"); + File b = new File("build/b"); + File c = new File("build/c"); + FileExtenderResource aRes = new FileExtenderResource(a); + FileExtenderResource bRes = new FileExtenderResource(b); + FileExtenderResource cRes = new FileExtenderResource(c); + + a.deleteOnExit(); + b.deleteOnExit(); + c.deleteOnExit(); + + TestUtils.writeToFile("build/a", "a"); + TestUtils.writeToFile("build/b", "b"); + TestUtils.writeToFile("build/c", "c"); + + List inputFiles = new ArrayList<>(); + inputFiles.add(aRes); + inputFiles.add(bRes); + inputFiles.add(cRes); + + + HttpEntity entity = extenderClient.createBuildRequestPayload(inputFiles); + assertNotNull(entity); + + DefaultMessageBuilder builder = new DefaultMessageBuilder(); + builder.setContentDecoding(true); + + String headers = "Content-Type: " + entity.getContentType().getValue() + "\r\n\r\n"; + + byte[] byteHeader = headers.getBytes(); + byte[] byteBody = EntityUtils.toByteArray(entity); + byte[] res = new byte[byteHeader.length + byteBody.length]; + for (int i = 0; i < byteHeader.length; ++i) { + res[i] = byteHeader[i]; } + for (int i = byteHeader.length; i < byteHeader.length + byteBody.length; ++i) { + res[i] = byteBody[i - byteHeader.length]; + } + Message message = builder.parseMessage(new ByteArrayInputStream(res)); + Body body = message.getBody(); + + assertTrue(body instanceof Multipart); + Multipart multipart = (Multipart) body; + List parts = multipart.getBodyParts(); - if (f.isDirectory()) { - if (hasExtensions(f)) { - return true; + Entity sourceCodeArchiveEntity = null; + + for (Entity part : parts) { + if (part.getFilename().equals(ExtenderClient.SOURCE_CODE_ARCHIVE_MAGIC_NAME)) { + sourceCodeArchiveEntity = part; + break; } } + assertNotNull(sourceCodeArchiveEntity); + assertTrue(sourceCodeArchiveEntity.getBody() instanceof BinaryBody); + BinaryBody archiveBody = (BinaryBody)sourceCodeArchiveEntity.getBody(); + ZipInputStream zis = new ZipInputStream(archiveBody.getInputStream()); + ZipEntry zipEntry = zis.getNextEntry(); + List zipEntriesFilenames = new ArrayList<>(); + while (zipEntry != null) { + zipEntriesFilenames.add(zipEntry.getName()); + zipEntry = zis.getNextEntry(); + } + assertTrue( + expectedFilenames.size() == zipEntriesFilenames.size() && + expectedFilenames.containsAll(zipEntriesFilenames) && + zipEntriesFilenames.containsAll(expectedFilenames) + ); + } + catch (Exception e) { + System.out.println("ERROR LOG:"); + throw e; } - return false; } } diff --git a/client/src/test/java/com/defold/extender/client/FileExtenderResourceTest.java b/client/src/test/java/com/defold/extender/client/FileExtenderResourceTest.java new file mode 100644 index 00000000..a03dc952 --- /dev/null +++ b/client/src/test/java/com/defold/extender/client/FileExtenderResourceTest.java @@ -0,0 +1,118 @@ +package com.defold.extender.client; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; + +import org.junit.jupiter.api.Test; + +public class FileExtenderResourceTest { + @Test + public void testClientGetSource() throws IOException { + List files = null; + + String platform = "x86-osx"; + files = getExtensionSource(new File("../server/test-data/testproject/a"), platform); + assertEquals(0, files.size()); + + files = getExtensionSource(new File("../server/test-data/testproject/b"), platform); + assertEquals(4, files.size()); + + files = getExtensionSource(new File("../server/test-data/testproject"), platform); + assertEquals(4, files.size()); + } + + private static List getExtensionSource(File root, String platform) throws IOException { + List source = new ArrayList<>(); + List extensions = listExtensionFolders(root); + + for (File f : extensions) { + + source.add(new FileExtenderResource(f.getAbsolutePath() + File.separator + ExtenderClient.extensionFilename)); + source.addAll(listFilesRecursive(new File(f.getAbsolutePath() + File.separator + "include"))); + source.addAll(listFilesRecursive(new File(f.getAbsolutePath() + File.separator + "src"))); + source.addAll(listFilesRecursive(new File(f.getAbsolutePath() + File.separator + "lib" + File.separator + platform))); + + String[] platformParts = platform.split("-"); + if (platformParts.length == 2) { + source.addAll(listFilesRecursive(new File(f.getAbsolutePath() + File.separator + "lib" + File.separator + platformParts[1]))); + } + } + return source; + } + + private static List listExtensionFolders(File dir) throws IOException { + if (!dir.isDirectory()) { + throw new IOException("Path is not a directory: " + dir.getAbsolutePath()); + } + + List folders = new ArrayList<>(); + + File[] files = dir.listFiles(); + for (File f : files) { + Matcher m = ExtenderClient.extensionPattern.matcher(f.getName()); + if (m.matches()) { + folders.add(dir); + return folders; + } + if (f.isDirectory()) { + folders.addAll(listExtensionFolders(f)); + } + } + return folders; + } + + private static List listFilesRecursive(File dir) { + List output = new ArrayList<>(); + if (!dir.isDirectory()) { + return output; // the extensions doesn't have to have all folders that we look for + } + + File[] files = dir.listFiles(); + for (File f : files) { + if (f.isFile()) { + output.add(new FileExtenderResource(f)); + } else { + output.addAll(listFilesRecursive(f)); + } + } + return output; + } + + + /* + Scans a directory and returns true if there are extensions available + */ + private static boolean hasExtensions(File dir) { + File[] files = dir.listFiles(); + if (!dir.exists()) { + return false; + } + for (File f : files) { + Matcher m = ExtenderClient.extensionPattern.matcher(f.getName()); + if (m.matches()) { + return true; + } + + if (f.isDirectory()) { + if (hasExtensions(f)) { + return true; + } + } + } + return false; + } + + @Test + public void testClientHasExtensions() { + assertFalse(hasExtensions(new File("../server/test-data/testproject/a"))); + assertTrue(hasExtensions(new File("../server/test-data/testproject/b"))); + assertTrue(hasExtensions(new File("../server/test-data/testproject"))); + } +} diff --git a/client/src/test/java/com/defold/extender/client/TestUtils.java b/client/src/test/java/com/defold/extender/client/TestUtils.java new file mode 100644 index 00000000..87316b9b --- /dev/null +++ b/client/src/test/java/com/defold/extender/client/TestUtils.java @@ -0,0 +1,17 @@ +package com.defold.extender.client; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.time.Instant; + +public class TestUtils { + public static void writeToFile(String path, String msg) throws IOException { + File f = new File(path); + FileWriter fwr = new FileWriter(f); + fwr.write(msg); + fwr.flush(); + fwr.close(); + f.setLastModified(Instant.now().toEpochMilli() + 23); + } +} diff --git a/server/src/main/java/com/defold/extender/ExtenderController.java b/server/src/main/java/com/defold/extender/ExtenderController.java index f14f2b11..4c525e28 100644 --- a/server/src/main/java/com/defold/extender/ExtenderController.java +++ b/server/src/main/java/com/defold/extender/ExtenderController.java @@ -56,6 +56,8 @@ public class ExtenderController { // Used to verify the uploaded filenames private static final Pattern FILENAME_RE = Pattern.compile("^([\\w ](?:[\\w+\\-\\/ @]|(?:\\.[\\w+\\-\\/ ]*))+)$"); + private static final String SOURCE_CODE_ARCHIVE_MAGIC_NAME = "__source_code__.zip"; + private final DefoldSdkService defoldSdkService; private final DataCacheService dataCacheService; private final MeterRegistry meterRegistry; @@ -384,6 +386,13 @@ static void receiveUpload(MultipartHttpServletRequest request, File uploadDirect System.out.printf(" %s\n", file.toPath()); } } + + // check if source code archive presented + File sourceCodeArchive = new File(uploadDirectory, SOURCE_CODE_ARCHIVE_MAGIC_NAME); + if (sourceCodeArchive.exists()) { + LOGGER.debug("Source code archive found. Unarchiving..."); + ZipUtils.unzip(new FileInputStream(sourceCodeArchive), uploadDirectory.toPath()); + } } private boolean isRemotePlatform(final String platform, String platformVersion) {