Improve memory management and error handling (#70)

* Improve memory management and error handling

Co-authored-by: zxBCN Farre_Basurte,Juan_Antonio (IT EDS) EXTERNAL <juan_antonio.farre_basurte.ext@boehringer-ingelheim.com>

(cherry picked from commit def9b0c)

Author: Juan Farré

CHANGELOG.md

@@ -1,7 +1,7 @@
 # Changelog
 
 ## Unreleased
-
+- Improve memory management and error handling ([#70](https://github.com/opendevstack/ods-document-generation-svc/pull/70))
 
 ## [4.0] - 2021-18-11
 

build.gradle

@@ -62,13 +62,13 @@ buildscript {
 }
 
 plugins {
-  id "com.github.johnrengelman.shadow" version "5.1.0"
+  id "com.github.johnrengelman.shadow"
   id "com.google.osdetector" version "1.6.2"
   id "io.spring.dependency-management" version "1.0.8.RELEASE"
 }
 
 apply plugin: "application"
-apply plugin: "distribution"
+apply plugin: "com.github.johnrengelman.shadow"
 apply plugin: "groovy"
 apply plugin: "jooby"
 apply plugin: "jacoco"
@@ -118,28 +118,39 @@ dependencyManagement {
 }
 
 dependencies {
-  compile "com.github.ben-manes.caffeine:caffeine:2.7.0"
-  compile "com.github.jknack:handlebars:4.1.2"
-  compile "commons-io:commons-io:2.6"
-  compile "io.github.openfeign:feign-core:10.2.3"
-  compile "io.github.openfeign:feign-gson:10.2.3"
-  compile "io.github.openfeign:feign-okhttp:10.2.3"
-  compile "net.lingala.zip4j:zip4j:1.3.3"
-  compile "org.apache.httpcomponents:httpclient:4.5.8"
-  compile "org.codehaus.groovy:groovy-all:2.5.7"
-  compile "org.jooby:jooby-jackson"
-  compile "org.jooby:jooby-netty"
-
-  compile "io.netty:netty-transport-native-epoll:${dependencyManagement.importedProperties['netty.version']}:${osdetector.classifier.contains('linux') ? 'linux-x86_64' : ''}"
-  compile "io.netty:netty-tcnative-boringssl-static:${dependencyManagement.importedProperties['boringssl.version']}" 
-
-  testCompile "junit:junit:4.12"
-  testCompile "com.github.stefanbirkner:system-rules:1.19.0" // for managing environment variables
-  testCompile "com.github.tomakehurst:wiremock:2.23.2"       // for mocking HTTP server reponses
-  testCompile "io.rest-assured:rest-assured:4.0.0"           // for validating REST services
-  testCompile "org.spockframework:spock-core:1.3-groovy-2.5"
+  implementation "com.github.ben-manes.caffeine:caffeine:2.7.0"
+  implementation "com.github.jknack:handlebars:4.1.2"
+  implementation "commons-io:commons-io:2.11.0"
+  implementation "io.github.openfeign:feign-core:10.2.3"
+  implementation "io.github.openfeign:feign-gson:10.2.3"
+  implementation "io.github.openfeign:feign-okhttp:10.2.3"
+  implementation "net.lingala.zip4j:zip4j:1.3.3"
+  implementation "org.apache.httpcomponents:httpclient:4.5.8"
+  implementation "org.codehaus.groovy:groovy-all:2.5.7"
+  implementation "org.jooby:jooby-jackson"
+  implementation "org.jooby:jooby-netty"
+
+  implementation "io.netty:netty-transport-native-epoll:${dependencyManagement.importedProperties['netty.version']}:${osdetector.classifier.contains('linux') ? 'linux-x86_64' : ''}"
+  implementation "io.netty:netty-tcnative-boringssl-static:${dependencyManagement.importedProperties['boringssl.version']}"
 
   implementation "org.apache.pdfbox:pdfbox:2.0.24"
+
+  testImplementation "junit:junit:4.12"
+  testImplementation "com.github.stefanbirkner:system-rules:1.19.0" // for managing environment variables
+  testImplementation "com.github.tomakehurst:wiremock:2.23.2"       // for mocking HTTP server reponses
+  testImplementation "io.rest-assured:rest-assured:4.0.0"           // for validating REST services
+  testImplementation "org.spockframework:spock-core:1.3-groovy-2.5"
+
+  testImplementation "cglib:cglib-nodep:3.3.0"                      // for mocking classes
+  testImplementation "org.objenesis:objenesis:3.1"
+}
+
+import com.github.jengelman.gradle.plugins.shadow.transformers.NewGroovyExtensionModuleTransformer
+
+shadowJar {
+  mergeServiceFiles()
+  mergeGroovyExtensionModules()
+  transform(NewGroovyExtensionModuleTransformer)
 }
 
 test {

buildSrc/build.gradle

@@ -0,0 +1,58 @@
+ext {
+    nexus_url = "${project.findProperty('nexus_url') ?: System.getenv('NEXUS_HOST')}"
+    nexus_user = "${project.findProperty('nexus_user') ?: System.getenv('NEXUS_USERNAME')}"
+    nexus_pw = "${project.findProperty('nexus_pw') ?: System.getenv('NEXUS_PASSWORD')}"
+    no_nexus = (project.findProperty('no_nexus') ?: System.getenv('NO_NEXUS') ?: false).toBoolean()
+    if (!no_nexus && (nexus_url == "null" || nexus_user == "null" || nexus_pw == "null")) {
+        throw new GradleException("property no_nexus='false' (or not defined) but at least one of the properties nexus_url, nexus_user or nexus_pw is not configured. Please configure those properties!")
+    }
+
+    def folderRel = (String)("${project.findProperty('nexus_folder_releases') ?: System.getenv('NEXUS_FOLDER_RELEASES')}")
+    nexusFolderReleases = folderRel == "null" ? "maven-releases" : folderRel
+
+    def folderSnaps = (String)("${project.findProperty('nexus_folder_snapshots') ?: System.getenv('NEXUS_FOLDER_SNAPSHOTS')}")
+    nexusFolderSnapshots = folderSnaps == "null" ? "maven-snapshots" : folderSnaps
+
+    snippetsDir = file('build/generated-snippets')
+}
+
+repositories {
+    if (no_nexus) {
+        println("using repositories 'jcenter' and 'mavenCentral', because property no_nexus=$no_nexus")
+        jcenter()
+        mavenCentral()
+    } else {
+        println("using nexus repositories")
+        maven() {
+            url "${nexus_url}/repository/jcenter/"
+            credentials {
+                username = "${nexus_user}"
+                password = "${nexus_pw}"
+            }
+        }
+
+        maven() {
+            url "${nexus_url}/repository/maven-public/"
+            credentials {
+                username = "${nexus_user}"
+                password = "${nexus_pw}"
+            }
+        }
+
+        maven() {
+            url "${nexus_url}/repository/atlassian_public/"
+            credentials {
+                username = "${nexus_user}"
+                password = "${nexus_pw}"
+            }
+        }
+    }
+}
+
+apply plugin: 'groovy'
+
+dependencies {
+    implementation gradleApi()
+    implementation 'com.github.jengelman.gradle.plugins:shadow:5.2.0'
+    implementation "org.codehaus.plexus:plexus-utils:3.0.24"
+}

buildSrc/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/NewGroovyExtensionModuleTransformer.groovy

@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.github.jengelman.gradle.plugins.shadow.transformers
+
+import shadow.org.apache.tools.zip.ZipEntry
+import shadow.org.apache.tools.zip.ZipOutputStream
+import org.codehaus.plexus.util.IOUtil
+import org.gradle.api.file.FileTreeElement
+
+/**
+ * Modified from https://github.com/johnrengelman/shadow/blob/7.1.1/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/GroovyExtensionModuleTransformer.groovy
+ * ---
+ * Modified from eu.appsatori.gradle.fatjar.tasks.PrepareFiles.groovy
+ * <p>
+ * Resource transformer that merges Groovy extension module descriptor files into a single file. If there are several
+ * META-INF/services/org.codehaus.groovy.runtime.ExtensionModule resources spread across many JARs the individual
+ * entries will all be merged into a single META-INF/services/org.codehaus.groovy.runtime.ExtensionModule resource
+ * packaged into the resultant JAR produced by the shadowing process.
+ */
+@CacheableTransformer
+class NewGroovyExtensionModuleTransformer implements Transformer {
+
+    private static final GROOVY_EXTENSION_MODULE_DESCRIPTOR_PATH =
+            "META-INF/groovy/org.codehaus.groovy.runtime.ExtensionModule"
+
+    private static final MODULE_NAME_KEY = 'moduleName'
+    private static final MODULE_VERSION_KEY = 'moduleVersion'
+    private static final EXTENSION_CLASSES_KEY = 'extensionClasses'
+    private static final STATIC_EXTENSION_CLASSES_KEY = 'staticExtensionClasses'
+
+    private static final MERGED_MODULE_NAME = 'MergedByShadowJar'
+    private static final MERGED_MODULE_VERSION = '1.0.0'
+
+    private final Properties module = new Properties()
+
+    @Override
+    boolean canTransformResource(FileTreeElement element) {
+        return element.relativePath.pathString == GROOVY_EXTENSION_MODULE_DESCRIPTOR_PATH
+    }
+
+    @Override
+    void transform(TransformerContext context) {
+        def props = new Properties()
+        props.load(context.is)
+        props.each { String key, String value ->
+            switch (key) {
+                case MODULE_NAME_KEY:
+                    handle(key, value) {
+                        module.setProperty(key, MERGED_MODULE_NAME)
+                    }
+                    break
+                case MODULE_VERSION_KEY:
+                    handle(key, value) {
+                        module.setProperty(key, MERGED_MODULE_VERSION)
+                    }
+                    break
+                case [EXTENSION_CLASSES_KEY, STATIC_EXTENSION_CLASSES_KEY]:
+                    handle(key, value) { String existingValue ->
+                        def newValue = "${existingValue},${value}"
+                        module.setProperty(key, newValue)
+                    }
+                    break
+            }
+        }
+    }
+
+    private handle(String key, String value, Closure mergeValue) {
+        def existingValue = module.getProperty(key)
+        if (existingValue) {
+            mergeValue(existingValue)
+        } else {
+            module.setProperty(key, value)
+        }
+    }
+
+    @Override
+    boolean hasTransformedResource() {
+        return module.size() > 0
+    }
+
+    @Override
+    void modifyOutputStream(ZipOutputStream os, boolean preserveFileTimestamps) {
+        ZipEntry entry = new ZipEntry(GROOVY_EXTENSION_MODULE_DESCRIPTOR_PATH)
+        entry.time = TransformerContext.getEntryTimestamp(preserveFileTimestamps, entry.time)
+        os.putNextEntry(entry)
+        IOUtil.copy(toInputStream(module), os)
+        os.closeEntry()
+    }
+
+    private static InputStream toInputStream(Properties props) {
+        def baos = new ByteArrayOutputStream()
+        props.store(baos, null)
+        return new ByteArrayInputStream(baos.toByteArray())
+    }
+
+}

conf/META-INF/groovy/org.codehaus.groovy.runtime.ExtensionModule

@@ -0,0 +1,3 @@
+moduleName=Resource Handling Extensions
+moduleVersion=1.0
+extensionClasses=util.Try

docker/Dockerfile

@@ -23,6 +23,6 @@ USER 1001
 EXPOSE 8080
 ENV JAVA_MEM_XMX="512m" \
     JAVA_MEM_XMS="128m" \
-    JAVA_OPTS="-XX:+UseCompressedOops -XX:+UseG1GC -XX:MaxGCPauseMillis=1000"
+    JAVA_OPTS="-XX:+UseCompressedOops -XX:+UseG1GC -XX:+UseStringDeduplication -XX:MaxGCPauseMillis=1000"
 
 ENTRYPOINT /app/entrypoint.sh

src/main/groovy/app/App.groovy

@@ -2,13 +2,14 @@ package app
 
 import com.typesafe.config.ConfigFactory
 import groovy.util.logging.Slf4j
-import org.apache.commons.io.IOUtils
-import org.apache.groovy.json.internal.LazyMap
 import org.jooby.Jooby
 import org.jooby.MediaType
 import org.jooby.json.Jackson
+import util.DocUtils
+import util.FileTools
 
 import java.nio.file.Files
+import java.nio.file.StandardOpenOption
 
 import static groovy.json.JsonOutput.prettyPrint
 import static groovy.json.JsonOutput.toJson
@@ -19,12 +20,13 @@ import static org.jooby.JoobyExtension.post
 class App extends Jooby {
 
     {
+
         use(new Jackson())
         use(new DocGen())
 
         post(this, "/document", { req, rsp ->
 
-            def body = req.body().to(LazyMap.class)
+            Map body = req.body().to(HashMap.class)
 
             validateRequestParams(body)
 
@@ -33,48 +35,56 @@ class App extends Jooby {
                 log.debug(prettyPrint(toJson(body.data)))
             }
 
-            def pdf = new DocGen().generate(body.metadata.type, body.metadata.version, body.data)
-            try{
-                pdf.withInputStream { is ->
-                    byte[] pdfBytes = IOUtils.toByteArray(is)
-                    rsp.send([
-                            data: Base64.getEncoder().encodeToString(pdfBytes)
-                    ])
+            FileTools.newTempFile('document', '.b64') { dataFile ->
+                new DocGen().generate(body.metadata.type, body.metadata.version, body.data).withFile { pdf ->
+                    body = null // Not used anymore. Let it be garbage-collected.
+                    dataFile.withOutputStream { os ->
+                        Base64.getEncoder().wrap(os).withStream { encOs ->
+                            Files.copy(pdf, encOs)
+                        }
+                    }
+                }
+                def dataLength = Files.size(dataFile)
+                rsp.length(dataLength + RES_PREFIX.length + RES_SUFFIX.length)
+                rsp.type(MediaType.json)
+                def prefixIs = new ByteArrayInputStream(RES_PREFIX)
+                def suffixIs = new ByteArrayInputStream(RES_SUFFIX)
+                Files.newInputStream(dataFile, StandardOpenOption.DELETE_ON_CLOSE).initResource { dataIs ->
+                    // Jooby is asynchronous. Upon return of the send method, the response has not necessarily
+                    // been sent. For this reason, we rely on Jooby to close the InputStream and the temporary file
+                    // will be deleted on close.
+                    rsp.send(new SequenceInputStream(Collections.enumeration([prefixIs, dataIs, suffixIs])))
                 }
-            }finally{
-                pdf.delete()
             }
 
         })
         .consumes(MediaType.json)
         .produces(MediaType.json)
 
         get(this, "/health", { req, rsp ->
-            def documentHtmlFile = Files.createTempFile("document", ".html") << "<html>document</html>"
-            def headerHtmlFile = Files.createTempFile("header", ".html") << "<html>header</html>"
-            def footerHtmlFile = Files.createTempFile("footer", ".html") << "<html>footer</html>"
-
             def message = null
             def status = "passing"
             def statusCode = 200
 
-            def data
             try {
-                data = DocGen.Util.convertHtmlToPDF(documentHtmlFile, headerHtmlFile, footerHtmlFile)
+                FileTools.withTempFile("document", ".html") { documentHtmlFile ->
+                    documentHtmlFile  << "<html>document</html>"
+
+                    DocGen.Util.convertHtmlToPDF(documentHtmlFile, null).withFile { pdf ->
+                        def header = DocUtils.getPDFHeader(pdf)
+                        if (header != PDF_HEADER) {
+                            message = "conversion form HTML to PDF failed"
+                            status = "failing"
+                            statusCode = 500
+                        }
+                        return pdf
+                    }
+
+                }
             } catch (e) {
                 message = e.message
                 status = "failing"
                 statusCode = 500
-            } finally {
-                Files.delete(documentHtmlFile)
-                Files.delete(headerHtmlFile)
-                Files.delete(footerHtmlFile)
-            }
-
-            if (!new String(data).startsWith("%PDF-1.4\n")) {
-                message = "conversion form HTML to PDF failed"
-                status = "failing"
-                statusCode = 500
             }
 
             def result = [
@@ -92,6 +102,10 @@ class App extends Jooby {
         .produces(MediaType.json)
     }
 
+    private static final RES_PREFIX = '{"data":"'.getBytes('US-ASCII')
+    private static final RES_SUFFIX = '"}'.getBytes('US-ASCII')
+    private static final PDF_HEADER = '%PDF-1.4'
+
     private static void validateRequestParams(Map body) {
         if (body?.metadata?.type == null) {
             throw new IllegalArgumentException("missing argument 'metadata.type'")