Merge pull request #1421 from WebFuzzing/builder-performance

Builder performance

Author: arcuri82

.github/workflows/ci.yml

@@ -42,7 +42,7 @@ env:
   build-jdk: 17
   retention-days: 5
   debug: false  # put to true if need to debug a specific test
-  debugTestName: "org.evomaster.e2etests.spring.examples.sort.SortEMTest" # replace with test to debug
+  debugTestName: "org.evomaster.core.problem.rest.SamplerVerifierTest" # replace with test to debug
 
 # This build is quite expensive (some hours), so we run it whole only on some JVM versions and OSs.
 # For the moment, we need to support JVM 8 and all following LTS versions (e.g, 11 and  17).

client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/AdditionalInfo.java

@@ -170,7 +170,10 @@ public void setRawAccessOfHttpBodyPayload(boolean rawAccessOfHttpBodyPayload) {
 
     public void addSpecialization(String taintInputName, StringSpecializationInfo info){
         if(!ExecutionTracer.getTaintType(taintInputName).isTainted()){
-            throw new IllegalArgumentException("No valid input name: " + taintInputName);
+            //this can happen in E2E where libraries used by "core" are instrumented (eg Kotlin)
+            SimpleLogger.error("No valid taint input name for specialization: " + taintInputName);
+            //throw new IllegalArgumentException();
+            return;
         }
         Objects.requireNonNull(info);
 

core-parent/pom.xml

@@ -15,9 +15,9 @@
 
     <properties>
         <!--   upgrading to 1.0.68  breaks Spring... TODO would need try upgrading in own branch    -->
-        <swagger.parser-v2.version>1.0.61</swagger.parser-v2.version>
+        <swagger.parser-v2.version>1.0.64</swagger.parser-v2.version>
         <!--   upgrading to 2.1.18  breaks Spring... TODO would need try upgrading in own branch -->
-        <swagger.parser-v3.version>2.1.8</swagger.parser-v3.version>
+        <swagger.parser-v3.version>2.1.14</swagger.parser-v3.version>
         <swagger.annotations.version>2.2.7</swagger.annotations.version>
     </properties>
 

core-tests/integration-tests/core-it/src/test/kotlin/org/evomaster/core/problem/rest/SamplerVerifierTest.kt

@@ -13,10 +13,11 @@ import org.evomaster.client.java.controller.api.dto.problem.param.DerivedParamCh
 import org.evomaster.client.java.controller.api.dto.problem.rpc.ScheduleTaskInvocationsDto
 import org.evomaster.client.java.controller.api.dto.problem.rpc.ScheduleTaskInvocationsResult
 import org.evomaster.core.BaseModule
+import org.evomaster.core.Main
 import org.evomaster.core.problem.rest.schema.RestSchema
 import org.evomaster.core.problem.rest.service.module.BlackBoxRestModule
 import org.evomaster.core.problem.rest.service.module.ResourceRestModule
-import org.evomaster.core.problem.rest.service.sampler.ResourceSampler
+import org.evomaster.core.problem.rest.service.sampler.AbstractRestSampler
 import org.evomaster.core.remote.SutProblemException
 import org.evomaster.core.remote.service.RemoteController
 import org.evomaster.core.search.Individual
@@ -28,10 +29,17 @@ import java.lang.reflect.InvocationTargetException
 import java.time.Duration
 import kotlin.io.path.Path
 import kotlin.sequences.filter
+import kotlin.text.contains
 
 
 class SamplerVerifierTest {
 
+    companion object {
+        init {
+            Main.applyGlobalJVMSettings()
+        }
+    }
+
 
     @Test
     fun testBase() {
@@ -47,7 +55,7 @@ class SamplerVerifierTest {
 
         val injector = getInjector(sutInfo, controllerInfo, false)
 
-        val sampler = injector.getInstance(ResourceSampler::class.java)
+        val sampler = injector.getInstance(AbstractRestSampler::class.java)
 
         sampler.sample() //should not crash
     }
@@ -64,11 +72,8 @@ class SamplerVerifierTest {
         )
         assertTrue(tests.isNotEmpty())
         return tests
-            //FIXME once handling performance issues
-            .filter { !it.displayName.contains("adyen") }
     }
 
-    @Disabled("Performance issue")
     @TestFactory
     fun testSamplingFromAllSchemasUnderCoreResourcesBlackBox(): Collection<DynamicTest>{
         val tests = sampleFromSchemasAndCheckInvariants(
@@ -81,8 +86,6 @@ class SamplerVerifierTest {
     }
 
 
-    //FIXME need to put back, and investigate performance bug
-    @Disabled("Major issues with timeouts. Even before, took more than 1 hour. Need refactoring. Maven was not showing the failures (likely bug in Surefire)")
     @TestFactory
     fun testSamplingFromAPIsGuru(): Collection<DynamicTest>{
         val tests = sampleFromSchemasAndCheckInvariants(
@@ -100,11 +103,22 @@ class SamplerVerifierTest {
         blackBox: Boolean
     ): Collection<DynamicTest> {
 
+        /*
+            In theory, we should not have such a high timeout for this kind of tests.
+            On local machine, all those analyses take at most 20s per test.
+            Maybe performance can be improved, but, considering some files are MBs, it is not
+            so unexpected. And it happens only for very large files.
+            Problem though is that CI is much SLOWER, and tests do timeout.
+            So that is why we have such high timeout, it is for CI.
+            Still, if it starts to fail there, then we really need to look into performance issues.
+         */
+        val timeout = 60L
+
         return scanForSchemas(relativePath, resourceFolder)
             .sorted().map {
             DynamicTest.dynamicTest(it) {
                 System.gc()
-                assertTimeoutPreemptively(Duration.ofSeconds(30), it) {
+                assertTimeoutPreemptively(Duration.ofSeconds(timeout), it) {
                     runInvariantCheck(it, 100, blackBox)
                 }
             }
@@ -121,16 +135,15 @@ class SamplerVerifierTest {
                 .filter { it.isFile }
                 .filter { !skipSchema(it.path) }
                 .map {
-//                    val s = it.path.replace("\\", "/")
-//                            .replace(relativePath, resourceFolder)
                     val s = Path(it.absolutePath).toAbsolutePath().normalize().toString()
                     s
                 }.toList()
     }
 
     private fun skipSchema(path: String) : Boolean {
-        return skipDueToOldChecks(path) ||
-                skipDueToMissingPath(path)
+        return skipDueToOldChecks(path)
+                || skipDueToIssuesStillToInvestigate(path)
+                || skipDueToMissingPath(path)
                 || skipDueToHashTag(path)
                 || skipDueToQuestionMarkInPath(path)
                 || skipDueToMissingReference(path)
@@ -143,6 +156,29 @@ class SamplerVerifierTest {
                 || skipDueToInvalidGenes(path)
     }
 
+    private fun skipDueToIssuesStillToInvestigate(path: String) : Boolean {
+        val toSkip = listOf(
+                    "api.video/1/openapi.yaml",
+                    "atlassian.com/jira/1001.0.0-SNAPSHOT/openapi.yaml",
+                    "cloud-elements.com/ecwid/api-v2/swagger.yaml",
+                    "github.com/api.github.com/1.1.4/openapi.yaml",
+                    "googleapis.com/discovery/v1/openapi.yaml",
+                    "here.com/positioning/2.1.1/openapi.yaml",
+                    "maif.local/otoroshi/1.5.0-dev/openapi.yaml",
+                    "mashape.com/geodb/1.0.0/swagger.yaml",
+                    "microsoft.com/cognitiveservices-Training/1.2/openapi.yaml",
+                    "microsoft.com/cognitiveservices-Training/2.0/openapi.yaml",
+                    "microsoft.com/cognitiveservices-Training/2.1/openapi.yaml",
+                    "microsoft.com/cognitiveservices-Training/2.2/openapi.yaml",
+                    "microsoft.com/cognitiveservices-Training/3.0/openapi.yaml",
+                    "microsoft.com/cognitiveservices-Training/3.1/openapi.yaml",
+                    "neutrinoapi.net/3.5.0/openapi.yaml",
+                    "openbankingproject.ch/1.3.8_2020-12-14 - Swiss edition 1.3.8.1-CH/openapi.yaml"
+        )
+
+        return  toSkip.any { path.contains(it) }
+    }
+    
     private fun skipDueToOldChecks(path: String) : Boolean {
         return path.endsWith("features_service_null.json") //issue with parser
                 || path.endsWith("trace_v2.json")  // no actions are parsed
@@ -244,7 +280,7 @@ class SamplerVerifierTest {
                     || (contains("visualstudio.com") && contains("v1"))
                     || (contains("youneedabudget.com") && contains("1.0.0"))
                     || (contains("zenoti.com") && contains("1.0.0")) //  No actions for schema
-                    || (contains("zoom.us") && contains("2.0.0")) // The incoming YAML document exceeds the limit: 3145728 code points.
+                 //   || (contains("zoom.us") && contains("2.0.0")) // The incoming YAML document exceeds the limit: 3145728 code points.
                     || (contains("zuora.com") && contains("2021-08-20")) //The incoming YAML document exceeds the limit: 3145728 code points.
         }
     }
@@ -381,7 +417,7 @@ class SamplerVerifierTest {
             throw e
         }
 
-        val sampler = injector.getInstance(ResourceSampler::class.java)
+        val sampler = injector.getInstance(AbstractRestSampler::class.java)
 
         if(sampler.numberOfDistinctActions() == 0){
             throw IllegalStateException("No actions for schema")

core/src/main/kotlin/org/evomaster/core/BaseModule.kt

@@ -3,10 +3,7 @@ package org.evomaster.core
 import com.google.inject.AbstractModule
 import com.google.inject.Provides
 import com.google.inject.Singleton
-import org.evomaster.core.output.service.NoTestCaseWriter
 import org.evomaster.core.output.service.PartialOracles
-import org.evomaster.core.output.service.TestCaseWriter
-import org.evomaster.core.output.service.TestSuiteWriter
 import org.evomaster.core.search.service.mutator.genemutation.ArchiveImpactSelector
 import org.evomaster.core.search.service.*
 import org.evomaster.core.search.service.monitor.SearchProcessMonitor

core/src/main/kotlin/org/evomaster/core/Main.kt

@@ -54,14 +54,26 @@ import kotlin.system.exitProcess
 class Main {
     companion object {
 
+        /**
+         * Anything that impact whole JVM, needs to be done here, and called as first step in the main.
+         * Note: this is done as a function because tests will usually not call Main, and might rely
+         * on those settings for their behavior
+         */
+        @JvmStatic
+        fun applyGlobalJVMSettings(){
+            Locale.setDefault(Locale.ENGLISH)
+            //otherwise parser will crash on large OpenAPI schemas
+            System.setProperty("maxYamlCodePoints", "" + (50 * 1024 * 1024))
+        }
+
         /**
          * Main entry point of the EvoMaster application
          */
         @JvmStatic
         fun main(args: Array<String>) {
 
             try {
-                Locale.setDefault(Locale.ENGLISH)
+                applyGlobalJVMSettings()
 
                 printLogo()
                 printVersion()

core/src/main/kotlin/org/evomaster/core/problem/rest/schema/OpenApiAccess.kt

@@ -31,19 +31,29 @@ object OpenApiAccess {
 
         var parseResults: SwaggerParseResult? = null
 
+        val messages = mutableSetOf<String>()
+
         for (extension in OpenAPIV3Parser.getExtensions()) {
             parseResults = try {
                 extension.readContents(schemaText, null, null)
             } catch (e: Exception) {
                 throw SutProblemException("Failed to parse OpenApi schema: ${e.message}")
             }
-            if (parseResults != null && parseResults.openAPI != null) {
-                break
+            if (parseResults != null){
+                if(parseResults.openAPI != null) {
+                    //all good
+                    break
+                } else {
+                   //we might try another parser... but still shouldn't lose info of attempted parsers
+                    messages.addAll(parseResults.messages.map {
+                        extension.javaClass.simpleName + " -> " + it + "\n"
+                    })
+                }
             }
         }
 
         val schema =  parseResults!!.openAPI
-                ?: throw SutProblemException("Failed to parse OpenApi schema: " + parseResults.messages.joinToString("\n"))
+                ?: throw SutProblemException("Failed to parse OpenApi schema:\n" + messages.joinToString(""))
 
         if(parseResults.messages.isNotEmpty()){
             LoggingUtil.getInfoLogger().warn(

core/src/main/kotlin/org/evomaster/core/problem/rest/schema/RestSchema.kt

@@ -3,7 +3,9 @@ package org.evomaster.core.problem.rest.schema
 import com.fasterxml.jackson.databind.JsonNode
 import com.fasterxml.jackson.databind.ObjectMapper
 import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactoryBuilder
 import org.evomaster.core.remote.SutProblemException
+import org.yaml.snakeyaml.LoaderOptions
 
 
 /**
@@ -62,7 +64,16 @@ class RestSchema(
 
         //https://swagger.io/docs/specification/v3_0/using-ref/
 
-        val mapper = ObjectMapper(YAMLFactory())
+        // snakeyaml has default limits on size, which are very low
+        val yaml = YAMLFactoryBuilder(YAMLFactory())
+            .loaderOptions(LoaderOptions().apply {
+                codePointLimit = 50 * 1024 * 1024 // 50MB
+                maxAliasesForCollections = 1000
+                nestingDepthLimit = 100
+            })
+            .build()
+
+        val mapper = ObjectMapper(yaml)
         val tree = mapper.readTree(schema.schemaRaw)
         val refs = findAllSRef(tree)
 

core/src/main/kotlin/org/evomaster/core/problem/rest/service/module/BlackBoxRestModule.kt

@@ -7,7 +7,6 @@ import org.evomaster.core.problem.rest.service.sampler.AbstractRestSampler
 import org.evomaster.core.problem.rest.service.fitness.BlackBoxRestFitness
 import org.evomaster.core.problem.rest.service.sampler.RestSampler
 import org.evomaster.core.problem.enterprise.service.EnterpriseSampler
-import org.evomaster.core.problem.rest.service.sampler.ResourceSampler
 import org.evomaster.core.remote.service.RemoteController
 import org.evomaster.core.remote.service.RemoteControllerImplementation
 import org.evomaster.core.search.service.FitnessFunction
@@ -22,7 +21,7 @@ class BlackBoxRestModule(
         super.configure()
 
         bind(object : TypeLiteral<EnterpriseSampler<RestIndividual>>() {})
-            .to(ResourceSampler::class.java)
+            .to(RestSampler::class.java)
             .asEagerSingleton()
         bind(object : TypeLiteral<Sampler<RestIndividual>>() {})
                 .to(RestSampler::class.java)

core/src/main/kotlin/org/evomaster/core/problem/rest/service/sampler/RestSampler.kt

@@ -274,7 +274,7 @@ class RestSampler : AbstractRestSampler(){
         if (created && !get.path.isLastElementAParameter()) {
 
             val lastPost = test[test.size - 2]
-            Lazy.assert{lastPost.verb == HttpVerb.POST}
+            Lazy.assert{lastPost.verb == HttpVerb.POST || lastPost.verb == HttpVerb.PUT}
 
             val available = getMaxTestSizeDuringSampler() - test.size