Merge pull request #1605 from WebFuzzing/N-FlakyHandle

N flaky handle

Author: arcuri82

core-tests/e2e-tests/spring/spring-rest-bb/src/test/kotlin/org/evomaster/e2etests/spring/rest/bb/flakinessdetect/FlakinessDetectBBEMTest.kt

@@ -37,11 +37,12 @@ class FlakinessDetectBBEMTest : SpringTestBase() {
         ){ args: MutableList<String> ->
 
             setOption(args, "handleFlakiness", "true")
+            setOption(args, "execNumForDetectFlakiness", "1")
 
             val solution = initAndRun(args)
 
             assertTrue(solution.individuals.isNotEmpty())
 
         }
     }
-}
+}

core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/flakinessdetect/FlakinessDetectRest.kt

@@ -10,6 +10,7 @@ import org.springframework.web.bind.annotation.RestController
 import java.time.LocalDateTime
 import java.time.format.DateTimeFormatter
 import java.util.UUID
+import java.util.concurrent.atomic.AtomicInteger
 import kotlin.math.abs
 import kotlin.math.max
 import kotlin.math.min
@@ -21,6 +22,7 @@ class FlakinessDetectRest {
     companion object{
         val formatter = DateTimeFormatter.ofPattern("HH:mm:ss.SSS yyyy-MM-dd EEEE 'Week' ww")
         private val START_UP_OBJECT_TAG = arrayOf(javax.validation.constraints.Pattern.Flag.CASE_INSENSITIVE).toString()
+        private val MULTI_EXECUTION_COUNTER = AtomicInteger(0)
     }
     @GetMapping(path = ["/objectFlag"])
     fun getEmptyFlag()  : ResponseEntity<String> {
@@ -54,6 +56,19 @@ class FlakinessDetectRest {
         return ResponseEntity.ok(FlakinessDetectData(msg, num))
     }
 
+    @GetMapping(path = ["/multiexecution"])
+    fun getMultiExecution() : ResponseEntity<MultiExecutionData> {
+        val index = MULTI_EXECUTION_COUNTER.getAndIncrement() % 3
+
+        return ResponseEntity.ok(
+            when (index) {
+                0 -> MultiExecutionData("same", "a", "b")
+                1 -> MultiExecutionData("same", "x", "b")
+                else -> MultiExecutionData("same", "a", "y")
+            }
+        )
+    }
+
     @GetMapping("/price/estimate")
     fun estimatePrice(@RequestParam base: Int): Map<String, Int> {
         val randomJitter = randomInt(1000)
@@ -103,4 +118,10 @@ data class TimeAgoData(
     val calculatedPastTime: String
 )
 
-class EmptyForFlag(val flag: String)
+data class MultiExecutionData(
+    val stable: String,
+    val first: String,
+    val second: String
+)
+
+class EmptyForFlag(val flag: String)

core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/flakinessdetect/FlakinessDetectBlackboxEMTest.kt

@@ -36,6 +36,7 @@ class FlakinessDetectBlackboxEMTest : SpringTestBase() {
 
 
             setOption(args, "handleFlakiness", "true")
+            setOption(args, "execNumForDetectFlakiness", "1")
 
             // we may still need to specify info in non bb-e2etest
             setOption(args, "blackBox", "true")
@@ -52,4 +53,4 @@ class FlakinessDetectBlackboxEMTest : SpringTestBase() {
 
         }
     }
-}
+}

core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/flakinessdetect/FlakinessDetectEMTest.kt

@@ -37,6 +37,7 @@ class FlakinessDetectEMTest : SpringTestBase() {
 
             setOption(args, "minimize", "true")
             setOption(args, "handleFlakiness", "true")
+            setOption(args, "execNumForDetectFlakiness", "1")
 
 
             val solution = initAndRun(args)

core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/flakinessdetect/FlakinessDetectWithMultiExecEMTest.kt

@@ -0,0 +1,44 @@
+package org.evomaster.e2etests.spring.openapi.v3.flakinessdetect
+
+import com.foo.rest.examples.spring.openapi.v3.flakinessdetect.FlakinessDetectController
+import org.evomaster.e2etests.spring.openapi.v3.SpringTestBase
+import org.junit.jupiter.api.Assertions.assertTrue
+import org.junit.jupiter.api.BeforeAll
+import org.junit.jupiter.api.Test
+
+class FlakinessDetectWithMultiExecEMTest : SpringTestBase() {
+
+    companion object {
+        @BeforeAll
+        @JvmStatic
+        fun init() {
+            initClass(FlakinessDetectController())
+        }
+    }
+
+    @Test
+    fun testRunEMWithMultipleFlakinessExecutions() {
+
+        val outputFolder = "FlakinessDetectMultipleExecutionsEM"
+        val outputClass = "org.foo.FlakinessDetectMultipleExecutionsEM"
+
+        runTestHandlingFlakyAndCompilation(
+            outputFolder,
+            outputClass,
+            30,
+            true
+        ) { args: MutableList<String> ->
+
+            setOption(args, "minimize", "true")
+            setOption(args, "handleFlakiness", "true")
+            setOption(args, "execNumForDetectFlakiness", "2")
+            setOption(args, "endpointFocus", "/api/flakinessdetect/multiexecution")
+
+            val solution = initAndRun(args)
+
+            assertTrue(solution.individuals.isNotEmpty())
+            assertTextInTests(outputFolder, outputClass, "Flaky value of field \"'first'\"")
+            assertTextInTests(outputFolder, outputClass, "Flaky value of field \"'second'\"")
+        }
+    }
+}

core/src/main/kotlin/org/evomaster/core/EMConfig.kt

@@ -2793,6 +2793,17 @@ class EMConfig {
             "Note that flakiness is now supported only for fuzzing REST APIs")
     var handleFlakiness = false
 
+    @Experimental
+    @DependsOnTrueFor("handleFlakiness")
+    @Cfg("Specify whether to infer potential flakiness statically from response values, such as timestamps, UUIDs, hashes and runtime-specific messages.")
+    var enableStaticFlakyInference = true
+
+    @Experimental
+    @Min(0.0)
+    @DependsOnTrueFor("handleFlakiness")
+    @Cfg("Specify the number of re-executions for detecting flakiness in tests. Set to 0 to disable re-execution based flakiness detection.")
+    var execNumForDetectFlakiness = 1
+
     @Experimental
     @Cfg("Use environment variables to define the paths required by External Drivers. " +
             "This is necessary when the generated tests are executed on the different machine. " +

core/src/main/kotlin/org/evomaster/core/Main.kt

@@ -451,7 +451,7 @@ class Main {
                     epc.markStartingFlakiness()
 
                     val flakinessDetector = injector.getInstance(Key.get(object : TypeLiteral<FlakinessDetector<RestIndividual>>() {}))
-                    flakinessDetector.reexecuteToDetectFlakiness()
+                    flakinessDetector.applyPhase()
                 } else -> {
                     LoggingUtil.getInfoLogger()
                         .warn("Flakiness detection phase currently not handled for problem type: ${config.problemType}")

core/src/main/kotlin/org/evomaster/core/output/service/HttpWsTestCaseWriter.kt

@@ -847,15 +847,15 @@ abstract class HttpWsTestCaseWriter : ApiTestCaseWriter() {
                 lines.append("JsonConvert.DeserializeObject(await $responseVariableName.Content.ReadAsStringAsync());")
             }
 
-            handleJsonStringAssertion(bodyString, res.getFlakyBody(), lines, bodyVarName, res.getTooLargeBody())
+            handleJsonStringAssertion(bodyString, res.getFlakyBodies()?.let { res.getMergedFlakyBody() }, lines, bodyVarName, res.getTooLargeBody())
 
         } else if (type.isCompatible(MediaType.TEXT_PLAIN_TYPE)) {
 
             if (format.isCsharp()) {
                 lines.append("await $responseVariableName.Content.ReadAsStringAsync();")
             }
 
-            handleTextPlainTextAssertion(bodyString, res.getFlakyBody(), lines, bodyVarName)
+            handleTextPlainTextAssertion(bodyString, res.getFlakyBodies()?.let { res.getMergedFlakyBody() }, lines, bodyVarName)
         } else {
             if (format.isCsharp()) {
                 lines.append("await $responseVariableName.Content.ReadAsStringAsync();")