starting to handle WFC Warnings and Examples

core/src/main/kotlin/org/evomaster/core/BaseModule.kt

@@ -92,6 +92,9 @@ class BaseModule(val args: Array<String>, val noTests: Boolean = false) : Abstra
         bind(ExecutionStats::class.java)
             .asEagerSingleton()
 
+        bind(WarningsAggregator::class.java)
+            .asEagerSingleton()
+
         //no longer needed if TestSuiteWriter is moved out?
 //        if(noTests){
 //            bind(TestCaseWriter::class.java)

core/src/main/kotlin/org/evomaster/core/output/TestCaseCode.kt

@@ -10,7 +10,8 @@ class TestCaseCode(
     val evaluatedIndividual: EvaluatedIndividual<*>,
     val code: String,
     val startLine: Int,
-    val endLine: Int
+    val endLine: Int,
+    val namedExamples: Set<String>?
 ) {
 
     init {

core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt

@@ -550,9 +550,7 @@ class RestTestCaseWriter : HttpWsTestCaseWriter {
 
 
     private fun getAllUsedExamples(ind: RestIndividual) : List<String>{
-        return ind.seeFullTreeGenes()
-            .filter { it is UserExamplesGene && it.isUsedForExamples() }
-            .filter { it.staticCheckIfImpactPhenotype() }
+        return ind.getAllActiveUsedExamples()
             .map {
                 val name = if(it is UserExamplesGene){
                     //always true

core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt

@@ -22,6 +22,7 @@ import org.evomaster.core.problem.rest.data.RestIndividual
 import org.evomaster.core.problem.security.service.HttpCallbackVerifier
 import org.evomaster.core.remote.service.RemoteController
 import org.evomaster.core.search.Solution
+import org.evomaster.core.search.gene.interfaces.UserExamplesGene
 import org.evomaster.core.search.service.Sampler
 import org.evomaster.core.search.service.time.SearchTimeController
 import org.evomaster.core.sql.schema.TableId
@@ -190,10 +191,16 @@ class TestSuiteWriter {
                 null
             } else {
                 lines.addEmpty(2)
+
                 val start = lines.nextLineNumber()
                 lines.add(testLines)
                 val end = lines.nextLineNumber() - 1
-                TestCaseCode(test.name,test.test,testLines.toString(), start, end)
+
+                val examples = test.test.individual.getAllActiveUsedExamples()
+                    .filterIsInstance<UserExamplesGene>()
+                    .mapNotNull { it.getValueName() }
+                    .toSet()
+                TestCaseCode(test.name,test.test,testLines.toString(), start, end, examples)
             }
         }
 

core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseSampler.kt

@@ -12,6 +12,7 @@ import org.evomaster.core.remote.SutProblemException
 import org.evomaster.core.remote.service.RemoteController
 import org.evomaster.core.search.Individual
 import org.evomaster.core.search.service.Sampler
+import org.evomaster.core.search.service.WarningsAggregator
 import org.evomaster.core.sql.SqlAction
 import org.evomaster.core.sql.SqlInsertBuilder
 import org.evomaster.core.sql.schema.TableId
@@ -27,6 +28,10 @@ abstract class EnterpriseSampler<T> : Sampler<T>() where T : Individual {
     @Inject(optional = true)
     protected lateinit var rc: RemoteController
 
+    @Inject
+    protected lateinit var warningsAggregator: WarningsAggregator
+
+
     protected val derivedParamHandler = DerivedParamHandler()
 
     var sqlInsertBuilder: SqlInsertBuilder? = null

core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/WFCReportWriter.kt

@@ -12,6 +12,7 @@ import org.evomaster.core.search.Solution
 import org.evomaster.core.logging.LoggingUtil
 import org.evomaster.core.search.service.Sampler
 import org.evomaster.core.search.service.Statistics
+import org.evomaster.core.search.service.WarningsAggregator
 import org.jsoup.Jsoup
 import org.jsoup.nodes.DataNode
 import java.nio.file.Files
@@ -32,6 +33,9 @@ class WFCReportWriter {
     @Inject
     private lateinit var sampler: Sampler<*>
 
+    @Inject
+    private lateinit var warningsAggregator: WarningsAggregator
+
     private var lastTestFilePaths: Set<String> = emptySet()
 
     private fun getTestId(suite: TestSuiteCode, test: TestCaseCode) =
@@ -178,6 +182,8 @@ class WFCReportWriter {
                 tc.filePath = suite.testSuitePath
                 tc.startLine = test.startLine
                 tc.endLine = test.endLine
+                tc.namedExamples = test.namedExamples?.toList()
+
                 report.testCases.add(tc)
             }
         }
@@ -234,6 +240,16 @@ class WFCReportWriter {
             report.extra.add(coverage)
         }
 
+        val warnings = warningsAggregator.getWarnings()
+        if(warnings.isNotEmpty()){
+            report.warnings = warnings.map { w ->
+                Warning().apply {
+                    message = w.message
+                    category = w.category.name
+                    displayPriority = w.category.displayPriority
+                }
+            }
+        }
 
         val jackson = ObjectMapper()
         val json = jackson.writeValueAsString(report)

core/src/main/kotlin/org/evomaster/core/problem/httpws/service/HttpWsSampler.kt

@@ -1,5 +1,6 @@
 package org.evomaster.core.problem.httpws.service
 
+import com.google.inject.Inject
 import com.webfuzzing.commons.auth.Header
 import org.evomaster.client.java.controller.api.dto.auth.AuthenticationDto
 import org.evomaster.client.java.controller.api.dto.SutInfoDto
@@ -12,6 +13,9 @@ import org.evomaster.core.problem.httpws.HttpWsAction
 import org.evomaster.core.problem.httpws.auth.*
 import org.evomaster.core.remote.SutProblemException
 import org.evomaster.core.search.Individual
+import org.evomaster.core.search.service.WarningsAggregator
+import org.evomaster.core.search.warning.GeneralWarning
+import org.evomaster.core.search.warning.WarningCategory
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 
@@ -25,6 +29,7 @@ abstract class HttpWsSampler<T> : ApiWsSampler<T>() where T : Individual{
         private val log: Logger = LoggerFactory.getLogger(HttpWsSampler::class.java)
     }
 
+
     //TODO move up to Enterprise
     val authentications = AuthSettings()
 
@@ -90,27 +95,28 @@ abstract class HttpWsSampler<T> : ApiWsSampler<T>() where T : Individual{
     private fun checkAuthSettings(){
         val n = authentications.size()
         if(n==0){
-            LoggingUtil.uniqueUserWarn(
-                "No authentication info was provided." +
+            val msg = "No authentication info was provided." +
                     " Unless you are testing an example API, you should setup some authentication info for different users." +
                     " If this is the first time you are using EvoMaster, and you just want to get a feeling of how it works," +
                     " then ignore this warning." +
                     " However, to get better results, you will need setup authentication info, eventually." +
-                    " More info is currently available at " + AnsiColor.inBlue(EM_AUTH_LINK)
-            )
-        }
-        if(n==1){
+                    " More info is currently available at "
+            LoggingUtil.uniqueUserWarn(msg + AnsiColor.inBlue(EM_AUTH_LINK))
+            warningsAggregator.addWarning(GeneralWarning(WarningCategory.FUZZER, msg + EM_AUTH_LINK))
+
+        }else if(n==1){
+            val msg = "You have provided authentication information only for a single user." +
+                    " Many of the automatic checks done by EvoMaster for access policy validation are based on the" +
+                    " interactions of 2 or more users." +
+                    " To get better results, you are strongly recommended to provide more user authentication info," +
+                    " at the very minimum 2 in total, but better if at least 1 for each different access role you have in your system" +
+                    " that you are testing." +
+                    " More info is currently available at "
+
             //TODO if/when in the future we enable dynamic registration of users, likely we will need to update this
             // warning message
-            LoggingUtil.uniqueUserWarn(
-                "You have provided authentication information only for a single user." +
-                        " Many of the automatic checks done by EvoMaster for access policy validation are based on the" +
-                        " interactions of 2 or more users." +
-                        " To get better results, you are strongly recommended to provide more user authentication info," +
-                        " at the very minimum 2 in total, but better if at least 1 for each different access role you have in your system" +
-                        " that you are testing." +
-                        " More info is currently available at " + AnsiColor.inBlue(EM_AUTH_LINK)
-            )
+            LoggingUtil.uniqueUserWarn(msg + AnsiColor.inBlue(EM_AUTH_LINK))
+            warningsAggregator.addWarning(GeneralWarning(WarningCategory.FUZZER, msg + EM_AUTH_LINK))
         }
     }
 

core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt

@@ -57,6 +57,9 @@ import org.evomaster.core.search.gene.string.StringGene
 import org.evomaster.core.search.gene.utils.GeneUtils
 import org.evomaster.core.search.service.DataPool
 import org.evomaster.core.search.service.ExecutionStats
+import org.evomaster.core.search.service.WarningsAggregator
+import org.evomaster.core.search.warning.GeneralWarning
+import org.evomaster.core.search.warning.WarningCategory
 import org.evomaster.core.taint.TaintAnalysis
 import org.evomaster.core.utils.TimeUtils
 import org.slf4j.Logger
@@ -104,6 +107,9 @@ abstract class AbstractRestFitness : HttpWsFitness<RestIndividual>() {
     @Inject
     protected lateinit var securityOracle: RestSecurityOracle
 
+    @Inject
+    protected lateinit var warningsAggregator: WarningsAggregator
+
     //TODO refactor
     private lateinit var schemaOracle: RestSchemaOracle
 
@@ -778,7 +784,10 @@ abstract class AbstractRestFitness : HttpWsFitness<RestIndividual>() {
              */
             LoggingUtil.getInfoLogger().warn("Hit a rate-limiter. Received a response with 429 'Too Many Requests'.")
 
-            //TODO should add these on warnings for WFC Report
+            val detailedMessage = "A 429 'Too Many Requests' was encountered." +
+                    " If you are fuzzing an API, it is strongly recommended to disable the rate limiter, if possible," +
+                    " as it hinders how many test cases the fuzzer can evaluate in the same amount of time."
+            warningsAggregator.addWarning(GeneralWarning(WarningCategory.SUT,detailedMessage))
 
             val retryAfter = response.getHeaderString("Retry-After")
             val delay = if(retryAfter == null){

core/src/main/kotlin/org/evomaster/core/problem/rest/service/sampler/AbstractRestSampler.kt

@@ -8,7 +8,6 @@ import org.evomaster.core.AnsiColor
 import org.evomaster.core.EMConfig
 import org.evomaster.core.config.ConfigProblemException
 import org.evomaster.core.logging.LoggingUtil
-import org.evomaster.core.output.service.PartialOracles
 import org.evomaster.core.problem.enterprise.SampleType
 import org.evomaster.core.problem.externalservice.ExternalService
 import org.evomaster.core.problem.externalservice.HostnameResolutionInfo
@@ -39,7 +38,10 @@ import org.evomaster.core.search.action.Action
 import org.evomaster.core.search.gene.wrapper.CustomMutationRateGene
 import org.evomaster.core.search.gene.wrapper.OptionalGene
 import org.evomaster.core.search.gene.string.StringGene
+import org.evomaster.core.search.service.WarningsAggregator
 import org.evomaster.core.search.tracer.Traceable
+import org.evomaster.core.search.warning.GeneralWarning
+import org.evomaster.core.search.warning.WarningCategory
 import org.slf4j.Logger
 import org.slf4j.LoggerFactory
 import javax.annotation.PostConstruct
@@ -124,7 +126,7 @@ abstract class AbstractRestSampler : HttpWsSampler<RestIndividual>() {
         actionCluster.clear()
         skippedEndpoints = EndpointFilter.getEndpointsToSkip(config, schemaHolder, infoDto)
         val messages = RestActionBuilderV3.addActionsFromSwagger(schemaHolder, actionCluster, skippedEndpoints, RestActionBuilderV3.Options(config))
-        printMessages(messages)
+        handleMessages(messages)
 
         if(config.extraQueryParam){
             addExtraQueryParam(actionCluster)
@@ -325,7 +327,7 @@ abstract class AbstractRestSampler : HttpWsSampler<RestIndividual>() {
         // Add all paths to list of paths to ignore except endpointFocus
         skippedEndpoints = EndpointFilter.getEndpointsToSkip(config,schemaHolder)
         val messages = RestActionBuilderV3.addActionsFromSwagger(schemaHolder, actionCluster, skippedEndpoints, RestActionBuilderV3.Options(config))
-        printMessages(messages)
+        handleMessages(messages)
 
         initAdHocInitialIndividuals()
         if (config.seedTestCases) {
@@ -335,7 +337,7 @@ abstract class AbstractRestSampler : HttpWsSampler<RestIndividual>() {
         log.debug("Done initializing {}", AbstractRestSampler::class.simpleName)
     }
 
-    private fun printMessages(messages: List<String>){
+    private fun handleMessages(messages: List<String>){
         if(messages.isEmpty()){
             return
         }
@@ -345,6 +347,8 @@ abstract class AbstractRestSampler : HttpWsSampler<RestIndividual>() {
                 " itself."))
         messages.forEachIndexed { index, s ->
             LoggingUtil.getInfoLogger().warn(AnsiColor.inYellow("$index: $s"))
+
+            warningsAggregator.addWarning(GeneralWarning(WarningCategory.SCHEMA,s))
         }
     }
 

core/src/main/kotlin/org/evomaster/core/search/Individual.kt

@@ -12,6 +12,7 @@ import org.evomaster.core.problem.externalservice.ApiExternalServiceAction
 import org.evomaster.core.search.action.*
 import org.evomaster.core.search.gene.Gene
 import org.evomaster.core.search.gene.interfaces.TaintableGene
+import org.evomaster.core.search.gene.interfaces.UserExamplesGene
 import org.evomaster.core.search.gene.wrapper.OptionalGene
 import org.evomaster.core.search.service.Randomness
 import org.evomaster.core.search.service.SearchGlobalState
@@ -233,6 +234,12 @@ abstract class Individual(
         return seeTopGenes(filter).flatMap { it.flatView() }
     }
 
+    fun getAllActiveUsedExamples() : List<Gene>  {
+        return seeFullTreeGenes()
+            .filter { it is UserExamplesGene && it.isUsedForExamples() }
+            .filter { it.staticCheckIfImpactPhenotype() }
+    }
+
     /**
      * An estimation of the "size" of this individual.
      * Longer/bigger individuals are usually considered worse,

core/src/main/kotlin/org/evomaster/core/search/service/WarningsAggregator.kt

@@ -0,0 +1,15 @@
+package org.evomaster.core.search.service
+
+import org.evomaster.core.search.warning.GeneralWarning
+import java.util.concurrent.CopyOnWriteArraySet
+
+class WarningsAggregator {
+
+    private val warnings : MutableSet<GeneralWarning> = CopyOnWriteArraySet()
+
+    fun addWarning(warning: GeneralWarning) {
+        warnings.add(warning)
+    }
+
+    fun getWarnings() : Set<GeneralWarning> = warnings
+}

core/src/main/kotlin/org/evomaster/core/search/warning/GeneralWarning.kt

@@ -0,0 +1,15 @@
+package org.evomaster.core.search.warning
+
+data class GeneralWarning(
+
+    /**
+     * Label to specify the type of warning, e.g., if related to the schema or
+     * fuzzer misconfiguration.
+     */
+    val category: WarningCategory,
+
+    /**
+     * The textual content of the warning message.
+     */
+    val message: String
+)

core/src/main/kotlin/org/evomaster/core/search/warning/WarningCategory.kt

@@ -0,0 +1,26 @@
+package org.evomaster.core.search.warning
+
+enum class WarningCategory(
+    /**
+     * Hint on the priority of this warning message, where lowest (eg, 1) values mean
+     * more important. This is ONLY meant to be used for display purposes, e.g., when
+     * sorting the list of warnings to show to user.
+     */
+    val displayPriority: Int
+) {
+
+    /**
+     * Issues related to the schema of the SUT
+     */
+    SCHEMA(3),
+
+    /**
+     * Issues related to how the fuzzer was configured (eg no auth configured)
+     */
+    FUZZER(1),
+
+    /**
+     * Issues related to what the SUT is returning or is configured (eg rate limiter is on)
+     */
+    SUT(2)
+}

pom.xml

@@ -103,7 +103,7 @@
         <java.version>17</java.version>
         <kotlin.version>2.2.20</kotlin.version>
         <kotlin.compiler.incremental>true</kotlin.compiler.incremental>
-        <wfc.version>0.4.1</wfc.version>
+        <wfc.version>0.5.0</wfc.version>
         <junit.jupiter.version>5.14.2</junit.jupiter.version>
         <junit.platform.version>1.14.2</junit.platform.version>
         <springboot3.version>3.1.5</springboot3.version>