diff --git a/core/src/main/kotlin/org/evomaster/core/BaseModule.kt b/core/src/main/kotlin/org/evomaster/core/BaseModule.kt index 372164494c..c5bae0bf09 100644 --- a/core/src/main/kotlin/org/evomaster/core/BaseModule.kt +++ b/core/src/main/kotlin/org/evomaster/core/BaseModule.kt @@ -92,6 +92,9 @@ class BaseModule(val args: Array, val noTests: Boolean = false) : Abstra bind(ExecutionStats::class.java) .asEagerSingleton() + bind(WarningsAggregator::class.java) + .asEagerSingleton() + //no longer needed if TestSuiteWriter is moved out? // if(noTests){ // bind(TestCaseWriter::class.java) diff --git a/core/src/main/kotlin/org/evomaster/core/output/TestCaseCode.kt b/core/src/main/kotlin/org/evomaster/core/output/TestCaseCode.kt index 0ec9d10c3c..5d5709dd48 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/TestCaseCode.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/TestCaseCode.kt @@ -10,7 +10,8 @@ class TestCaseCode( val evaluatedIndividual: EvaluatedIndividual<*>, val code: String, val startLine: Int, - val endLine: Int + val endLine: Int, + val namedExamples: Set? ) { init { diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt index 0a592c26f0..9ae514e51d 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt @@ -550,9 +550,7 @@ class RestTestCaseWriter : HttpWsTestCaseWriter { private fun getAllUsedExamples(ind: RestIndividual) : List{ - return ind.seeFullTreeGenes() - .filter { it is UserExamplesGene && it.isUsedForExamples() } - .filter { it.staticCheckIfImpactPhenotype() } + return ind.getAllActiveUsedExamples() .map { val name = if(it is UserExamplesGene){ //always true diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt index 3d80d3d5c7..fe08c169ec 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt @@ -22,6 +22,7 @@ import org.evomaster.core.problem.rest.data.RestIndividual import org.evomaster.core.problem.security.service.HttpCallbackVerifier import org.evomaster.core.remote.service.RemoteController import org.evomaster.core.search.Solution +import org.evomaster.core.search.gene.interfaces.UserExamplesGene import org.evomaster.core.search.service.Sampler import org.evomaster.core.search.service.time.SearchTimeController import org.evomaster.core.sql.schema.TableId @@ -190,10 +191,16 @@ class TestSuiteWriter { null } else { lines.addEmpty(2) + val start = lines.nextLineNumber() lines.add(testLines) val end = lines.nextLineNumber() - 1 - TestCaseCode(test.name,test.test,testLines.toString(), start, end) + + val examples = test.test.individual.getAllActiveUsedExamples() + .filterIsInstance() + .mapNotNull { it.getValueName() } + .toSet() + TestCaseCode(test.name,test.test,testLines.toString(), start, end, examples) } } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseSampler.kt b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseSampler.kt index 7e5bc7b013..5d00925fa4 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseSampler.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseSampler.kt @@ -12,6 +12,7 @@ import org.evomaster.core.remote.SutProblemException import org.evomaster.core.remote.service.RemoteController import org.evomaster.core.search.Individual import org.evomaster.core.search.service.Sampler +import org.evomaster.core.search.service.WarningsAggregator import org.evomaster.core.sql.SqlAction import org.evomaster.core.sql.SqlInsertBuilder import org.evomaster.core.sql.schema.TableId @@ -27,6 +28,10 @@ abstract class EnterpriseSampler : Sampler() where T : Individual { @Inject(optional = true) protected lateinit var rc: RemoteController + @Inject + protected lateinit var warningsAggregator: WarningsAggregator + + protected val derivedParamHandler = DerivedParamHandler() var sqlInsertBuilder: SqlInsertBuilder? = null diff --git a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/WFCReportWriter.kt b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/WFCReportWriter.kt index d5b8cf0165..a92babea66 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/WFCReportWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/WFCReportWriter.kt @@ -12,6 +12,7 @@ import org.evomaster.core.search.Solution import org.evomaster.core.logging.LoggingUtil import org.evomaster.core.search.service.Sampler import org.evomaster.core.search.service.Statistics +import org.evomaster.core.search.service.WarningsAggregator import org.jsoup.Jsoup import org.jsoup.nodes.DataNode import java.nio.file.Files @@ -32,6 +33,9 @@ class WFCReportWriter { @Inject private lateinit var sampler: Sampler<*> + @Inject + private lateinit var warningsAggregator: WarningsAggregator + private var lastTestFilePaths: Set = emptySet() private fun getTestId(suite: TestSuiteCode, test: TestCaseCode) = @@ -178,6 +182,8 @@ class WFCReportWriter { tc.filePath = suite.testSuitePath tc.startLine = test.startLine tc.endLine = test.endLine + tc.namedExamples = test.namedExamples?.toList() + report.testCases.add(tc) } } @@ -234,6 +240,16 @@ class WFCReportWriter { report.extra.add(coverage) } + val warnings = warningsAggregator.getWarnings() + if(warnings.isNotEmpty()){ + report.warnings = warnings.map { w -> + Warning().apply { + message = w.message + category = w.category.name + displayPriority = w.category.displayPriority + } + } + } val jackson = ObjectMapper() val json = jackson.writeValueAsString(report) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/httpws/service/HttpWsSampler.kt b/core/src/main/kotlin/org/evomaster/core/problem/httpws/service/HttpWsSampler.kt index 170a47604a..94ae1048c9 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/httpws/service/HttpWsSampler.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/httpws/service/HttpWsSampler.kt @@ -1,5 +1,6 @@ package org.evomaster.core.problem.httpws.service +import com.google.inject.Inject import com.webfuzzing.commons.auth.Header import org.evomaster.client.java.controller.api.dto.auth.AuthenticationDto import org.evomaster.client.java.controller.api.dto.SutInfoDto @@ -12,6 +13,9 @@ import org.evomaster.core.problem.httpws.HttpWsAction import org.evomaster.core.problem.httpws.auth.* import org.evomaster.core.remote.SutProblemException import org.evomaster.core.search.Individual +import org.evomaster.core.search.service.WarningsAggregator +import org.evomaster.core.search.warning.GeneralWarning +import org.evomaster.core.search.warning.WarningCategory import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -25,6 +29,7 @@ abstract class HttpWsSampler : ApiWsSampler() where T : Individual{ private val log: Logger = LoggerFactory.getLogger(HttpWsSampler::class.java) } + //TODO move up to Enterprise val authentications = AuthSettings() @@ -90,27 +95,28 @@ abstract class HttpWsSampler : ApiWsSampler() where T : Individual{ private fun checkAuthSettings(){ val n = authentications.size() if(n==0){ - LoggingUtil.uniqueUserWarn( - "No authentication info was provided." + + val msg = "No authentication info was provided." + " Unless you are testing an example API, you should setup some authentication info for different users." + " If this is the first time you are using EvoMaster, and you just want to get a feeling of how it works," + " then ignore this warning." + " However, to get better results, you will need setup authentication info, eventually." + - " More info is currently available at " + AnsiColor.inBlue(EM_AUTH_LINK) - ) - } - if(n==1){ + " More info is currently available at " + LoggingUtil.uniqueUserWarn(msg + AnsiColor.inBlue(EM_AUTH_LINK)) + warningsAggregator.addWarning(GeneralWarning(WarningCategory.FUZZER, msg + EM_AUTH_LINK)) + + }else if(n==1){ + val msg = "You have provided authentication information only for a single user." + + " Many of the automatic checks done by EvoMaster for access policy validation are based on the" + + " interactions of 2 or more users." + + " To get better results, you are strongly recommended to provide more user authentication info," + + " at the very minimum 2 in total, but better if at least 1 for each different access role you have in your system" + + " that you are testing." + + " More info is currently available at " + //TODO if/when in the future we enable dynamic registration of users, likely we will need to update this // warning message - LoggingUtil.uniqueUserWarn( - "You have provided authentication information only for a single user." + - " Many of the automatic checks done by EvoMaster for access policy validation are based on the" + - " interactions of 2 or more users." + - " To get better results, you are strongly recommended to provide more user authentication info," + - " at the very minimum 2 in total, but better if at least 1 for each different access role you have in your system" + - " that you are testing." + - " More info is currently available at " + AnsiColor.inBlue(EM_AUTH_LINK) - ) + LoggingUtil.uniqueUserWarn(msg + AnsiColor.inBlue(EM_AUTH_LINK)) + warningsAggregator.addWarning(GeneralWarning(WarningCategory.FUZZER, msg + EM_AUTH_LINK)) } } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt index 176926ee47..bc0fba16c2 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt @@ -57,6 +57,9 @@ import org.evomaster.core.search.gene.string.StringGene import org.evomaster.core.search.gene.utils.GeneUtils import org.evomaster.core.search.service.DataPool import org.evomaster.core.search.service.ExecutionStats +import org.evomaster.core.search.service.WarningsAggregator +import org.evomaster.core.search.warning.GeneralWarning +import org.evomaster.core.search.warning.WarningCategory import org.evomaster.core.taint.TaintAnalysis import org.evomaster.core.utils.TimeUtils import org.slf4j.Logger @@ -104,6 +107,9 @@ abstract class AbstractRestFitness : HttpWsFitness() { @Inject protected lateinit var securityOracle: RestSecurityOracle + @Inject + protected lateinit var warningsAggregator: WarningsAggregator + //TODO refactor private lateinit var schemaOracle: RestSchemaOracle @@ -778,7 +784,10 @@ abstract class AbstractRestFitness : HttpWsFitness() { */ LoggingUtil.getInfoLogger().warn("Hit a rate-limiter. Received a response with 429 'Too Many Requests'.") - //TODO should add these on warnings for WFC Report + val detailedMessage = "A 429 'Too Many Requests' was encountered." + + " If you are fuzzing an API, it is strongly recommended to disable the rate limiter, if possible," + + " as it hinders how many test cases the fuzzer can evaluate in the same amount of time." + warningsAggregator.addWarning(GeneralWarning(WarningCategory.SUT,detailedMessage)) val retryAfter = response.getHeaderString("Retry-After") val delay = if(retryAfter == null){ diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/sampler/AbstractRestSampler.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/sampler/AbstractRestSampler.kt index 05b4a40231..f826a3bdfc 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/sampler/AbstractRestSampler.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/sampler/AbstractRestSampler.kt @@ -8,7 +8,6 @@ import org.evomaster.core.AnsiColor import org.evomaster.core.EMConfig import org.evomaster.core.config.ConfigProblemException import org.evomaster.core.logging.LoggingUtil -import org.evomaster.core.output.service.PartialOracles import org.evomaster.core.problem.enterprise.SampleType import org.evomaster.core.problem.externalservice.ExternalService import org.evomaster.core.problem.externalservice.HostnameResolutionInfo @@ -39,7 +38,10 @@ import org.evomaster.core.search.action.Action import org.evomaster.core.search.gene.wrapper.CustomMutationRateGene import org.evomaster.core.search.gene.wrapper.OptionalGene import org.evomaster.core.search.gene.string.StringGene +import org.evomaster.core.search.service.WarningsAggregator import org.evomaster.core.search.tracer.Traceable +import org.evomaster.core.search.warning.GeneralWarning +import org.evomaster.core.search.warning.WarningCategory import org.slf4j.Logger import org.slf4j.LoggerFactory import javax.annotation.PostConstruct @@ -124,7 +126,7 @@ abstract class AbstractRestSampler : HttpWsSampler() { actionCluster.clear() skippedEndpoints = EndpointFilter.getEndpointsToSkip(config, schemaHolder, infoDto) val messages = RestActionBuilderV3.addActionsFromSwagger(schemaHolder, actionCluster, skippedEndpoints, RestActionBuilderV3.Options(config)) - printMessages(messages) + handleMessages(messages) if(config.extraQueryParam){ addExtraQueryParam(actionCluster) @@ -325,7 +327,7 @@ abstract class AbstractRestSampler : HttpWsSampler() { // Add all paths to list of paths to ignore except endpointFocus skippedEndpoints = EndpointFilter.getEndpointsToSkip(config,schemaHolder) val messages = RestActionBuilderV3.addActionsFromSwagger(schemaHolder, actionCluster, skippedEndpoints, RestActionBuilderV3.Options(config)) - printMessages(messages) + handleMessages(messages) initAdHocInitialIndividuals() if (config.seedTestCases) { @@ -335,7 +337,7 @@ abstract class AbstractRestSampler : HttpWsSampler() { log.debug("Done initializing {}", AbstractRestSampler::class.simpleName) } - private fun printMessages(messages: List){ + private fun handleMessages(messages: List){ if(messages.isEmpty()){ return } @@ -345,6 +347,8 @@ abstract class AbstractRestSampler : HttpWsSampler() { " itself.")) messages.forEachIndexed { index, s -> LoggingUtil.getInfoLogger().warn(AnsiColor.inYellow("$index: $s")) + + warningsAggregator.addWarning(GeneralWarning(WarningCategory.SCHEMA,s)) } } diff --git a/core/src/main/kotlin/org/evomaster/core/search/Individual.kt b/core/src/main/kotlin/org/evomaster/core/search/Individual.kt index 77a1971473..f2f763ba9f 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/Individual.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/Individual.kt @@ -12,6 +12,7 @@ import org.evomaster.core.problem.externalservice.ApiExternalServiceAction import org.evomaster.core.search.action.* import org.evomaster.core.search.gene.Gene import org.evomaster.core.search.gene.interfaces.TaintableGene +import org.evomaster.core.search.gene.interfaces.UserExamplesGene import org.evomaster.core.search.gene.wrapper.OptionalGene import org.evomaster.core.search.service.Randomness import org.evomaster.core.search.service.SearchGlobalState @@ -233,6 +234,12 @@ abstract class Individual( return seeTopGenes(filter).flatMap { it.flatView() } } + fun getAllActiveUsedExamples() : List { + return seeFullTreeGenes() + .filter { it is UserExamplesGene && it.isUsedForExamples() } + .filter { it.staticCheckIfImpactPhenotype() } + } + /** * An estimation of the "size" of this individual. * Longer/bigger individuals are usually considered worse, diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/WarningsAggregator.kt b/core/src/main/kotlin/org/evomaster/core/search/service/WarningsAggregator.kt new file mode 100644 index 0000000000..a1f742ead4 --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/search/service/WarningsAggregator.kt @@ -0,0 +1,15 @@ +package org.evomaster.core.search.service + +import org.evomaster.core.search.warning.GeneralWarning +import java.util.concurrent.CopyOnWriteArraySet + +class WarningsAggregator { + + private val warnings : MutableSet = CopyOnWriteArraySet() + + fun addWarning(warning: GeneralWarning) { + warnings.add(warning) + } + + fun getWarnings() : Set = warnings +} \ No newline at end of file diff --git a/core/src/main/kotlin/org/evomaster/core/search/warning/GeneralWarning.kt b/core/src/main/kotlin/org/evomaster/core/search/warning/GeneralWarning.kt new file mode 100644 index 0000000000..2b0c34849f --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/search/warning/GeneralWarning.kt @@ -0,0 +1,15 @@ +package org.evomaster.core.search.warning + +data class GeneralWarning( + + /** + * Label to specify the type of warning, e.g., if related to the schema or + * fuzzer misconfiguration. + */ + val category: WarningCategory, + + /** + * The textual content of the warning message. + */ + val message: String +) diff --git a/core/src/main/kotlin/org/evomaster/core/search/warning/WarningCategory.kt b/core/src/main/kotlin/org/evomaster/core/search/warning/WarningCategory.kt new file mode 100644 index 0000000000..87f1d67bb8 --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/search/warning/WarningCategory.kt @@ -0,0 +1,26 @@ +package org.evomaster.core.search.warning + +enum class WarningCategory( + /** + * Hint on the priority of this warning message, where lowest (eg, 1) values mean + * more important. This is ONLY meant to be used for display purposes, e.g., when + * sorting the list of warnings to show to user. + */ + val displayPriority: Int +) { + + /** + * Issues related to the schema of the SUT + */ + SCHEMA(3), + + /** + * Issues related to how the fuzzer was configured (eg no auth configured) + */ + FUZZER(1), + + /** + * Issues related to what the SUT is returning or is configured (eg rate limiter is on) + */ + SUT(2) +} \ No newline at end of file diff --git a/pom.xml b/pom.xml index ce9cf52161..4217edf58e 100644 --- a/pom.xml +++ b/pom.xml @@ -103,7 +103,7 @@ 17 2.2.20 true - 0.4.1 + 0.5.0 5.14.2 1.14.2 3.1.5