Skip to content

Commit bb1a9e5

Browse files
lmasrocalmasroca
committed
Merge remote-tracking branch 'origin/master' into regex-support-extension
2 parents 5d722be + e009506 commit bb1a9e5

14 files changed

Lines changed: 138 additions & 26 deletions

File tree

‎core/src/main/kotlin/org/evomaster/core/BaseModule.kt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,9 @@ class BaseModule(val args: Array<String>, val noTests: Boolean = false) : Abstra
9292
bind(ExecutionStats::class.java)
9393
.asEagerSingleton()
9494

95+
bind(WarningsAggregator::class.java)
96+
.asEagerSingleton()
97+
9598
//no longer needed if TestSuiteWriter is moved out?
9699
// if(noTests){
97100
// bind(TestCaseWriter::class.java)

‎core/src/main/kotlin/org/evomaster/core/output/TestCaseCode.kt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,8 @@ class TestCaseCode(
1010
val evaluatedIndividual: EvaluatedIndividual<*>,
1111
val code: String,
1212
val startLine: Int,
13-
val endLine: Int
13+
val endLine: Int,
14+
val namedExamples: Set<String>?
1415
) {
1516

1617
init {

‎core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -550,9 +550,7 @@ class RestTestCaseWriter : HttpWsTestCaseWriter {
550550

551551

552552
private fun getAllUsedExamples(ind: RestIndividual) : List<String>{
553-
return ind.seeFullTreeGenes()
554-
.filter { it is UserExamplesGene && it.isUsedForExamples() }
555-
.filter { it.staticCheckIfImpactPhenotype() }
553+
return ind.getAllActiveUsedExamples()
556554
.map {
557555
val name = if(it is UserExamplesGene){
558556
//always true

‎core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import org.evomaster.core.problem.rest.data.RestIndividual
2222
import org.evomaster.core.problem.security.service.HttpCallbackVerifier
2323
import org.evomaster.core.remote.service.RemoteController
2424
import org.evomaster.core.search.Solution
25+
import org.evomaster.core.search.gene.interfaces.UserExamplesGene
2526
import org.evomaster.core.search.service.Sampler
2627
import org.evomaster.core.search.service.time.SearchTimeController
2728
import org.evomaster.core.sql.schema.TableId
@@ -190,10 +191,16 @@ class TestSuiteWriter {
190191
null
191192
} else {
192193
lines.addEmpty(2)
194+
193195
val start = lines.nextLineNumber()
194196
lines.add(testLines)
195197
val end = lines.nextLineNumber() - 1
196-
TestCaseCode(test.name,test.test,testLines.toString(), start, end)
198+
199+
val examples = test.test.individual.getAllActiveUsedExamples()
200+
.filterIsInstance<UserExamplesGene>()
201+
.mapNotNull { it.getValueName() }
202+
.toSet()
203+
TestCaseCode(test.name,test.test,testLines.toString(), start, end, examples)
197204
}
198205
}
199206

‎core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseSampler.kt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import org.evomaster.core.remote.SutProblemException
1212
import org.evomaster.core.remote.service.RemoteController
1313
import org.evomaster.core.search.Individual
1414
import org.evomaster.core.search.service.Sampler
15+
import org.evomaster.core.search.service.WarningsAggregator
1516
import org.evomaster.core.sql.SqlAction
1617
import org.evomaster.core.sql.SqlInsertBuilder
1718
import org.evomaster.core.sql.schema.TableId
@@ -27,6 +28,10 @@ abstract class EnterpriseSampler<T> : Sampler<T>() where T : Individual {
2728
@Inject(optional = true)
2829
protected lateinit var rc: RemoteController
2930

31+
@Inject
32+
protected lateinit var warningsAggregator: WarningsAggregator
33+
34+
3035
protected val derivedParamHandler = DerivedParamHandler()
3136

3237
var sqlInsertBuilder: SqlInsertBuilder? = null

‎core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/WFCReportWriter.kt‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import org.evomaster.core.search.Solution
1212
import org.evomaster.core.logging.LoggingUtil
1313
import org.evomaster.core.search.service.Sampler
1414
import org.evomaster.core.search.service.Statistics
15+
import org.evomaster.core.search.service.WarningsAggregator
1516
import org.jsoup.Jsoup
1617
import org.jsoup.nodes.DataNode
1718
import java.nio.file.Files
@@ -32,6 +33,9 @@ class WFCReportWriter {
3233
@Inject
3334
private lateinit var sampler: Sampler<*>
3435

36+
@Inject
37+
private lateinit var warningsAggregator: WarningsAggregator
38+
3539
private var lastTestFilePaths: Set<String> = emptySet()
3640

3741
private fun getTestId(suite: TestSuiteCode, test: TestCaseCode) =
@@ -178,6 +182,8 @@ class WFCReportWriter {
178182
tc.filePath = suite.testSuitePath
179183
tc.startLine = test.startLine
180184
tc.endLine = test.endLine
185+
tc.namedExamples = test.namedExamples?.toList()
186+
181187
report.testCases.add(tc)
182188
}
183189
}
@@ -234,6 +240,16 @@ class WFCReportWriter {
234240
report.extra.add(coverage)
235241
}
236242

243+
val warnings = warningsAggregator.getWarnings()
244+
if(warnings.isNotEmpty()){
245+
report.warnings = warnings.map { w ->
246+
Warning().apply {
247+
message = w.message
248+
category = w.category.name
249+
displayPriority = w.category.displayPriority
250+
}
251+
}
252+
}
237253

238254
val jackson = ObjectMapper()
239255
val json = jackson.writeValueAsString(report)

‎core/src/main/kotlin/org/evomaster/core/problem/httpws/service/HttpWsSampler.kt‎

Lines changed: 21 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package org.evomaster.core.problem.httpws.service
22

3+
import com.google.inject.Inject
34
import com.webfuzzing.commons.auth.Header
45
import org.evomaster.client.java.controller.api.dto.auth.AuthenticationDto
56
import org.evomaster.client.java.controller.api.dto.SutInfoDto
@@ -12,6 +13,9 @@ import org.evomaster.core.problem.httpws.HttpWsAction
1213
import org.evomaster.core.problem.httpws.auth.*
1314
import org.evomaster.core.remote.SutProblemException
1415
import org.evomaster.core.search.Individual
16+
import org.evomaster.core.search.service.WarningsAggregator
17+
import org.evomaster.core.search.warning.GeneralWarning
18+
import org.evomaster.core.search.warning.WarningCategory
1519
import org.slf4j.Logger
1620
import org.slf4j.LoggerFactory
1721

@@ -25,6 +29,7 @@ abstract class HttpWsSampler<T> : ApiWsSampler<T>() where T : Individual{
2529
private val log: Logger = LoggerFactory.getLogger(HttpWsSampler::class.java)
2630
}
2731

32+
2833
//TODO move up to Enterprise
2934
val authentications = AuthSettings()
3035

@@ -90,27 +95,28 @@ abstract class HttpWsSampler<T> : ApiWsSampler<T>() where T : Individual{
9095
private fun checkAuthSettings(){
9196
val n = authentications.size()
9297
if(n==0){
93-
LoggingUtil.uniqueUserWarn(
94-
"No authentication info was provided." +
98+
val msg = "No authentication info was provided." +
9599
" Unless you are testing an example API, you should setup some authentication info for different users." +
96100
" If this is the first time you are using EvoMaster, and you just want to get a feeling of how it works," +
97101
" then ignore this warning." +
98102
" However, to get better results, you will need setup authentication info, eventually." +
99-
" More info is currently available at " + AnsiColor.inBlue(EM_AUTH_LINK)
100-
)
101-
}
102-
if(n==1){
103+
" More info is currently available at "
104+
LoggingUtil.uniqueUserWarn(msg + AnsiColor.inBlue(EM_AUTH_LINK))
105+
warningsAggregator.addWarning(GeneralWarning(WarningCategory.FUZZER, msg + EM_AUTH_LINK))
106+
107+
}else if(n==1){
108+
val msg = "You have provided authentication information only for a single user." +
109+
" Many of the automatic checks done by EvoMaster for access policy validation are based on the" +
110+
" interactions of 2 or more users." +
111+
" To get better results, you are strongly recommended to provide more user authentication info," +
112+
" at the very minimum 2 in total, but better if at least 1 for each different access role you have in your system" +
113+
" that you are testing." +
114+
" More info is currently available at "
115+
103116
//TODO if/when in the future we enable dynamic registration of users, likely we will need to update this
104117
// warning message
105-
LoggingUtil.uniqueUserWarn(
106-
"You have provided authentication information only for a single user." +
107-
" Many of the automatic checks done by EvoMaster for access policy validation are based on the" +
108-
" interactions of 2 or more users." +
109-
" To get better results, you are strongly recommended to provide more user authentication info," +
110-
" at the very minimum 2 in total, but better if at least 1 for each different access role you have in your system" +
111-
" that you are testing." +
112-
" More info is currently available at " + AnsiColor.inBlue(EM_AUTH_LINK)
113-
)
118+
LoggingUtil.uniqueUserWarn(msg + AnsiColor.inBlue(EM_AUTH_LINK))
119+
warningsAggregator.addWarning(GeneralWarning(WarningCategory.FUZZER, msg + EM_AUTH_LINK))
114120
}
115121
}
116122

‎core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,9 @@ import org.evomaster.core.search.gene.string.StringGene
5757
import org.evomaster.core.search.gene.utils.GeneUtils
5858
import org.evomaster.core.search.service.DataPool
5959
import org.evomaster.core.search.service.ExecutionStats
60+
import org.evomaster.core.search.service.WarningsAggregator
61+
import org.evomaster.core.search.warning.GeneralWarning
62+
import org.evomaster.core.search.warning.WarningCategory
6063
import org.evomaster.core.taint.TaintAnalysis
6164
import org.evomaster.core.utils.TimeUtils
6265
import org.slf4j.Logger
@@ -104,6 +107,9 @@ abstract class AbstractRestFitness : HttpWsFitness<RestIndividual>() {
104107
@Inject
105108
protected lateinit var securityOracle: RestSecurityOracle
106109

110+
@Inject
111+
protected lateinit var warningsAggregator: WarningsAggregator
112+
107113
//TODO refactor
108114
private lateinit var schemaOracle: RestSchemaOracle
109115

@@ -778,7 +784,10 @@ abstract class AbstractRestFitness : HttpWsFitness<RestIndividual>() {
778784
*/
779785
LoggingUtil.getInfoLogger().warn("Hit a rate-limiter. Received a response with 429 'Too Many Requests'.")
780786

781-
//TODO should add these on warnings for WFC Report
787+
val detailedMessage = "A 429 'Too Many Requests' was encountered." +
788+
" If you are fuzzing an API, it is strongly recommended to disable the rate limiter, if possible," +
789+
" as it hinders how many test cases the fuzzer can evaluate in the same amount of time."
790+
warningsAggregator.addWarning(GeneralWarning(WarningCategory.SUT,detailedMessage))
782791

783792
val retryAfter = response.getHeaderString("Retry-After")
784793
val delay = if(retryAfter == null){

‎core/src/main/kotlin/org/evomaster/core/problem/rest/service/sampler/AbstractRestSampler.kt‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ import org.evomaster.core.AnsiColor
88
import org.evomaster.core.EMConfig
99
import org.evomaster.core.config.ConfigProblemException
1010
import org.evomaster.core.logging.LoggingUtil
11-
import org.evomaster.core.output.service.PartialOracles
1211
import org.evomaster.core.problem.enterprise.SampleType
1312
import org.evomaster.core.problem.externalservice.ExternalService
1413
import org.evomaster.core.problem.externalservice.HostnameResolutionInfo
@@ -39,7 +38,10 @@ import org.evomaster.core.search.action.Action
3938
import org.evomaster.core.search.gene.wrapper.CustomMutationRateGene
4039
import org.evomaster.core.search.gene.wrapper.OptionalGene
4140
import org.evomaster.core.search.gene.string.StringGene
41+
import org.evomaster.core.search.service.WarningsAggregator
4242
import org.evomaster.core.search.tracer.Traceable
43+
import org.evomaster.core.search.warning.GeneralWarning
44+
import org.evomaster.core.search.warning.WarningCategory
4345
import org.slf4j.Logger
4446
import org.slf4j.LoggerFactory
4547
import javax.annotation.PostConstruct
@@ -124,7 +126,7 @@ abstract class AbstractRestSampler : HttpWsSampler<RestIndividual>() {
124126
actionCluster.clear()
125127
skippedEndpoints = EndpointFilter.getEndpointsToSkip(config, schemaHolder, infoDto)
126128
val messages = RestActionBuilderV3.addActionsFromSwagger(schemaHolder, actionCluster, skippedEndpoints, RestActionBuilderV3.Options(config))
127-
printMessages(messages)
129+
handleMessages(messages)
128130

129131
if(config.extraQueryParam){
130132
addExtraQueryParam(actionCluster)
@@ -325,7 +327,7 @@ abstract class AbstractRestSampler : HttpWsSampler<RestIndividual>() {
325327
// Add all paths to list of paths to ignore except endpointFocus
326328
skippedEndpoints = EndpointFilter.getEndpointsToSkip(config,schemaHolder)
327329
val messages = RestActionBuilderV3.addActionsFromSwagger(schemaHolder, actionCluster, skippedEndpoints, RestActionBuilderV3.Options(config))
328-
printMessages(messages)
330+
handleMessages(messages)
329331

330332
initAdHocInitialIndividuals()
331333
if (config.seedTestCases) {
@@ -335,7 +337,7 @@ abstract class AbstractRestSampler : HttpWsSampler<RestIndividual>() {
335337
log.debug("Done initializing {}", AbstractRestSampler::class.simpleName)
336338
}
337339

338-
private fun printMessages(messages: List<String>){
340+
private fun handleMessages(messages: List<String>){
339341
if(messages.isEmpty()){
340342
return
341343
}
@@ -345,6 +347,8 @@ abstract class AbstractRestSampler : HttpWsSampler<RestIndividual>() {
345347
" itself."))
346348
messages.forEachIndexed { index, s ->
347349
LoggingUtil.getInfoLogger().warn(AnsiColor.inYellow("$index: $s"))
350+
351+
warningsAggregator.addWarning(GeneralWarning(WarningCategory.SCHEMA,s))
348352
}
349353
}
350354

‎core/src/main/kotlin/org/evomaster/core/search/Individual.kt‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import org.evomaster.core.problem.externalservice.ApiExternalServiceAction
1212
import org.evomaster.core.search.action.*
1313
import org.evomaster.core.search.gene.Gene
1414
import org.evomaster.core.search.gene.interfaces.TaintableGene
15+
import org.evomaster.core.search.gene.interfaces.UserExamplesGene
1516
import org.evomaster.core.search.gene.wrapper.OptionalGene
1617
import org.evomaster.core.search.service.Randomness
1718
import org.evomaster.core.search.service.SearchGlobalState
@@ -233,6 +234,12 @@ abstract class Individual(
233234
return seeTopGenes(filter).flatMap { it.flatView() }
234235
}
235236

237+
fun getAllActiveUsedExamples() : List<Gene> {
238+
return seeFullTreeGenes()
239+
.filter { it is UserExamplesGene && it.isUsedForExamples() }
240+
.filter { it.staticCheckIfImpactPhenotype() }
241+
}
242+
236243
/**
237244
* An estimation of the "size" of this individual.
238245
* Longer/bigger individuals are usually considered worse,

0 commit comments

Comments
 (0)