Skip to content

Commit 177436a

Browse files
arcuri82arcuri82
authored
Merge pull request #1419 from WebFuzzing/issue-test-factory
Issue test factory
2 parents b3b04a0 + 3e8d36c commit 177436a

19 files changed

Lines changed: 1121 additions & 49 deletions

File tree

‎core-tests/integration-tests/core-driver-it/src/test/kotlin/com/foo/base/BaseIT.kt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ class BaseIT {
7575
assertTrue(actionRegistered, "Failed to register action")
7676

7777
val results = remote.getTestResults()
78-
assertNotNull(results, "Failed to get results")
78+
org.junit.jupiter.api.assertNotNull(results, "Failed to get results")
7979
assertEquals(0, results!!.targets.size)
8080
}
8181

@@ -85,12 +85,12 @@ class BaseIT {
8585
//make sure it is started
8686
assertTrue(remote.startSUT(), "Failed to start SUT")
8787
var info = remote.getSutInfo()
88-
assertNotNull(info, "Failed to get SUT info")
88+
org.junit.jupiter.api.assertNotNull(info, "Failed to get SUT info")
8989

9090
//stop it
9191
assertTrue(remote.stopSUT(), "Failed to stop SUT")
9292
info = remote.getSutInfo()
93-
assertNull(info, "Failed to get SUT info after stop")
93+
org.junit.jupiter.api.assertNull(info, "Failed to get SUT info after stop")
9494

9595
val n = 3
9696
for(i in 0 until n){

‎core-tests/integration-tests/core-it/src/test/kotlin/org/evomaster/core/problem/rest/SamplerVerifierTest.kt‎

Lines changed: 79 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,21 @@ import org.evomaster.client.java.controller.api.dto.problem.param.DerivedParamCh
1313
import org.evomaster.client.java.controller.api.dto.problem.rpc.ScheduleTaskInvocationsDto
1414
import org.evomaster.client.java.controller.api.dto.problem.rpc.ScheduleTaskInvocationsResult
1515
import org.evomaster.core.BaseModule
16+
import org.evomaster.core.problem.rest.schema.RestSchema
17+
import org.evomaster.core.problem.rest.service.module.BlackBoxRestModule
1618
import org.evomaster.core.problem.rest.service.module.ResourceRestModule
1719
import org.evomaster.core.problem.rest.service.sampler.ResourceSampler
20+
import org.evomaster.core.remote.SutProblemException
1821
import org.evomaster.core.remote.service.RemoteController
1922
import org.evomaster.core.search.Individual
2023
import org.evomaster.core.search.gene.Gene
2124
import org.junit.jupiter.api.*
2225
import org.junit.jupiter.api.Assertions.assertTrue
2326
import java.io.File
27+
import java.lang.reflect.InvocationTargetException
2428
import java.time.Duration
29+
import kotlin.io.path.Path
30+
import kotlin.sequences.filter
2531

2632

2733
class SamplerVerifierTest {
@@ -39,7 +45,7 @@ class SamplerVerifierTest {
3945

4046
val controllerInfo = ControllerInfoDto()
4147

42-
val injector = getInjector(sutInfo, controllerInfo)
48+
val injector = getInjector(sutInfo, controllerInfo, false)
4349

4450
val sampler = injector.getInstance(ResourceSampler::class.java)
4551

@@ -50,27 +56,56 @@ class SamplerVerifierTest {
5056
// @Execution(ExecutionMode.CONCURRENT) //issues with shared caches
5157

5258
@TestFactory
53-
fun testSamplingFromAllSchemasUnderCoreResources(): Collection<DynamicTest>{
54-
return sampleFromSchemasAndCheckInvariants("../../../core/src/test/resources/swagger", "swagger")
59+
fun testSamplingFromAllSchemasUnderCoreResourcesWhiteBox(): Collection<DynamicTest>{
60+
val tests = sampleFromSchemasAndCheckInvariants(
61+
"../../../core/src/test/resources/swagger",
62+
"swagger",
63+
false
64+
)
65+
assertTrue(tests.isNotEmpty())
66+
return tests
67+
//FIXME once handling performance issues
68+
.filter { !it.displayName.contains("adyen") }
69+
}
70+
71+
@Disabled("Performance issue")
72+
@TestFactory
73+
fun testSamplingFromAllSchemasUnderCoreResourcesBlackBox(): Collection<DynamicTest>{
74+
val tests = sampleFromSchemasAndCheckInvariants(
75+
"../../../core/src/test/resources/swagger",
76+
"swagger",
77+
true
78+
)
79+
assertTrue(tests.isNotEmpty())
80+
return tests
5581
}
5682

5783

5884
//FIXME need to put back, and investigate performance bug
5985
@Disabled("Major issues with timeouts. Even before, took more than 1 hour. Need refactoring. Maven was not showing the failures (likely bug in Surefire)")
6086
@TestFactory
6187
fun testSamplingFromAPIsGuru(): Collection<DynamicTest>{
62-
return sampleFromSchemasAndCheckInvariants("./src/test/resources/APIs_guru", "APIs_guru")
88+
val tests = sampleFromSchemasAndCheckInvariants(
89+
"./src/test/resources/APIs_guru",
90+
"APIs_guru",
91+
false)
92+
assertTrue(tests.isNotEmpty())
93+
return tests
6394
}
6495

6596

66-
private fun sampleFromSchemasAndCheckInvariants(relativePath: String, resourceFolder: String): Collection<DynamicTest> {
97+
private fun sampleFromSchemasAndCheckInvariants(
98+
relativePath: String,
99+
resourceFolder: String,
100+
blackBox: Boolean
101+
): Collection<DynamicTest> {
67102

68103
return scanForSchemas(relativePath, resourceFolder)
69104
.sorted().map {
70105
DynamicTest.dynamicTest(it) {
71106
System.gc()
72107
assertTimeoutPreemptively(Duration.ofSeconds(30), it) {
73-
runInvariantCheck(it, 100)
108+
runInvariantCheck(it, 100, blackBox)
74109
}
75110
}
76111
}.toList()
@@ -84,18 +119,18 @@ class SamplerVerifierTest {
84119

85120
return target.walk()
86121
.filter { it.isFile }
87-
.filter { !it.name.endsWith("features_service_null.json") } //issue with parser
88-
.filter { !it.name.endsWith("trace_v2.json") } // no actions are parsed
89122
.filter { !skipSchema(it.path) }
90123
.map {
91-
val s = it.path.replace("\\", "/")
92-
.replace(relativePath, resourceFolder)
124+
// val s = it.path.replace("\\", "/")
125+
// .replace(relativePath, resourceFolder)
126+
val s = Path(it.absolutePath).toAbsolutePath().normalize().toString()
93127
s
94128
}.toList()
95129
}
96130

97131
private fun skipSchema(path: String) : Boolean {
98-
return skipDueToMissingPath(path)
132+
return skipDueToOldChecks(path) ||
133+
skipDueToMissingPath(path)
99134
|| skipDueToHashTag(path)
100135
|| skipDueToQuestionMarkInPath(path)
101136
|| skipDueToMissingReference(path)
@@ -108,6 +143,11 @@ class SamplerVerifierTest {
108143
|| skipDueToInvalidGenes(path)
109144
}
110145

146+
private fun skipDueToOldChecks(path: String) : Boolean {
147+
return path.endsWith("features_service_null.json") //issue with parser
148+
|| path.endsWith("trace_v2.json") // no actions are parsed
149+
}
150+
111151
//TODO should look into theses
112152
private fun skipDueToOverflow(path: String) : Boolean{
113153
return (path.contains("spectrocoin") && path.contains("1.0.0"))
@@ -311,16 +351,35 @@ class SamplerVerifierTest {
311351
}
312352

313353

314-
private fun runInvariantCheck(resourcePath: String, iterations: Int){
354+
private fun runInvariantCheck(resourcePath: String, iterations: Int, blackBox: Boolean){
315355

316356
val sutInfo = SutInfoDto()
357+
sutInfo.baseUrlOfSUT = "http://localhost:8080"
317358
sutInfo.restProblem = RestProblemDto()
318-
sutInfo.restProblem.openApiSchema = this::class.java.classLoader.getResource(resourcePath).readText()
359+
//sutInfo.restProblem.openApiSchema = this::class.java.classLoader.getResource(resourcePath).readText()
360+
//some schemas have relative paths, so cannot load in memory directly
361+
sutInfo.restProblem.openApiUrl = resourcePath
319362
sutInfo.defaultOutputFormat = SutInfoDto.OutputFormat.JAVA_JUNIT_4
320363

321364
val controllerInfo = ControllerInfoDto()
322365

323-
val injector = getInjector(sutInfo, controllerInfo, listOf("--seed","42"))
366+
val injector = try{
367+
getInjector(sutInfo, controllerInfo, blackBox, listOf("--seed","42"))
368+
} catch (e: Throwable){
369+
370+
val s = if(e.cause is InvocationTargetException) {
371+
e.cause!!.cause
372+
} else {
373+
e.cause!!
374+
}
375+
376+
if(s is SutProblemException && s.tag == RestSchema.TAG_NO_ACTIONS){
377+
//expected to fail
378+
//we have few schemas that only contains components
379+
return
380+
}
381+
throw e
382+
}
324383

325384
val sampler = injector.getInstance(ResourceSampler::class.java)
326385

@@ -358,10 +417,15 @@ class SamplerVerifierTest {
358417
private fun getInjector(
359418
sutInfoDto: SutInfoDto?,
360419
controllerInfoDto: ControllerInfoDto?,
420+
blackBox: Boolean,
361421
args: List<String> = listOf()): Injector {
362422

363423
val base = BaseModule(args.toTypedArray())
364-
val problemModule = ResourceRestModule(false)
424+
val problemModule = if(blackBox){
425+
BlackBoxRestModule(false)
426+
} else {
427+
ResourceRestModule(false)
428+
}
365429
val faker = FakeModule(sutInfoDto, controllerInfoDto)
366430

367431
return LifecycleInjector.builder()

‎core/src/main/kotlin/org/evomaster/core/problem/rest/builder/RestActionBuilderV3.kt‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1173,7 +1173,8 @@ object RestActionBuilderV3 {
11731173
else -> null
11741174
}
11751175
}.flatten()
1176-
return assembleObjectGene(name, options, schema, allFields.plus(fields), additionalFieldTemplate, referenceTypeName, examples, messages)
1176+
val fields = allFields.plus(fields).distinctBy { it.name }
1177+
return assembleObjectGene(name, options, schema, fields, additionalFieldTemplate, referenceTypeName, examples, messages)
11771178
}
11781179

11791180
if (!oneOf.isNullOrEmpty()){
@@ -1195,7 +1196,19 @@ object RestActionBuilderV3 {
11951196
/*
11961197
currently, we handle anyOf as oneOf plus all combined one
11971198
*/
1198-
return ChoiceGene(name, if (anyOf.size > 1) anyOf.plus(assembleObjectGene(name, options, schema, allFields.plus(fields), additionalFieldTemplate, referenceTypeName, examples, messages)) else anyOf)
1199+
1200+
val choices = if (anyOf.size > 1) {
1201+
anyOf.plus(
1202+
assembleObjectGene(
1203+
name, options, schema, allFields.plus(fields).distinctBy { it.name },
1204+
additionalFieldTemplate, referenceTypeName, examples, messages
1205+
)
1206+
)
1207+
} else {
1208+
anyOf
1209+
}
1210+
1211+
return ChoiceGene(name, choices)
11991212
// /*
12001213
// handle all combinations of anyOf
12011214
// comment it out for the moment

‎core/src/main/kotlin/org/evomaster/core/problem/rest/data/RestCallAction.kt‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,9 @@ class RestCallAction(
138138
override fun copyContent(): Action {
139139

140140
if(weakReference != null) {
141-
throw IllegalStateException("'weakReference' must handled before trying to make a copy")
141+
throw IllegalStateException("'weakReference' must handled before trying to make a copy." +
142+
" If needed, should rather use copyKeepingSameWeakRef(), but that can only be done in" +
143+
" very special cases.")
142144
}
143145

144146
val p = parameters.asSequence().map(Param::copy).toMutableList()

‎core/src/main/kotlin/org/evomaster/core/problem/rest/resource/RestResourceNode.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -524,7 +524,7 @@ open class RestResourceNode(
524524

525525
//append extra patch
526526
if (ats.last() == HttpVerb.PATCH && results.size +1 <= maxTestSize && randomness.nextBoolean(PROB_EXTRA_PATCH)){
527-
val second = results.last().copy() as RestCallAction
527+
val second = results.last().copyKeepingSameWeakRef()
528528
if (lastPost != null)
529529
CreateResourceUtils.linkDynamicCreateResource(lastPost, second)
530530
results.add(second)

‎core/src/main/kotlin/org/evomaster/core/problem/rest/schema/RestSchema.kt‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ class RestSchema(
1818
companion object{
1919
private val log = org.slf4j.LoggerFactory.getLogger(RestSchema::class.java)
2020

21+
const val TAG_NO_ACTIONS = "No actions"
22+
2123
fun fromLocation(location: String) = RestSchema(OpenApiAccess.getOpenAPIFromLocation(location))
2224

2325
fun fromResource(path: String) = RestSchema(OpenApiAccess.getOpenAPIFromResource(path))
@@ -123,12 +125,12 @@ class RestSchema(
123125

124126
fun validate(){
125127
if (main.schemaParsed.paths == null) {
126-
throw SutProblemException("There is no endpoint definition in the retrieved OpenAPI file")
128+
throw SutProblemException("There is no endpoint definition in the retrieved OpenAPI file", TAG_NO_ACTIONS)
127129
}
128130
// give the error message if there is nothing to test
129-
if (main.schemaParsed.paths.size == 0){
131+
if (main.schemaParsed.paths.isEmpty()){
130132
throw SutProblemException("The OpenAPI file from ${main.sourceLocation} " +
131-
"is either invalid or it does not define endpoints")
133+
"is either invalid or it does not define endpoints", TAG_NO_ACTIONS)
132134
}
133135
}
134136
}

‎core/src/main/kotlin/org/evomaster/core/remote/SutProblemException.kt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,6 @@ package org.evomaster.core.remote
22

33

44
open class SutProblemException(
5-
message : String = "Problem with the system under test."
5+
message : String = "Problem with the system under test.",
6+
val tag: String? = null
67
) : RuntimeException(message)

‎core/src/main/kotlin/org/evomaster/core/search/gene/Gene.kt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -880,6 +880,10 @@ abstract class Gene(
880880
* If the type of [other] is different, this method might throw an [IllegalArgumentException]
881881
*
882882
* TODO refactor, in which type-check is done here
883+
*
884+
* FIXME do we really need to throw an exception???
885+
* issue is that even if [other] is of right type, it's internal genes might not match,
886+
* eg, look at ChoiceGene
883887
*/
884888
abstract fun containsSameValueAs(other: Gene): Boolean
885889

‎core/src/main/kotlin/org/evomaster/core/search/gene/ObjectGene.kt‎

Lines changed: 29 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import org.evomaster.core.search.service.Randomness
2222
import org.evomaster.core.search.service.mutator.MutationWeightControl
2323
import org.evomaster.core.search.service.mutator.genemutation.AdditionalGeneMutationInfo
2424
import org.evomaster.core.search.service.mutator.genemutation.SubsetGeneMutationSelectionStrategy
25+
import org.evomaster.core.utils.CollectionUtils
2526
import org.slf4j.Logger
2627
import org.slf4j.LoggerFactory
2728
import java.net.URLEncoder
@@ -68,6 +69,11 @@ class ObjectGene(
6869
if (additionalFields != null)
6970
throw IllegalArgumentException("cannot specify additional field when the ObjectGene is fixed")
7071
}
72+
val duplicateNames = CollectionUtils.duplicates(fixedFields.map { it.name })
73+
if(duplicateNames.isNotEmpty()){
74+
throw IllegalArgumentException("In an Object, cannot have more than one field with same name." +
75+
" Duplicates: ${duplicateNames.keys.joinToString(",")}")
76+
}
7177
}
7278

7379
constructor(name: String, fields: List<out Gene>, refType: String? = null) : this(name, fields, refType, true, null, null)
@@ -100,7 +106,14 @@ class ObjectGene(
100106
override fun canBeChildless() = true
101107

102108
override fun copyContent(): Gene {
103-
return ObjectGene(name, fixedFields.map(Gene::copy), refType, isFixed, template, additionalFields?.map {it.copy() as PairGene<StringGene, Gene> }?.toMutableList())
109+
return ObjectGene(
110+
name,
111+
fixedFields.map(Gene::copy),
112+
refType,
113+
isFixed,
114+
template,
115+
additionalFields?.map {it.copy() as PairGene<StringGene, Gene> }?.toMutableList()
116+
)
104117
}
105118

106119
override fun checkForLocallyValidIgnoringChildren(): Boolean {
@@ -238,7 +251,10 @@ class ObjectGene(
238251

239252
return this.fixedFields.size == other.fixedFields.size
240253
&& (isFixed || additionalFields!!.size == other.additionalFields!!.size)
241-
&& this.fixedFields.zip(other.fixedFields) { thisField, otherField -> thisField.containsSameValueAs(otherField) }.all { it }
254+
&& (this.fixedFields.sortedBy{it.name}
255+
.zip(other.fixedFields.sortedBy { it.name })
256+
{ thisField, otherField -> thisField.containsSameValueAs(otherField) }
257+
.all { it })
242258
&& (isFixed || this.additionalFields!!.zip(other.additionalFields!!) { thisField, otherField -> thisField.containsSameValueAs(otherField) }.all { it }
243259
)
244260
}
@@ -259,8 +275,17 @@ class ObjectGene(
259275

260276
var ok = true
261277

262-
for (i in fixedFields.indices) {
263-
ok = ok && this.fixedFields[i].unsafeCopyValueFrom(other.fixedFields[i])
278+
// for (i in fixedFields.indices) {
279+
// ok = ok && this.fixedFields[i].unsafeCopyValueFrom(other.fixedFields[i])
280+
// }
281+
for(field in fixedFields){
282+
val name = field.name
283+
val toCopy = other.fixedFields.find { it.name == name }
284+
ok = if(toCopy == null){
285+
false
286+
} else {
287+
field.unsafeCopyValueFrom(toCopy)
288+
}
264289
}
265290

266291
if(!isFixed){

‎core/src/main/kotlin/org/evomaster/core/search/gene/regex/AnyCharacterRxGene.kt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ class AnyCharacterRxGene : RxAtom, SimpleGene("."){
3434
override fun copyContent(): Gene {
3535
val copy = AnyCharacterRxGene()
3636
copy.value = this.value
37+
copy.name = this.name //in case name is changed from its default
3738
return copy
3839
}
3940

0 commit comments

Comments
 (0)