Skip to content

Commit a7e1397

Browse files
arcuri82arcuri82
authored
Merge pull request #1632 from WebFuzzing/integrate-dictionary
Integrate dictionary
2 parents 322a1d5 + 8be7f56 commit a7e1397

9 files changed

Lines changed: 220 additions & 9 deletions

File tree

‎core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/dictionarybase/DictionaryBaseApplication.kt‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
package com.foo.rest.examples.spring.openapi.v3.dictionarybase
2+
3+
import org.springframework.boot.SpringApplication
4+
import org.springframework.boot.autoconfigure.SpringBootApplication
5+
import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration
6+
import org.springframework.context.annotation.ComponentScan
7+
import org.springframework.http.ResponseEntity
8+
import org.springframework.web.bind.annotation.GetMapping
9+
import org.springframework.web.bind.annotation.PathVariable
10+
import org.springframework.web.bind.annotation.RequestMapping
11+
import org.springframework.web.bind.annotation.RestController
12+
13+
@RestController
14+
@RequestMapping(path = ["/api/dictionarybase"])
15+
@SpringBootApplication(exclude = [SecurityAutoConfiguration::class])
16+
open class DictionaryBaseApplication {
17+
companion object {
18+
@JvmStatic
19+
fun main(args: Array<String>) {
20+
SpringApplication.run(DictionaryBaseApplication::class.java, *args)
21+
}
22+
}
23+
24+
@GetMapping(path = ["/{aggregatortype}"])
25+
fun get(@PathVariable aggregatortype: String) : ResponseEntity<String> {
26+
27+
/*
28+
based on actual entry in core/src/main/resources/llm_dictionary.jsonl
29+
{"aggregatortype":["sum","avg","count","min","max","first","last","median","mode","stddev","variance","group_concat","array_agg","string_agg","bit_and","bit_or","bool_and","bool_or","every","some"]}
30+
if change dictionary, we might need to update this test
31+
*/
32+
33+
if(listOf("sum","avg","count","min","max","first","last","median","mode","stddev","variance").contains(aggregatortype)){
34+
return ResponseEntity.ok("OK")
35+
}
36+
37+
return ResponseEntity.status(400).build()
38+
}
39+
}

‎core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/dictionaryllm/DictionaryLlmApplication.kt‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
package com.foo.rest.examples.spring.openapi.v3.dictionaryllm
2+
3+
import org.springframework.boot.SpringApplication
4+
import org.springframework.boot.autoconfigure.SpringBootApplication
5+
import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration
6+
import org.springframework.http.ResponseEntity
7+
import org.springframework.web.bind.annotation.PostMapping
8+
import org.springframework.web.bind.annotation.RequestBody
9+
import org.springframework.web.bind.annotation.RequestMapping
10+
import org.springframework.web.bind.annotation.RestController
11+
12+
@RestController
13+
@RequestMapping(path = ["/api/dictionaryllm"])
14+
@SpringBootApplication(exclude = [SecurityAutoConfiguration::class])
15+
open class DictionaryLlmApplication {
16+
companion object {
17+
@JvmStatic
18+
fun main(args: Array<String>) {
19+
SpringApplication.run(DictionaryLlmApplication::class.java, *args)
20+
}
21+
}
22+
23+
@PostMapping
24+
fun post(@RequestBody dto: DictionaryLlmDto) : ResponseEntity<String> {
25+
26+
27+
if(dto.giganotosaurus == "triceratops") {
28+
return ResponseEntity.ok("OK")
29+
}
30+
31+
return ResponseEntity.status(400).build()
32+
}
33+
}

‎core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/dictionaryllm/DictionaryLlmDto.kt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
package com.foo.rest.examples.spring.openapi.v3.dictionaryllm
2+
3+
class DictionaryLlmDto(
4+
var giganotosaurus: String? = null
5+
)

‎core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/dictionarybase/DictionaryBaseController.kt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
package com.foo.rest.examples.spring.openapi.v3.dictionarybase
2+
3+
import com.foo.rest.examples.spring.openapi.v3.SpringController
4+
5+
6+
class DictionaryBaseController : SpringController(DictionaryBaseApplication::class.java)

‎core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/dictionaryllm/DictionaryLlmController.kt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
package com.foo.rest.examples.spring.openapi.v3.dictionaryllm
2+
3+
import com.foo.rest.examples.spring.openapi.v3.SpringController
4+
5+
6+
class DictionaryLlmController : SpringController(DictionaryLlmApplication::class.java)

‎core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/dictionarybase/DictionaryBaseEMTest.kt‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
package org.evomaster.e2etests.spring.openapi.v3.dictionarybase
2+
3+
import com.foo.rest.examples.spring.openapi.v3.dictionarybase.DictionaryBaseController
4+
import org.evomaster.core.problem.rest.data.HttpVerb
5+
import org.evomaster.e2etests.spring.openapi.v3.SpringTestBase
6+
import org.junit.jupiter.api.Assertions.assertTrue
7+
import org.junit.jupiter.api.BeforeAll
8+
import org.junit.jupiter.api.Test
9+
10+
11+
class DictionaryBaseEMTest : SpringTestBase() {
12+
13+
companion object {
14+
@BeforeAll
15+
@JvmStatic
16+
fun init() {
17+
initClass(DictionaryBaseController())
18+
}
19+
}
20+
21+
22+
@Test
23+
fun testRunEM() {
24+
25+
defaultSeed = 43
26+
27+
runTestHandlingFlakyAndCompilation(
28+
"DictionaryBaseEM",
29+
100
30+
) { args: MutableList<String> ->
31+
32+
//White-box would trivially cover it via taint analysis
33+
setOption(args, "blackBox", "true")
34+
setOption(args, "base", baseUrlOfSut)
35+
setOption(args, "schema", "$baseUrlOfSut/v3/api-docs")
36+
setOption(args, "useDictionaryDataPool", "true")
37+
38+
val solution = initAndRun(args)
39+
40+
assertTrue(solution.individuals.size >= 1)
41+
assertHasAtLeastOne(solution, HttpVerb.GET, 400, "/api/dictionarybase/{aggregatortype}", null)
42+
assertHasAtLeastOne(solution, HttpVerb.GET, 200, "/api/dictionarybase/{aggregatortype}", "OK")
43+
}
44+
}
45+
}

‎core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/dictionaryllm/DictionaryLlmEMTest.kt‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
package org.evomaster.e2etests.spring.openapi.v3.dictionaryllm
2+
3+
import com.foo.rest.examples.spring.openapi.v3.dictionaryllm.DictionaryLlmController
4+
import org.evomaster.core.llm.LlmProvider
5+
import org.evomaster.core.llm.mock.MockChatModel
6+
import org.evomaster.core.problem.rest.data.HttpVerb
7+
import org.evomaster.e2etests.spring.openapi.v3.SpringTestBase
8+
import org.junit.jupiter.api.Assertions.assertTrue
9+
import org.junit.jupiter.api.BeforeAll
10+
import org.junit.jupiter.api.Test
11+
12+
13+
class DictionaryLlmEMTest : SpringTestBase() {
14+
15+
companion object {
16+
@BeforeAll
17+
@JvmStatic
18+
fun init() {
19+
initClass(DictionaryLlmController())
20+
}
21+
}
22+
23+
24+
@Test
25+
fun testRunEM() {
26+
27+
defaultSeed = 43
28+
29+
runTestHandlingFlakyAndCompilation(
30+
"DictionaryLlmEM",
31+
100
32+
) { args: MutableList<String> ->
33+
34+
//White-box would trivially cover it via taint analysis
35+
setOption(args, "blackBox", "true")
36+
setOption(args, "base", baseUrlOfSut)
37+
setOption(args, "schema", "$baseUrlOfSut/v3/api-docs")
38+
setOption(args, "useDictionaryDataPool", "true")
39+
setOption(args, "llm", "true")
40+
setOption(args, "llmProvider", LlmProvider.MOCK.name)
41+
42+
MockChatModel.reset()
43+
MockChatModel.mockResponse("[\"triceratops\"]"){ it.contains("giganotosaurus")}
44+
45+
val solution = initAndRun(args)
46+
47+
assertTrue(solution.individuals.size >= 1)
48+
assertHasAtLeastOne(solution, HttpVerb.POST, 400, "/api/dictionaryllm", null)
49+
assertHasAtLeastOne(solution, HttpVerb.POST, 200, "/api/dictionaryllm", "OK")
50+
}
51+
}
52+
}

‎core/src/main/kotlin/org/evomaster/core/llm/service/DictionaryService.kt‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ class DictionaryService {
1919
private const val location = "/llm_dictionary.jsonl"
2020

2121
/**
22-
* WARNING: possibly quite expensive... should only be used for debugging/testing/statistics
22+
* in theory expensive, but on a Mac M4 took less than 1 sec to load
2323
*/
2424
fun loadAll() : Map<String, Set<String>> {
2525

@@ -148,8 +148,6 @@ class DictionaryService {
148148
throw IllegalStateException("Dictionary service is not active")
149149
}
150150

151-
//TODO check how expensive this is... put on thread if too expensive
152-
153151
val result = searchForNames(fields.map { it.name })
154152

155153
result.data.entries.forEach { entry ->

‎core/src/main/kotlin/org/evomaster/core/problem/rest/service/sampler/AbstractRestSampler.kt‎

Lines changed: 33 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ import org.evomaster.client.java.instrumentation.shared.TaintInputName
77
import org.evomaster.core.AnsiColor
88
import org.evomaster.core.EMConfig
99
import org.evomaster.core.config.ConfigProblemException
10+
import org.evomaster.core.llm.FieldInfo
11+
import org.evomaster.core.llm.service.DictionaryService
1012
import org.evomaster.core.logging.LoggingUtil
1113
import org.evomaster.core.problem.enterprise.SampleType
1214
import org.evomaster.core.problem.externalservice.ExternalService
@@ -47,6 +49,7 @@ import org.evomaster.core.search.warning.WarningCategory
4749
import org.slf4j.Logger
4850
import org.slf4j.LoggerFactory
4951
import javax.annotation.PostConstruct
52+
import kotlin.sequences.forEach
5053

5154

5255

@@ -67,6 +70,9 @@ abstract class AbstractRestSampler : HttpWsSampler<RestIndividual>() {
6770
@Inject
6871
protected lateinit var responseClassifier: AIResponseClassifier
6972

73+
@Inject
74+
protected lateinit var dictionaryService: DictionaryService
75+
7076
// TODO: This will moved under ApiWsSampler once RPC and GraphQL support is completed
7177
@Inject
7278
protected lateinit var externalServiceHandler: HttpWsExternalServiceHandler
@@ -145,9 +151,7 @@ abstract class AbstractRestSampler : HttpWsSampler<RestIndividual>() {
145151
initializeDerivedParamRules(problem.derivedParams)
146152
}
147153

148-
if(config.useObjectExampleDataPool){
149-
feedObjectExamplesToDataPool(actionCluster)
150-
}
154+
updateDataPoolBasedOnSchema(actionCluster)
151155

152156
initSqlInfo(infoDto)
153157

@@ -174,6 +178,7 @@ abstract class AbstractRestSampler : HttpWsSampler<RestIndividual>() {
174178

175179

176180

181+
177182
override fun sampleRandomAction(noAuthP: Double): HttpWsAction {
178183

179184
val action = super.sampleRandomAction(noAuthP)
@@ -346,9 +351,8 @@ abstract class AbstractRestSampler : HttpWsSampler<RestIndividual>() {
346351
initSeededTests()
347352
}
348353

349-
if(config.useObjectExampleDataPool){
350-
feedObjectExamplesToDataPool(actionCluster)
351-
}
354+
updateDataPoolBasedOnSchema(actionCluster)
355+
352356

353357
log.debug("Done initializing {}", AbstractRestSampler::class.simpleName)
354358
}
@@ -450,6 +454,29 @@ abstract class AbstractRestSampler : HttpWsSampler<RestIndividual>() {
450454
}
451455
}
452456

457+
458+
private fun updateDataPoolBasedOnSchema(actionCluster: MutableMap<String, Action>){
459+
460+
//pre-filled dictionary and LLM
461+
if(dictionaryService.isActive()){
462+
updateDictionaryService(actionCluster)
463+
}
464+
465+
//fields inside object examples
466+
if(config.useObjectExampleDataPool){
467+
feedObjectExamplesToDataPool(actionCluster)
468+
}
469+
}
470+
471+
private fun updateDictionaryService(actionCluster: MutableMap<String, Action>) {
472+
actionCluster.values
473+
.flatMap { it.seeAllGenes() }
474+
.filterIsInstance<StringGene>()
475+
.filter{g -> RestGeneSpecialNames.entries.none { e -> e.name == g.name } }
476+
.map { FieldInfo(it.name, it.description) }
477+
.let { dictionaryService.updatePoolFromDictionary(it.toList()) }
478+
}
479+
453480
private fun feedObjectExamplesToDataPool(actionCluster: Map<String, Action>) {
454481

455482
actionCluster.values.asSequence()

0 commit comments

Comments
 (0)