diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/dictionarybase/DictionaryBaseApplication.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/dictionarybase/DictionaryBaseApplication.kt new file mode 100644 index 0000000000..7d70e94858 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/dictionarybase/DictionaryBaseApplication.kt @@ -0,0 +1,39 @@ +package com.foo.rest.examples.spring.openapi.v3.dictionarybase + +import org.springframework.boot.SpringApplication +import org.springframework.boot.autoconfigure.SpringBootApplication +import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration +import org.springframework.context.annotation.ComponentScan +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PathVariable +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController + +@RestController +@RequestMapping(path = ["/api/dictionarybase"]) +@SpringBootApplication(exclude = [SecurityAutoConfiguration::class]) +open class DictionaryBaseApplication { + companion object { + @JvmStatic + fun main(args: Array) { + SpringApplication.run(DictionaryBaseApplication::class.java, *args) + } + } + + @GetMapping(path = ["/{aggregatortype}"]) + fun get(@PathVariable aggregatortype: String) : ResponseEntity { + + /* + based on actual entry in core/src/main/resources/llm_dictionary.jsonl + {"aggregatortype":["sum","avg","count","min","max","first","last","median","mode","stddev","variance","group_concat","array_agg","string_agg","bit_and","bit_or","bool_and","bool_or","every","some"]} + if change dictionary, we might need to update this test + */ + + if(listOf("sum","avg","count","min","max","first","last","median","mode","stddev","variance").contains(aggregatortype)){ + return ResponseEntity.ok("OK") + } + + return ResponseEntity.status(400).build() + } +} \ No newline at end of file diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/dictionaryllm/DictionaryLlmApplication.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/dictionaryllm/DictionaryLlmApplication.kt new file mode 100644 index 0000000000..f06a805587 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/dictionaryllm/DictionaryLlmApplication.kt @@ -0,0 +1,33 @@ +package com.foo.rest.examples.spring.openapi.v3.dictionaryllm + +import org.springframework.boot.SpringApplication +import org.springframework.boot.autoconfigure.SpringBootApplication +import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.RequestBody +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController + +@RestController +@RequestMapping(path = ["/api/dictionaryllm"]) +@SpringBootApplication(exclude = [SecurityAutoConfiguration::class]) +open class DictionaryLlmApplication { + companion object { + @JvmStatic + fun main(args: Array) { + SpringApplication.run(DictionaryLlmApplication::class.java, *args) + } + } + + @PostMapping + fun post(@RequestBody dto: DictionaryLlmDto) : ResponseEntity { + + + if(dto.giganotosaurus == "triceratops") { + return ResponseEntity.ok("OK") + } + + return ResponseEntity.status(400).build() + } +} \ No newline at end of file diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/dictionaryllm/DictionaryLlmDto.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/dictionaryllm/DictionaryLlmDto.kt new file mode 100644 index 0000000000..87a42ee48c --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/dictionaryllm/DictionaryLlmDto.kt @@ -0,0 +1,5 @@ +package com.foo.rest.examples.spring.openapi.v3.dictionaryllm + +class DictionaryLlmDto( + var giganotosaurus: String? = null +) \ No newline at end of file diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/dictionarybase/DictionaryBaseController.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/dictionarybase/DictionaryBaseController.kt new file mode 100644 index 0000000000..2b1dff6b7a --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/dictionarybase/DictionaryBaseController.kt @@ -0,0 +1,6 @@ +package com.foo.rest.examples.spring.openapi.v3.dictionarybase + +import com.foo.rest.examples.spring.openapi.v3.SpringController + + +class DictionaryBaseController : SpringController(DictionaryBaseApplication::class.java) \ No newline at end of file diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/dictionaryllm/DictionaryLlmController.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/dictionaryllm/DictionaryLlmController.kt new file mode 100644 index 0000000000..995b7ca889 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/dictionaryllm/DictionaryLlmController.kt @@ -0,0 +1,6 @@ +package com.foo.rest.examples.spring.openapi.v3.dictionaryllm + +import com.foo.rest.examples.spring.openapi.v3.SpringController + + +class DictionaryLlmController : SpringController(DictionaryLlmApplication::class.java) \ No newline at end of file diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/dictionarybase/DictionaryBaseEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/dictionarybase/DictionaryBaseEMTest.kt new file mode 100644 index 0000000000..e8ca6cb70a --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/dictionarybase/DictionaryBaseEMTest.kt @@ -0,0 +1,45 @@ +package org.evomaster.e2etests.spring.openapi.v3.dictionarybase + +import com.foo.rest.examples.spring.openapi.v3.dictionarybase.DictionaryBaseController +import org.evomaster.core.problem.rest.data.HttpVerb +import org.evomaster.e2etests.spring.openapi.v3.SpringTestBase +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test + + +class DictionaryBaseEMTest : SpringTestBase() { + + companion object { + @BeforeAll + @JvmStatic + fun init() { + initClass(DictionaryBaseController()) + } + } + + + @Test + fun testRunEM() { + + defaultSeed = 43 + + runTestHandlingFlakyAndCompilation( + "DictionaryBaseEM", + 100 + ) { args: MutableList -> + + //White-box would trivially cover it via taint analysis + setOption(args, "blackBox", "true") + setOption(args, "base", baseUrlOfSut) + setOption(args, "schema", "$baseUrlOfSut/v3/api-docs") + setOption(args, "useDictionaryDataPool", "true") + + val solution = initAndRun(args) + + assertTrue(solution.individuals.size >= 1) + assertHasAtLeastOne(solution, HttpVerb.GET, 400, "/api/dictionarybase/{aggregatortype}", null) + assertHasAtLeastOne(solution, HttpVerb.GET, 200, "/api/dictionarybase/{aggregatortype}", "OK") + } + } +} \ No newline at end of file diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/dictionaryllm/DictionaryLlmEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/dictionaryllm/DictionaryLlmEMTest.kt new file mode 100644 index 0000000000..068ee8972a --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/dictionaryllm/DictionaryLlmEMTest.kt @@ -0,0 +1,52 @@ +package org.evomaster.e2etests.spring.openapi.v3.dictionaryllm + +import com.foo.rest.examples.spring.openapi.v3.dictionaryllm.DictionaryLlmController +import org.evomaster.core.llm.LlmProvider +import org.evomaster.core.llm.mock.MockChatModel +import org.evomaster.core.problem.rest.data.HttpVerb +import org.evomaster.e2etests.spring.openapi.v3.SpringTestBase +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test + + +class DictionaryLlmEMTest : SpringTestBase() { + + companion object { + @BeforeAll + @JvmStatic + fun init() { + initClass(DictionaryLlmController()) + } + } + + + @Test + fun testRunEM() { + + defaultSeed = 43 + + runTestHandlingFlakyAndCompilation( + "DictionaryLlmEM", + 100 + ) { args: MutableList -> + + //White-box would trivially cover it via taint analysis + setOption(args, "blackBox", "true") + setOption(args, "base", baseUrlOfSut) + setOption(args, "schema", "$baseUrlOfSut/v3/api-docs") + setOption(args, "useDictionaryDataPool", "true") + setOption(args, "llm", "true") + setOption(args, "llmProvider", LlmProvider.MOCK.name) + + MockChatModel.reset() + MockChatModel.mockResponse("[\"triceratops\"]"){ it.contains("giganotosaurus")} + + val solution = initAndRun(args) + + assertTrue(solution.individuals.size >= 1) + assertHasAtLeastOne(solution, HttpVerb.POST, 400, "/api/dictionaryllm", null) + assertHasAtLeastOne(solution, HttpVerb.POST, 200, "/api/dictionaryllm", "OK") + } + } +} \ No newline at end of file diff --git a/core/src/main/kotlin/org/evomaster/core/llm/service/DictionaryService.kt b/core/src/main/kotlin/org/evomaster/core/llm/service/DictionaryService.kt index f527404d95..502923374d 100644 --- a/core/src/main/kotlin/org/evomaster/core/llm/service/DictionaryService.kt +++ b/core/src/main/kotlin/org/evomaster/core/llm/service/DictionaryService.kt @@ -19,7 +19,7 @@ class DictionaryService { private const val location = "/llm_dictionary.jsonl" /** - * WARNING: possibly quite expensive... should only be used for debugging/testing/statistics + * in theory expensive, but on a Mac M4 took less than 1 sec to load */ fun loadAll() : Map> { @@ -148,8 +148,6 @@ class DictionaryService { throw IllegalStateException("Dictionary service is not active") } - //TODO check how expensive this is... put on thread if too expensive - val result = searchForNames(fields.map { it.name }) result.data.entries.forEach { entry -> diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/sampler/AbstractRestSampler.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/sampler/AbstractRestSampler.kt index 8a6a43d48a..10b04cb6a4 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/sampler/AbstractRestSampler.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/sampler/AbstractRestSampler.kt @@ -7,6 +7,8 @@ import org.evomaster.client.java.instrumentation.shared.TaintInputName import org.evomaster.core.AnsiColor import org.evomaster.core.EMConfig import org.evomaster.core.config.ConfigProblemException +import org.evomaster.core.llm.FieldInfo +import org.evomaster.core.llm.service.DictionaryService import org.evomaster.core.logging.LoggingUtil import org.evomaster.core.problem.enterprise.SampleType import org.evomaster.core.problem.externalservice.ExternalService @@ -47,6 +49,7 @@ import org.evomaster.core.search.warning.WarningCategory import org.slf4j.Logger import org.slf4j.LoggerFactory import javax.annotation.PostConstruct +import kotlin.sequences.forEach @@ -67,6 +70,9 @@ abstract class AbstractRestSampler : HttpWsSampler() { @Inject protected lateinit var responseClassifier: AIResponseClassifier + @Inject + protected lateinit var dictionaryService: DictionaryService + // TODO: This will moved under ApiWsSampler once RPC and GraphQL support is completed @Inject protected lateinit var externalServiceHandler: HttpWsExternalServiceHandler @@ -145,9 +151,7 @@ abstract class AbstractRestSampler : HttpWsSampler() { initializeDerivedParamRules(problem.derivedParams) } - if(config.useObjectExampleDataPool){ - feedObjectExamplesToDataPool(actionCluster) - } + updateDataPoolBasedOnSchema(actionCluster) initSqlInfo(infoDto) @@ -174,6 +178,7 @@ abstract class AbstractRestSampler : HttpWsSampler() { + override fun sampleRandomAction(noAuthP: Double): HttpWsAction { val action = super.sampleRandomAction(noAuthP) @@ -346,9 +351,8 @@ abstract class AbstractRestSampler : HttpWsSampler() { initSeededTests() } - if(config.useObjectExampleDataPool){ - feedObjectExamplesToDataPool(actionCluster) - } + updateDataPoolBasedOnSchema(actionCluster) + log.debug("Done initializing {}", AbstractRestSampler::class.simpleName) } @@ -450,6 +454,29 @@ abstract class AbstractRestSampler : HttpWsSampler() { } } + + private fun updateDataPoolBasedOnSchema(actionCluster: MutableMap){ + + //pre-filled dictionary and LLM + if(dictionaryService.isActive()){ + updateDictionaryService(actionCluster) + } + + //fields inside object examples + if(config.useObjectExampleDataPool){ + feedObjectExamplesToDataPool(actionCluster) + } + } + + private fun updateDictionaryService(actionCluster: MutableMap) { + actionCluster.values + .flatMap { it.seeAllGenes() } + .filterIsInstance() + .filter{g -> RestGeneSpecialNames.entries.none { e -> e.name == g.name } } + .map { FieldInfo(it.name, it.description) } + .let { dictionaryService.updatePoolFromDictionary(it.toList()) } + } + private fun feedObjectExamplesToDataPool(actionCluster: Map) { actionCluster.values.asSequence()