From b48a3d911fa3066252cd2336c69a83643985a364 Mon Sep 17 00:00:00 2001 From: Omur Sahin Date: Thu, 7 May 2026 23:48:27 +0300 Subject: [PATCH 1/4] non-idempotent-put starter --- .../json/HttpNonIdempotentPutApplication.kt | 71 +++++++++ ...tpNonIdempotentPutUrlEncodedApplication.kt | 82 ++++++++++ .../xml/HttpNonIdempotentPutXMLApplication.kt | 94 ++++++++++++ .../HttpNonIdempotentPutController.kt | 11 ++ ...ttpNonIdempotentPutUrlencodedController.kt | 12 ++ .../HttpNonIdempotentPutXMLController.kt | 12 ++ .../HttpNonIdempotentPutEMTest.kt | 45 ++++++ .../HttpNonIdempotentPutUrlencodedEMTest.kt | 47 ++++++ .../HttpNonIdempotentPutXMLEMTest.kt | 46 ++++++ .../core/output/formatter/OutputFormatter.kt | 143 ++++++++++++++++-- .../enterprise/ExperimentalFaultCategory.kt | 3 + .../rest/oracle/HttpSemanticsOracle.kt | 74 +++++++++ .../rest/service/HttpSemanticsService.kt | 62 ++++++++ .../service/fitness/AbstractRestFitness.kt | 23 +++ 14 files changed, 711 insertions(+), 14 deletions(-) create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/nonidempotentput/json/HttpNonIdempotentPutApplication.kt create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/nonidempotentput/urlencoded/HttpNonIdempotentPutUrlEncodedApplication.kt create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/nonidempotentput/xml/HttpNonIdempotentPutXMLApplication.kt create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/nonidempotentput/HttpNonIdempotentPutController.kt create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/nonidempotentput/HttpNonIdempotentPutUrlencodedController.kt create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/nonidempotentput/HttpNonIdempotentPutXMLController.kt create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/nonidempotentput/HttpNonIdempotentPutEMTest.kt create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/nonidempotentput/HttpNonIdempotentPutUrlencodedEMTest.kt create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/nonidempotentput/HttpNonIdempotentPutXMLEMTest.kt diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/nonidempotentput/json/HttpNonIdempotentPutApplication.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/nonidempotentput/json/HttpNonIdempotentPutApplication.kt new file mode 100644 index 0000000000..c4f3cb66a5 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/nonidempotentput/json/HttpNonIdempotentPutApplication.kt @@ -0,0 +1,71 @@ +package com.foo.rest.examples.spring.openapi.v3.httporacle.nonidempotentput.json + +import org.springframework.boot.SpringApplication +import org.springframework.boot.autoconfigure.SpringBootApplication +import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PathVariable +import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.PutMapping +import org.springframework.web.bind.annotation.RequestBody +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController + + +@SpringBootApplication(exclude = [SecurityAutoConfiguration::class]) +@RequestMapping(path = ["/api/accounts"]) +@RestController +open class HttpNonIdempotentPutApplication { + + companion object { + @JvmStatic + fun main(args: Array) { + SpringApplication.run(HttpNonIdempotentPutApplication::class.java, *args) + } + + private val data = mutableMapOf() + + fun reset(){ + data.clear() + } + } + + data class AccountData( + var balance: Int + ) + + data class DepositRequest( + val amount: Int + ) + + + @PostMapping() + open fun create(@RequestBody body: AccountData): ResponseEntity { + val id = data.size + 1 + data[id] = body.copy() + return ResponseEntity.status(201).body(data[id]) + } + + @GetMapping(path = ["/{id}"]) + open fun get(@PathVariable("id") id: Int): ResponseEntity { + val resource = data[id] + ?: return ResponseEntity.status(404).build() + return ResponseEntity.status(200).body(resource) + } + + @PutMapping(path = ["/{id}/deposit"]) + open fun deposit( + @PathVariable("id") id: Int, + @RequestBody body: DepositRequest + ): ResponseEntity { + + val resource = data[id] + ?: return ResponseEntity.status(404).build() + + // wrong: PUT must be idempotent, but each call accumulates the deposit + resource.balance += body.amount + + return ResponseEntity.status(200).body(resource) + } +} diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/nonidempotentput/urlencoded/HttpNonIdempotentPutUrlEncodedApplication.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/nonidempotentput/urlencoded/HttpNonIdempotentPutUrlEncodedApplication.kt new file mode 100644 index 0000000000..5af87db8d6 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/nonidempotentput/urlencoded/HttpNonIdempotentPutUrlEncodedApplication.kt @@ -0,0 +1,82 @@ +package com.foo.rest.examples.spring.openapi.v3.httporacle.nonidempotentput.urlencoded + +import org.springframework.boot.SpringApplication +import org.springframework.boot.autoconfigure.SpringBootApplication +import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration +import org.springframework.http.MediaType +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.ModelAttribute +import org.springframework.web.bind.annotation.PathVariable +import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.PutMapping +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController + + +@SpringBootApplication(exclude = [SecurityAutoConfiguration::class]) +@RequestMapping(path = ["/api/accounts"]) +@RestController +open class HttpNonIdempotentPutUrlEncodedApplication { + + companion object { + @JvmStatic + fun main(args: Array) { + SpringApplication.run(HttpNonIdempotentPutUrlEncodedApplication::class.java, *args) + } + + private val data = mutableMapOf() + + fun reset(){ + data.clear() + } + } + + open class AccountData( + var balance: Int = 0 + ) + + open class DepositRequest( + var amount: Int = 0 + ) + + + @PostMapping( + consumes = [MediaType.APPLICATION_FORM_URLENCODED_VALUE], + produces = [MediaType.APPLICATION_JSON_VALUE] + ) + open fun create(@ModelAttribute body: AccountData): ResponseEntity { + val id = data.size + 1 + data[id] = AccountData(balance = body.balance) + return ResponseEntity.status(201).body(data[id]) + } + + @GetMapping( + path = ["/{id}"], + produces = [MediaType.APPLICATION_JSON_VALUE] + ) + open fun get(@PathVariable("id") id: Int): ResponseEntity { + val resource = data[id] + ?: return ResponseEntity.status(404).build() + return ResponseEntity.status(200).body(resource) + } + + @PutMapping( + path = ["/{id}/deposit"], + consumes = [MediaType.APPLICATION_FORM_URLENCODED_VALUE], + produces = [MediaType.APPLICATION_JSON_VALUE] + ) + open fun deposit( + @PathVariable("id") id: Int, + @ModelAttribute body: DepositRequest + ): ResponseEntity { + + val resource = data[id] + ?: return ResponseEntity.status(404).build() + + // wrong: PUT must be idempotent, but each call accumulates the deposit + resource.balance += body.amount + + return ResponseEntity.status(200).body(resource) + } +} diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/nonidempotentput/xml/HttpNonIdempotentPutXMLApplication.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/nonidempotentput/xml/HttpNonIdempotentPutXMLApplication.kt new file mode 100644 index 0000000000..684ba250fa --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/nonidempotentput/xml/HttpNonIdempotentPutXMLApplication.kt @@ -0,0 +1,94 @@ +package com.foo.rest.examples.spring.openapi.v3.httporacle.nonidempotentput.xml + +import org.springframework.boot.SpringApplication +import org.springframework.boot.autoconfigure.SpringBootApplication +import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration +import org.springframework.http.MediaType +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PathVariable +import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.PutMapping +import org.springframework.web.bind.annotation.RequestBody +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController +import javax.xml.bind.annotation.XmlAccessType +import javax.xml.bind.annotation.XmlAccessorType +import javax.xml.bind.annotation.XmlRootElement + + +@SpringBootApplication(exclude = [SecurityAutoConfiguration::class]) +@RequestMapping(path = ["/api/accounts"]) +@RestController +open class HttpNonIdempotentPutXMLApplication { + + companion object { + @JvmStatic + fun main(args: Array) { + SpringApplication.run(HttpNonIdempotentPutXMLApplication::class.java, *args) + } + + private val data = mutableMapOf() + + fun reset(){ + data.clear() + } + } + + @XmlRootElement(name = "accountData") + @XmlAccessorType(XmlAccessType.FIELD) + open class AccountData( + var balance: Int = 0 + ) + + @XmlRootElement(name = "depositRequest") + @XmlAccessorType(XmlAccessType.FIELD) + open class DepositRequest( + var amount: Int = 0, + // EvoMaster's ObjectGene XML serializer inlines single-field bodies as + // 5 instead of 5, + // which JAXB cannot bind (amount falls back to its Kotlin default). + // This unused field forces the multi-field path so the random `amount` actually reaches the server. + var note: String = "" + ) + + + @PostMapping( + consumes = [MediaType.APPLICATION_XML_VALUE], + produces = [MediaType.APPLICATION_XML_VALUE] + ) + open fun create(@RequestBody body: AccountData): ResponseEntity { + val id = data.size + 1 + data[id] = AccountData(balance = body.balance) + return ResponseEntity.status(201).body(data[id]) + } + + @GetMapping( + path = ["/{id}"], + produces = [MediaType.APPLICATION_XML_VALUE] + ) + open fun get(@PathVariable("id") id: Int): ResponseEntity { + val resource = data[id] + ?: return ResponseEntity.status(404).build() + return ResponseEntity.status(200).body(resource) + } + + @PutMapping( + path = ["/{id}/deposit"], + consumes = [MediaType.APPLICATION_XML_VALUE], + produces = [MediaType.APPLICATION_XML_VALUE] + ) + open fun deposit( + @PathVariable("id") id: Int, + @RequestBody body: DepositRequest + ): ResponseEntity { + + val resource = data[id] + ?: return ResponseEntity.status(404).build() + + // wrong: PUT must be idempotent, but each call accumulates the deposit + resource.balance += body.amount + + return ResponseEntity.status(200).body(resource) + } +} diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/nonidempotentput/HttpNonIdempotentPutController.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/nonidempotentput/HttpNonIdempotentPutController.kt new file mode 100644 index 0000000000..1eccb676c5 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/nonidempotentput/HttpNonIdempotentPutController.kt @@ -0,0 +1,11 @@ +package com.foo.rest.examples.spring.openapi.v3.httporacle.nonidempotentput.json + +import com.foo.rest.examples.spring.openapi.v3.SpringController + + +class HttpNonIdempotentPutController: SpringController(HttpNonIdempotentPutApplication::class.java){ + + override fun resetStateOfSUT() { + HttpNonIdempotentPutApplication.reset() + } +} \ No newline at end of file diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/nonidempotentput/HttpNonIdempotentPutUrlencodedController.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/nonidempotentput/HttpNonIdempotentPutUrlencodedController.kt new file mode 100644 index 0000000000..62f2ae3cc1 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/nonidempotentput/HttpNonIdempotentPutUrlencodedController.kt @@ -0,0 +1,12 @@ +package com.foo.rest.examples.spring.openapi.v3.httporacle.nonidempotentput.urlencoded + +import com.foo.rest.examples.spring.openapi.v3.SpringController +import com.foo.rest.examples.spring.openapi.v3.httporacle.nonidempotentput.xml.HttpNonIdempotentPutXMLApplication + + +class HttpNonIdempotentPutUrlencodedController: SpringController(HttpNonIdempotentPutUrlEncodedApplication::class.java){ + + override fun resetStateOfSUT() { + HttpNonIdempotentPutUrlEncodedApplication.reset() + } +} \ No newline at end of file diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/nonidempotentput/HttpNonIdempotentPutXMLController.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/nonidempotentput/HttpNonIdempotentPutXMLController.kt new file mode 100644 index 0000000000..8b7abf074f --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/nonidempotentput/HttpNonIdempotentPutXMLController.kt @@ -0,0 +1,12 @@ +package com.foo.rest.examples.spring.openapi.v3.httporacle.nonidempotentput.xml + +import com.foo.rest.examples.spring.openapi.v3.SpringController +import com.foo.rest.examples.spring.openapi.v3.httporacle.nonidempotentput.xml.HttpNonIdempotentPutXMLApplication + + +class HttpNonIdempotentPutXMLController: SpringController(HttpNonIdempotentPutXMLApplication::class.java){ + + override fun resetStateOfSUT() { + HttpNonIdempotentPutXMLApplication.reset() + } +} \ No newline at end of file diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/nonidempotentput/HttpNonIdempotentPutEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/nonidempotentput/HttpNonIdempotentPutEMTest.kt new file mode 100644 index 0000000000..39fb101130 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/nonidempotentput/HttpNonIdempotentPutEMTest.kt @@ -0,0 +1,45 @@ +package org.evomaster.e2etests.spring.openapi.v3.httporacle.nonidempotentput + +import com.foo.rest.examples.spring.openapi.v3.httporacle.nonidempotentput.json.HttpNonIdempotentPutController +import org.evomaster.core.problem.enterprise.DetectedFaultUtils +import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory +import org.evomaster.e2etests.spring.openapi.v3.SpringTestBase +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test + +class HttpNonIdempotentPutEMTest : SpringTestBase(){ + + companion object { + @BeforeAll + @JvmStatic + fun init() { + initClass(HttpNonIdempotentPutController()) + } + } + + + @Test + fun testRunEM() { + + runTestHandlingFlakyAndCompilation( + "HttpNonIdempotentPutEM", + 500 + ) { args: MutableList -> + + setOption(args, "security", "false") + setOption(args, "schemaOracles", "false") + setOption(args, "httpOracles", "true") + setOption(args, "useExperimentalOracles", "true") + + val solution = initAndRun(args) + + assertTrue(solution.individuals.size >= 1) + + val faults = DetectedFaultUtils.getDetectedFaultCategories(solution) + assertEquals(1, faults.size) + assertEquals(ExperimentalFaultCategory.HTTP_NON_IDEMPOTENT_PUT, faults.first()) + } + } +} \ No newline at end of file diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/nonidempotentput/HttpNonIdempotentPutUrlencodedEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/nonidempotentput/HttpNonIdempotentPutUrlencodedEMTest.kt new file mode 100644 index 0000000000..fecfa38240 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/nonidempotentput/HttpNonIdempotentPutUrlencodedEMTest.kt @@ -0,0 +1,47 @@ +package org.evomaster.e2etests.spring.openapi.v3.httporacle.nonidempotentput + +import com.foo.rest.examples.spring.openapi.v3.httporacle.nonidempotentput.json.HttpNonIdempotentPutController +import com.foo.rest.examples.spring.openapi.v3.httporacle.nonidempotentput.urlencoded.HttpNonIdempotentPutUrlencodedController +import com.foo.rest.examples.spring.openapi.v3.httporacle.nonidempotentput.xml.HttpNonIdempotentPutXMLController +import org.evomaster.core.problem.enterprise.DetectedFaultUtils +import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory +import org.evomaster.e2etests.spring.openapi.v3.SpringTestBase +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test + +class HttpNonIdempotentPutUrlencodedEMTest : SpringTestBase(){ + + companion object { + @BeforeAll + @JvmStatic + fun init() { + initClass(HttpNonIdempotentPutUrlencodedController()) + } + } + + + @Test + fun testRunEM() { + + runTestHandlingFlakyAndCompilation( + "HttpNonIdempotentPutUrlencodedEM", + 1000 + ) { args: MutableList -> + + setOption(args, "security", "false") + setOption(args, "schemaOracles", "false") + setOption(args, "httpOracles", "true") + setOption(args, "useExperimentalOracles", "true") + + val solution = initAndRun(args) + + assertTrue(solution.individuals.size >= 1) + + val faults = DetectedFaultUtils.getDetectedFaultCategories(solution) + assertEquals(1, faults.size) + assertEquals(ExperimentalFaultCategory.HTTP_NON_IDEMPOTENT_PUT, faults.first()) + } + } +} \ No newline at end of file diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/nonidempotentput/HttpNonIdempotentPutXMLEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/nonidempotentput/HttpNonIdempotentPutXMLEMTest.kt new file mode 100644 index 0000000000..2946a1673c --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/nonidempotentput/HttpNonIdempotentPutXMLEMTest.kt @@ -0,0 +1,46 @@ +package org.evomaster.e2etests.spring.openapi.v3.httporacle.nonidempotentput + +import com.foo.rest.examples.spring.openapi.v3.httporacle.nonidempotentput.json.HttpNonIdempotentPutController +import com.foo.rest.examples.spring.openapi.v3.httporacle.nonidempotentput.xml.HttpNonIdempotentPutXMLController +import org.evomaster.core.problem.enterprise.DetectedFaultUtils +import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory +import org.evomaster.e2etests.spring.openapi.v3.SpringTestBase +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test + +class HttpNonIdempotentPutXMLEMTest : SpringTestBase(){ + + companion object { + @BeforeAll + @JvmStatic + fun init() { + initClass(HttpNonIdempotentPutXMLController()) + } + } + + + @Test + fun testRunEM() { + + runTestHandlingFlakyAndCompilation( + "HttpNonIdempotentPutXMLEM", + 1000 + ) { args: MutableList -> + + setOption(args, "security", "false") + setOption(args, "schemaOracles", "false") + setOption(args, "httpOracles", "true") + setOption(args, "useExperimentalOracles", "true") + + val solution = initAndRun(args) + + assertTrue(solution.individuals.size >= 1) + + val faults = DetectedFaultUtils.getDetectedFaultCategories(solution) + assertEquals(1, faults.size) + assertEquals(ExperimentalFaultCategory.HTTP_NON_IDEMPOTENT_PUT, faults.first()) + } + } +} \ No newline at end of file diff --git a/core/src/main/kotlin/org/evomaster/core/output/formatter/OutputFormatter.kt b/core/src/main/kotlin/org/evomaster/core/output/formatter/OutputFormatter.kt index f2b6504eaa..303da0cc76 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/formatter/OutputFormatter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/formatter/OutputFormatter.kt @@ -5,12 +5,19 @@ import com.fasterxml.jackson.databind.DeserializationFeature import com.fasterxml.jackson.databind.ObjectMapper import java.io.ByteArrayInputStream import java.io.StringWriter +import java.net.URLDecoder import javax.xml.parsers.DocumentBuilderFactory import javax.xml.transform.OutputKeys import javax.xml.transform.TransformerFactory import javax.xml.transform.dom.DOMSource import javax.xml.transform.stream.StreamResult +private fun looksLikeNumberOrBoolean(s: String): Boolean { + val t = s.trim() + if (t == "true" || t == "false") return true + return t.toBigDecimalOrNull() != null +} + /** * @javatypes: manzhang * @date: 27/08/2018 @@ -68,17 +75,26 @@ abstract class OutputFormatter (val name: String) { throw MismatchedFormatException(this, content) } - override fun readFields(content: String, fieldNames: Set): Map? { + override fun readFields( + content: String, + fieldNames: Set?, + numericAndBooleanOnly: Boolean + ): Map? { return try { val node = objectMapper.readTree(content) if (!node.isObject) return null - fieldNames.mapNotNull { field -> - val value = node.get(field) ?: return@mapNotNull null + val out = mutableMapOf() + val it = node.fields() + while (it.hasNext()) { + val (name, value) = it.next() // JSON null is reported as field-absent so callers cannot confuse // it with the literal 4-char string "null" (asText() collapses both). - if (value.isNull) return@mapNotNull null - field to value.asText() - }.toMap() + if (value.isNull) continue + if (fieldNames != null && name !in fieldNames) continue + if (numericAndBooleanOnly && !value.isNumber && !value.isBoolean) continue + out[name] = value.asText() + } + out } catch (e: Exception) { null } @@ -111,24 +127,110 @@ abstract class OutputFormatter (val name: String) { return writer.toString().replace("\r\n", "\n") } - override fun readFields(content: String, fieldNames: Set): Map? { + override fun readFields( + content: String, + fieldNames: Set?, + numericAndBooleanOnly: Boolean + ): Map? { return try { val doc = xmlFactory.newDocumentBuilder() .parse(ByteArrayInputStream(content.toByteArray(Charsets.UTF_8))) doc.documentElement.normalize() - fieldNames.mapNotNull { field -> - val nodes = doc.getElementsByTagName(field) - if (nodes.length > 0) field to nodes.item(0).textContent else null - }.toMap() + val out = mutableMapOf() + if (fieldNames != null) { + // resolve by tag name (any depth) to preserve historical behavior + for (field in fieldNames) { + val nodes = doc.getElementsByTagName(field) + if (nodes.length == 0) continue + val text = nodes.item(0).textContent ?: continue + if (numericAndBooleanOnly && !looksLikeNumberOrBoolean(text)) continue + out[field] = text + } + } else { + // walk the tree and collect leaf elements (those with no child elements). + // Spring/JAXB often wrap data in envelope/root elements, so iterating only + // top-level children would miss nested fields and produce concatenated + // text content from non-leaf elements. + collectLeafElements(doc.documentElement, out, numericAndBooleanOnly) + } + out } catch (e: Exception) { null } } + + private fun collectLeafElements( + element: org.w3c.dom.Element, + out: MutableMap, + numericAndBooleanOnly: Boolean + ) { + val children = element.childNodes + val elementChildren = mutableListOf() + for (i in 0 until children.length) { + val n = children.item(i) + if (n.nodeType == org.w3c.dom.Node.ELEMENT_NODE) { + elementChildren.add(n as org.w3c.dom.Element) + } + } + if (elementChildren.isEmpty()) { + val text = element.textContent ?: return + if (numericAndBooleanOnly && !looksLikeNumberOrBoolean(text)) return + out[element.tagName] = text + } else { + for (child in elementChildren) { + collectLeafElements(child, out, numericAndBooleanOnly) + } + } + } + } + + + val FORM_FORMATTER = object : OutputFormatter("FORM_FORMATTER") { + + override fun isValid(content: String): Boolean { + return parseForm(content).isNotEmpty() + } + + override fun getFormatted(content: String): String = content + + override fun readFields( + content: String, + fieldNames: Set?, + numericAndBooleanOnly: Boolean + ): Map? { + return try { + val parsed = parseForm(content) + if (parsed.isEmpty()) return null + val out = mutableMapOf() + for ((name, value) in parsed) { + if (fieldNames != null && name !in fieldNames) continue + if (numericAndBooleanOnly && !looksLikeNumberOrBoolean(value)) continue + out[name] = value + } + out + } catch (e: Exception) { + null + } + } + + private fun parseForm(body: String): Map { + if (body.isBlank()) return emptyMap() + return body.split("&").mapNotNull { pair -> + val parts = pair.split("=", limit = 2) + if (parts.size == 2) { + try { + URLDecoder.decode(parts[0], "UTF-8") to + URLDecoder.decode(parts[1], "UTF-8") + } catch (e: Exception) { null } + } else null + }.toMap() + } } init { registerFormatter(JSON_FORMATTER) registerFormatter(XML_FORMATTER) + registerFormatter(FORM_FORMATTER) } @@ -136,8 +238,21 @@ abstract class OutputFormatter (val name: String) { abstract fun isValid(content: String): Boolean abstract fun getFormatted(content: String): String - abstract fun readFields(content: String, fieldNames: Set): Map? - + /** + * Read fields from [content]. + * + * @param fieldNames if non-null, only these fields are returned; if null, all top-level + * primitive fields are returned. + * @param numericAndBooleanOnly if true, fields whose value is not a number or boolean are + * skipped (useful for oracles that compare two responses but + * want to ignore strings to avoid flakiness from timestamps, + * UUIDs, etc.) + * @return map from field name to canonical string value, or null if [content] cannot be parsed + */ + abstract fun readFields( + content: String, + fieldNames: Set? = null, + numericAndBooleanOnly: Boolean = false + ): Map? } - diff --git a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/ExperimentalFaultCategory.kt b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/ExperimentalFaultCategory.kt index 273e924173..2480cc7b75 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/ExperimentalFaultCategory.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/ExperimentalFaultCategory.kt @@ -35,6 +35,9 @@ enum class ExperimentalFaultCategory( HTTP_MISLEADING_CREATE_PUT(917, "PUT if creating, must get 201", "misleadingCreatePut", "TODO"), + HTTP_NON_IDEMPOTENT_PUT(918, "PUT is idempotent", "nonIdempotentPut", + "TODO"), + HTTP_STATUS_NO_NON_STANDARD_CODES(950, "no-non-standard-codes", "invalidStatusCode", "TODO"), HTTP_STATUS_NO_201_IF_DELETE(951, "no-201-if-delete", "201OnDelete", "TODO"), HTTP_STATUS_NO_201_IF_GET(952, "no-201-if-get", "201OnGet", "TODO"), diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracle.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracle.kt index e14cda71b5..3855ca3368 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracle.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracle.kt @@ -610,6 +610,80 @@ object HttpSemanticsOracle { return StatusGroup.G_2xx.isInGroup(resGet.getStatusCode()) } + /** + * Checks the non-idempotent PUT oracle: + * + * PUT /X -> 2xx + * GET /X (or ancestor) -> 2xx (state after 1st PUT) + * PUT /X -> 2xx (same body) + * GET /X (or ancestor) -> 2xx (state after 2nd PUT) + * + * Returns true if any numeric or boolean field differs between the two GET responses. + * String fields are intentionally ignored (timestamps/UUIDs etc. cause flakiness). + */ + fun hasNonIdempotentPut( + individual: RestIndividual, + actionResults: List + ): Boolean { + + if (individual.size() < 4) return false + + val actions = individual.seeMainExecutableActions() + val put1 = actions[actions.size - 4] + val get1 = actions[actions.size - 3] + val put2 = actions[actions.size - 2] + val get2 = actions[actions.size - 1] + + if (put1.verb != HttpVerb.PUT || put2.verb != HttpVerb.PUT) return false + if (get1.verb != HttpVerb.GET || get2.verb != HttpVerb.GET) return false + + // both PUTs on same resolved path with same auth + if (!put1.usingSameResolvedPath(put2)) return false + if (put1.auth.isDifferentFrom(put2.auth)) return false + + // both GETs on same resolved path with same auth + if (!get1.usingSameResolvedPath(get2)) return false + if (get1.auth.isDifferentFrom(get2.auth)) return false + + val resPut1 = actionResults.find { it.sourceLocalId == put1.getLocalId() } as RestCallResult? + ?: return false + val resGet1 = actionResults.find { it.sourceLocalId == get1.getLocalId() } as RestCallResult? + ?: return false + val resPut2 = actionResults.find { it.sourceLocalId == put2.getLocalId() } as RestCallResult? + ?: return false + val resGet2 = actionResults.find { it.sourceLocalId == get2.getLocalId() } as RestCallResult? + ?: return false + + // all four must be 2xx for the oracle to apply + if (!StatusGroup.G_2xx.isInGroup(resPut1.getStatusCode())) return false + if (!StatusGroup.G_2xx.isInGroup(resGet1.getStatusCode())) return false + if (!StatusGroup.G_2xx.isInGroup(resPut2.getStatusCode())) return false + if (!StatusGroup.G_2xx.isInGroup(resGet2.getStatusCode())) return false + + val body1 = resGet1.getBody() ?: return false + val body2 = resGet2.getBody() ?: return false + if (body1.isEmpty() || body2.isEmpty()) return false + + val formatters = listOf( + OutputFormatter.JSON_FORMATTER, + OutputFormatter.XML_FORMATTER, + OutputFormatter.FORM_FORMATTER + ) + for (formatter in formatters) { + val fields1 = formatter.readFields(body1, numericAndBooleanOnly = true) ?: continue + val fields2 = formatter.readFields(body2, numericAndBooleanOnly = true) ?: continue + + if (fields1.isEmpty() && fields2.isEmpty()) continue + + for ((name, v1) in fields1) { + val v2 = fields2[name] ?: continue + if (v1 != v2) return true + } + return false + } + return false + } + private fun parseFormBody(body: String): Map { return body.split("&").mapNotNull { pair -> val parts = pair.split("=", limit = 2) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt index 5596ec8a30..7d426343f4 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt @@ -119,6 +119,9 @@ class HttpSemanticsService : TimeBoxedPhase{ if(hasPhaseTimedOut()) return misleadingCreatePut() + + if(hasPhaseTimedOut()) return + nonIdempotentPut() } /** @@ -490,4 +493,63 @@ class HttpSemanticsService : TimeBoxedPhase{ prepareEvaluateAndSave(ind) } } + + /** + * HTTP_NON_IDEMPOTENT_PUT oracle: PUT must be idempotent — applying it once or N times must + * leave the resource in the same state. Calling the same PUT twice and observing different + * resource state in two GET responses indicates a bug (e.g. a "deposit" that accumulates). + * + * Sequence checked: + * [...resource creation...] + * PUT /X -> 2xx (the 1st PUT, already in T) + * GET /X (or ancestor) -> 2xx (state after 1st PUT) + * PUT /X → 2xx (a COPY of the 1st PUT, same body) + * GET /X (or ancestor) -> 2xx (state after 2nd PUT) + * + * The two GETs must observe identical state; if any number/boolean leaf field differs, + * idempotency is broken. Strings are intentionally ignored to avoid flakiness from + * timestamps/UUIDs etc. + * + * For endpoints like PUT /accounts/{id}/deposit, the GET is taken on the closest ancestor + * (e.g. GET /accounts/{id}) so the resource state can be observed. + */ + private fun nonIdempotentPut() { + + val putOperations = RestIndividualSelectorUtils.getAllActionDefinitions(actionDefinitions, HttpVerb.PUT) + + putOperations.forEach { putOp -> + + if (hasPhaseTimedOut()) return + + // GET on the same path, or longest ancestor with a GET + val getDef = actionDefinitions.find { it.verb == HttpVerb.GET && it.path == putOp.path } + ?: actionDefinitions + .filter { it.verb == HttpVerb.GET && it.path.isSameOrAncestorOf(putOp.path) } + .maxByOrNull { it.path.levels() } + ?: return@forEach + + // T: smallest individual ending with PUT 2xx on this path + val ind = RestIndividualSelectorUtils.findAndSlice( + individualsInSolution, HttpVerb.PUT, putOp.path, statusGroup = StatusGroup.G_2xx + ).minByOrNull { it.size() } ?: return@forEach + + val firstPut = ind.seeMainExecutableActions().last() // PUT 2xx (1st) + + // GET after the 1st PUT: bound to firstPut's resolved path and auth + val get1 = builder.createBoundActionFor(getDef, firstPut) + + // 2nd PUT: exact copy of the 1st PUT (same body) to test idempotency of that request + val secondPut = firstPut.copy() as RestCallAction + secondPut.resetLocalIdRecursively() + + // GET after the 2nd PUT + val get2 = builder.createBoundActionFor(getDef, firstPut) + + ind.addMainActionInEmptyEnterpriseGroup(-1, get1) + ind.addMainActionInEmptyEnterpriseGroup(-1, secondPut) + ind.addMainActionInEmptyEnterpriseGroup(-1, get2) + + prepareEvaluateAndSave(ind) + } + } } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt index 44352621b0..56a1edff89 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt @@ -1267,6 +1267,10 @@ abstract class AbstractRestFitness : HttpWsFitness() { if(config.isEnabledFaultCategory(ExperimentalFaultCategory.HTTP_MISLEADING_CREATE_PUT)) { handleMisleadingCreatePut(individual, actionResults, fv) } + + if(config.isEnabledFaultCategory(ExperimentalFaultCategory.HTTP_NON_IDEMPOTENT_PUT)) { + handleNonIdempotentPut(individual, actionResults, fv) + } } private fun handleFailedModification( @@ -1381,6 +1385,25 @@ abstract class AbstractRestFitness : HttpWsFitness() { ar.addFault(DetectedFault(category, put.getName(), null)) } + private fun handleNonIdempotentPut( + individual: RestIndividual, + actionResults: List, + fv: FitnessValue + ) { + if (!HttpSemanticsOracle.hasNonIdempotentPut(individual, actionResults)) return + + val actions = individual.seeMainExecutableActions() + // sequence ends with: PUT, GET, PUT, GET — flag the 2nd PUT as the offending action + val secondPut = actions[actions.size - 2] + + val category = ExperimentalFaultCategory.HTTP_NON_IDEMPOTENT_PUT + val scenarioId = idMapper.handleLocalTarget(idMapper.getFaultDescriptiveId(category, secondPut.getName())) + fv.updateTarget(scenarioId, 1.0, actions.size - 2) + + val ar = actionResults.find { it.sourceLocalId == secondPut.getLocalId() } as RestCallResult? ?: return + ar.addFault(DetectedFault(category, secondPut.getName(), null)) + } + protected fun recordResponseData(individual: RestIndividual, actionResults: List) { From dbac16ed54d24c0d4a671473a44c074d318fa69d Mon Sep 17 00:00:00 2001 From: Omur Date: Tue, 12 May 2026 11:50:29 +0300 Subject: [PATCH 2/4] fix brittle test --- .../org/evomaster/core/output/formatter/OutputFormatterTest.kt | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/src/test/kotlin/org/evomaster/core/output/formatter/OutputFormatterTest.kt b/core/src/test/kotlin/org/evomaster/core/output/formatter/OutputFormatterTest.kt index 4cdb6d76ee..36f09722b6 100644 --- a/core/src/test/kotlin/org/evomaster/core/output/formatter/OutputFormatterTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/output/formatter/OutputFormatterTest.kt @@ -158,7 +158,6 @@ class OutputFormatterTest { @Test fun testXml(){ - assertTrue(OutputFormatter.getFormatters()?.size == 2) val body = """ VZyJz8z_Eu2 @@ -173,7 +172,6 @@ class OutputFormatterTest { @Test fun testXmlMismatched(){ - assertTrue(OutputFormatter.getFormatters()?.size == 2) val body = """ VZyJz8z_Eu2 From 12fb115c01e63ae696dc14d3320c9c75f826ab94 Mon Sep 17 00:00:00 2001 From: Omur Sahin Date: Thu, 14 May 2026 23:07:33 +0300 Subject: [PATCH 3/4] remove unused field --- .../xml/HttpNonIdempotentPutXMLApplication.kt | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/nonidempotentput/xml/HttpNonIdempotentPutXMLApplication.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/nonidempotentput/xml/HttpNonIdempotentPutXMLApplication.kt index 684ba250fa..15238b0f4b 100644 --- a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/nonidempotentput/xml/HttpNonIdempotentPutXMLApplication.kt +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/nonidempotentput/xml/HttpNonIdempotentPutXMLApplication.kt @@ -44,12 +44,7 @@ open class HttpNonIdempotentPutXMLApplication { @XmlRootElement(name = "depositRequest") @XmlAccessorType(XmlAccessType.FIELD) open class DepositRequest( - var amount: Int = 0, - // EvoMaster's ObjectGene XML serializer inlines single-field bodies as - // 5 instead of 5, - // which JAXB cannot bind (amount falls back to its Kotlin default). - // This unused field forces the multi-field path so the random `amount` actually reaches the server. - var note: String = "" + var amount: Int = 0 ) From e7b59fb707d5a228bbf88e981b2e3a84dc232cdf Mon Sep 17 00:00:00 2001 From: Omur Date: Fri, 15 May 2026 11:38:57 +0300 Subject: [PATCH 4/4] moving looksLikeNumberOrBoolean into StringUtils --- .../core/output/formatter/OutputFormatter.kt | 13 +++++-------- .../kotlin/org/evomaster/core/utils/StringUtils.kt | 12 ++++++++++++ 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/output/formatter/OutputFormatter.kt b/core/src/main/kotlin/org/evomaster/core/output/formatter/OutputFormatter.kt index 303da0cc76..a9c6ad5ae7 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/formatter/OutputFormatter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/formatter/OutputFormatter.kt @@ -3,6 +3,7 @@ package org.evomaster.core.output.formatter import com.fasterxml.jackson.core.JsonProcessingException import com.fasterxml.jackson.databind.DeserializationFeature import com.fasterxml.jackson.databind.ObjectMapper +import org.evomaster.core.utils.StringUtils import java.io.ByteArrayInputStream import java.io.StringWriter import java.net.URLDecoder @@ -12,11 +13,7 @@ import javax.xml.transform.TransformerFactory import javax.xml.transform.dom.DOMSource import javax.xml.transform.stream.StreamResult -private fun looksLikeNumberOrBoolean(s: String): Boolean { - val t = s.trim() - if (t == "true" || t == "false") return true - return t.toBigDecimalOrNull() != null -} + /** * @javatypes: manzhang @@ -143,7 +140,7 @@ abstract class OutputFormatter (val name: String) { val nodes = doc.getElementsByTagName(field) if (nodes.length == 0) continue val text = nodes.item(0).textContent ?: continue - if (numericAndBooleanOnly && !looksLikeNumberOrBoolean(text)) continue + if (numericAndBooleanOnly && !StringUtils.looksLikeNumberOrBoolean(text)) continue out[field] = text } } else { @@ -174,7 +171,7 @@ abstract class OutputFormatter (val name: String) { } if (elementChildren.isEmpty()) { val text = element.textContent ?: return - if (numericAndBooleanOnly && !looksLikeNumberOrBoolean(text)) return + if (numericAndBooleanOnly && !StringUtils.looksLikeNumberOrBoolean(text)) return out[element.tagName] = text } else { for (child in elementChildren) { @@ -204,7 +201,7 @@ abstract class OutputFormatter (val name: String) { val out = mutableMapOf() for ((name, value) in parsed) { if (fieldNames != null && name !in fieldNames) continue - if (numericAndBooleanOnly && !looksLikeNumberOrBoolean(value)) continue + if (numericAndBooleanOnly && !StringUtils.looksLikeNumberOrBoolean(value)) continue out[name] = value } out diff --git a/core/src/main/kotlin/org/evomaster/core/utils/StringUtils.kt b/core/src/main/kotlin/org/evomaster/core/utils/StringUtils.kt index 7557b4ccfa..26ec83ac4b 100644 --- a/core/src/main/kotlin/org/evomaster/core/utils/StringUtils.kt +++ b/core/src/main/kotlin/org/evomaster/core/utils/StringUtils.kt @@ -155,4 +155,16 @@ object StringUtils { 'Ɍ' to "R", 'ɍ' to "r", // R with stroke 'Ɏ' to "Y", 'ɏ' to "y", // Y with stroke ) + + /** + * Checks whether the given string looks like a number or a boolean value. + * + * The function trims leading and trailing whitespace, then returns true if the + * resulting value is either "true", "false", or a valid decimal number. + */ + fun looksLikeNumberOrBoolean(s: String): Boolean { + val t = s.trim() + if (t == "true" || t == "false") return true + return t.toBigDecimalOrNull() != null + } }