From 88a0537be29e1ad93285d098722f8fc0e1c7956f Mon Sep 17 00:00:00 2001 From: Omur Date: Tue, 17 Feb 2026 17:02:42 +0300 Subject: [PATCH 01/13] failed modification starter --- .../FailModificationApplication.kt | 92 +++++++++++++++++++ .../FailModificationController.kt | 11 +++ .../FailModificationEMTest.kt | 46 ++++++++++ .../enterprise/ExperimentalFaultCategory.kt | 2 + .../rest/service/HttpSemanticsService.kt | 79 ++++++++++++++++ .../service/fitness/AbstractRestFitness.kt | 14 +++ 6 files changed, 244 insertions(+) create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationApplication.kt create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationController.kt create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationEMTest.kt diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationApplication.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationApplication.kt new file mode 100644 index 0000000000..afeb1edb24 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationApplication.kt @@ -0,0 +1,92 @@ +package com.foo.rest.examples.spring.openapi.v3.httporacle.failmodification + +import org.springframework.boot.SpringApplication +import org.springframework.boot.autoconfigure.SpringBootApplication +import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.* + + +@SpringBootApplication(exclude = [SecurityAutoConfiguration::class]) +@RequestMapping(path = ["/api/resources"]) +@RestController +open class FailModificationApplication { + + companion object { + @JvmStatic + fun main(args: Array) { + SpringApplication.run(FailModificationApplication::class.java, *args) + } + + private val data = mutableMapOf() + + fun reset(){ + data.clear() + } + } + + data class ResourceData( + var name: String, + var value: Int + ) + + data class UpdateRequest( + val name: String?, + val value: Int? + ) + + + @PostMapping + open fun create(@RequestBody body: ResourceData): ResponseEntity { + val id = data.size + 1 + data[id] = body.copy() + return ResponseEntity.status(201).body(data[id]) + } + + @GetMapping(path = ["/{id}"]) + open fun get(@PathVariable("id") id: Int): ResponseEntity { + val resource = data[id] + ?: return ResponseEntity.status(404).build() + return ResponseEntity.status(200).body(resource) + } + + @PutMapping(path = ["/{id}"]) + open fun put( + @PathVariable("id") id: Int, + @RequestBody body: UpdateRequest + ): ResponseEntity { + + val resource = data[id] + ?: return ResponseEntity.status(404).build() + + // bug: modifies data even though it will return 4xx + if(body.name != null) { + resource.name = body.name + } + if(body.value != null) { + resource.value = body.value + } + + // returns 400 Bad Request, but the data was already modified above + return ResponseEntity.status(400).body("Invalid request") + } + + @PatchMapping(path = ["/{id}"]) + open fun patch( + @PathVariable("id") id: Int, + @RequestBody body: UpdateRequest + ): ResponseEntity { + + val resource = data[id] + ?: return ResponseEntity.status(404).build() + + // correct: validation first, reject without modifying + if(body.name == null && body.value == null) { + return ResponseEntity.status(400).body("No fields to update") + } + + // correct: does NOT modify data, just returns 4xx + return ResponseEntity.status(403).body("Forbidden") + } + +} diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationController.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationController.kt new file mode 100644 index 0000000000..a507132fb1 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationController.kt @@ -0,0 +1,11 @@ +package com.foo.rest.examples.spring.openapi.v3.httporacle.failmodification + +import com.foo.rest.examples.spring.openapi.v3.SpringController + + +class FailModificationController: SpringController(FailModificationApplication::class.java){ + + override fun resetStateOfSUT() { + FailModificationApplication.reset() + } +} diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationEMTest.kt new file mode 100644 index 0000000000..6f9ea126c9 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationEMTest.kt @@ -0,0 +1,46 @@ +package org.evomaster.e2etests.spring.openapi.v3.httporacle.failmodification + +import com.foo.rest.examples.spring.openapi.v3.httporacle.failmodification.FailModificationController +import org.evomaster.core.problem.enterprise.DetectedFaultUtils +import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory +import org.evomaster.core.problem.rest.data.HttpVerb +import org.evomaster.e2etests.spring.openapi.v3.SpringTestBase +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test + +class FailModificationEMTest : SpringTestBase(){ + + companion object { + @BeforeAll + @JvmStatic + fun init() { + initClass(FailModificationController()) + } + } + + + @Test + fun testRunEM() { + + runTestHandlingFlakyAndCompilation( + "FailedModificationEM", + 100 + ) { args: MutableList -> + + setOption(args, "security", "false") + setOption(args, "schemaOracles", "false") + setOption(args, "httpOracles", "true") + setOption(args, "useExperimentalOracles", "true") + + val solution = initAndRun(args) + + assertTrue(solution.individuals.size >= 1) + + val faults = DetectedFaultUtils.getDetectedFaultCategories(solution) + assertEquals(1, faults.size) + assertEquals(ExperimentalFaultCategory.HTTP_SIDE_EFFECTS_FAILED_MODIFICATION, faults.first()) + } + } +} diff --git a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/ExperimentalFaultCategory.kt b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/ExperimentalFaultCategory.kt index 2ffccf9836..eb510cb72c 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/ExperimentalFaultCategory.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/ExperimentalFaultCategory.kt @@ -35,6 +35,8 @@ enum class ExperimentalFaultCategory( "TODO"), HTTP_REPEATED_CREATE_PUT(914, "Repeated PUT Creates Resource With 201", "repeatedCreatePut", "TODO"), + HTTP_SIDE_EFFECTS_FAILED_MODIFICATION(915, "A failed PUT or PATCH must not change the resource", "sideEffectsFailedModification", + "TODO"), //3xx: GraphQL diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt index 762bf3759d..cad8de5945 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt @@ -90,6 +90,8 @@ class HttpSemanticsService { // – A repeated followup PUT with 201 on same endpoint should not return 201 (must enforce 200 or 204) putRepeatedCreated() + + sideEffectsOfFailedModification() } /** @@ -198,4 +200,81 @@ class HttpSemanticsService { } } + /** + * Checking bugs like: + * GET /X 2xx (before state) + * PUT|PATCH /X 4xx (failed modification) + * GET /X 2xx (after state - should be same as before) + * + * If a PUT/PATCH fails with 4xx, it should have no side-effects. + * A GET before and after should return the same resource state. + */ + private fun sideEffectsOfFailedModification() { + + val verbs = listOf(HttpVerb.PUT, HttpVerb.PATCH) + + for (verb in verbs) { + + val modifyOperations = RestIndividualSelectorUtils.getAllActionDefinitions(actionDefinitions, verb) + + modifyOperations.forEach { modOp -> + + // check that a GET definition exists for this same path + val getDef = actionDefinitions.find { it.verb == HttpVerb.GET && it.path == modOp.path } + ?: return@forEach + + // check that a 2xx GET exists in the solution for this path + val successGet = RestIndividualSelectorUtils.findAction( + individualsInSolution, + HttpVerb.GET, + modOp.path, + statusGroup = StatusGroup.G_2xx + ) ?: return@forEach + + // find individuals where this PUT/PATCH returned 4xx + val failedModifications = RestIndividualSelectorUtils.findAndSlice( + individualsInSolution, + verb, + modOp.path, + statusGroup = StatusGroup.G_4xx + ) + if (failedModifications.isEmpty()) { + return@forEach + } + + val ind = failedModifications.minBy { it.size() } + val actions = ind.seeMainExecutableActions() + val last = actions[actions.size - 1] // the failed PUT/PATCH + + // check if there is already a 2xx GET right before the failed PUT/PATCH + val hasPreviousGet = ind.size() > 1 + && actions[actions.size - 2].let { + it.verb == HttpVerb.GET && it.path == modOp.path + && it.usingSameResolvedPath(last) + && !it.auth.isDifferentFrom(last.auth) + } + + val previous = if (!hasPreviousGet) { + val getOp = getDef.copy() as RestCallAction + getOp.doInitialize(randomness) + getOp.forceNewTaints() + getOp.bindToSamePathResolution(last) + getOp.auth = last.auth + ind.addMainActionInEmptyEnterpriseGroup(actions.size - 1, getOp) + getOp + } else { + actions[actions.size - 2] + } + + // add GET after the failed PUT/PATCH to verify no side-effects + val after = previous.copy() as RestCallAction + after.resetLocalIdRecursively() + ind.addMainActionInEmptyEnterpriseGroup(-1, after) + + prepareEvaluateAndSave(ind) + } + } + } + + } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt index f90e48d33f..4cc118384d 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt @@ -1217,6 +1217,20 @@ abstract class AbstractRestFitness : HttpWsFitness() { } else { handleRepeatedCreatePut(individual, actionResults, fv) } + + if(!config.isEnabledFaultCategory(ExperimentalFaultCategory.HTTP_SIDE_EFFECTS_FAILED_MODIFICATION)) { + LoggingUtil.uniqueUserInfo("Skipping experimental security test for repeated PUT after CREATE, as it has been disabled via configuration") + } else { + handleFailedModification(individual, actionResults, fv) + } + } + + private fun handleFailedModification( + individual: RestIndividual, + actionResults: List, + fv: FitnessValue + ) { + } private fun handleRepeatedCreatePut( From b69d9cec00037432ba2bcbd2437141beffaa5046 Mon Sep 17 00:00:00 2001 From: Omur Date: Wed, 18 Feb 2026 16:18:36 +0300 Subject: [PATCH 02/13] failed modification create ind --- .../FailModificationEMTest.kt | 2 +- .../rest/oracle/HttpSemanticsOracle.kt | 143 +++++++++++++++++- .../rest/service/HttpSemanticsService.kt | 96 +++++++----- .../service/fitness/AbstractRestFitness.kt | 16 ++ 4 files changed, 217 insertions(+), 40 deletions(-) diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationEMTest.kt index 6f9ea126c9..5fa5a8b617 100644 --- a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationEMTest.kt +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationEMTest.kt @@ -26,7 +26,7 @@ class FailModificationEMTest : SpringTestBase(){ runTestHandlingFlakyAndCompilation( "FailedModificationEM", - 100 + 500 ) { args: MutableList -> setOption(args, "security", "false") diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracle.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracle.kt index c934b40862..bfef646911 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracle.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracle.kt @@ -1,10 +1,14 @@ package org.evomaster.core.problem.rest.oracle +import com.google.gson.JsonParser import org.evomaster.core.problem.rest.data.HttpVerb +import org.evomaster.core.problem.rest.data.RestCallAction import org.evomaster.core.problem.rest.data.RestCallResult import org.evomaster.core.problem.rest.data.RestIndividual +import org.evomaster.core.problem.rest.param.BodyParam import org.evomaster.core.problem.rest.StatusGroup import org.evomaster.core.search.action.ActionResult +import org.evomaster.core.search.gene.ObjectGene object HttpSemanticsOracle { @@ -103,4 +107,141 @@ object HttpSemanticsOracle { return NonWorkingDeleteResult(checkingDelete, nonWorking, delete.getName(), actions.size - 2) } -} \ No newline at end of file + + fun hasSideEffectFailedModification(individual: RestIndividual, + actionResults: List + ): Boolean{ + + if(individual.size() < 3){ + return false + } + + val actions = individual.seeMainExecutableActions() + + val before = actions[actions.size - 3] // GET (before state) + val modify = actions[actions.size - 2] // PUT or PATCH (failed modification) + val after = actions[actions.size - 1] // GET (after state) + + // check verbs: GET, PUT|PATCH, GET + if(before.verb != HttpVerb.GET) { + return false + } + if(modify.verb != HttpVerb.PUT && modify.verb != HttpVerb.PATCH) { + return false + } + if(after.verb != HttpVerb.GET) { + return false + } + + // all three must be on the same resolved path + if(!before.usingSameResolvedPath(modify) || !after.usingSameResolvedPath(modify)) { + return false + } + + // auth should be consistent + if(before.auth.isDifferentFrom(modify.auth) || after.auth.isDifferentFrom(modify.auth)) { + return false + } + + val resBefore = actionResults.find { it.sourceLocalId == before.getLocalId() } as RestCallResult? + ?: return false + val resModify = actionResults.find { it.sourceLocalId == modify.getLocalId() } as RestCallResult? + ?: return false + val resAfter = actionResults.find { it.sourceLocalId == after.getLocalId() } as RestCallResult? + ?: return false + + // before GET must be 2xx + if(!StatusGroup.G_2xx.isInGroup(resBefore.getStatusCode())) { + return false + } + + // PUT/PATCH must have failed with 4xx + if(!StatusGroup.G_4xx.isInGroup(resModify.getStatusCode())) { + return false + } + + // after GET must be 2xx + if(!StatusGroup.G_2xx.isInGroup(resAfter.getStatusCode())) { + return false + } + + val bodyBefore = resBefore.getBody() + val bodyAfter = resAfter.getBody() + + // if both are null/empty, no side-effect detected + if(bodyBefore.isNullOrEmpty() && bodyAfter.isNullOrEmpty()) { + return false + } + + // extract the field names sent in the PUT/PATCH request body + val modifiedFieldNames = extractModifiedFieldNames(modify) + + // if we can identify specific fields, compare only those to avoid false positives from timestamps etc. + if(modifiedFieldNames.isNotEmpty() + && !bodyBefore.isNullOrEmpty() + && !bodyAfter.isNullOrEmpty()) { + return hasChangedModifiedFields(bodyBefore, bodyAfter, modifiedFieldNames) + } + + // otherwise compare entire bodies + return bodyBefore != bodyAfter + } + + /** + * Extract field names from the PUT/PATCH request body. + * These are the fields that the client attempted to modify. + */ + private fun extractModifiedFieldNames(modify: RestCallAction): Set { + + val bodyParam = modify.parameters.find { it is BodyParam } as BodyParam? + ?: return emptySet() + + val gene = bodyParam.primaryGene() + val objectGene = gene.getWrappedGene(ObjectGene::class.java) as ObjectGene? + ?: if (gene is ObjectGene) gene else null + + if(objectGene == null){ + return emptySet() + } + + return objectGene.fields.map { it.name }.toSet() + } + + /** + * Compare only the fields that were sent in the PUT/PATCH request. + * Returns true if any of those fields changed between before and after GET responses. + */ + private fun hasChangedModifiedFields( + bodyBefore: String, + bodyAfter: String, + fieldNames: Set + ): Boolean { + + try { + val jsonBefore = JsonParser.parseString(bodyBefore) + val jsonAfter = JsonParser.parseString(bodyAfter) + + if(!jsonBefore.isJsonObject || !jsonAfter.isJsonObject){ + // not JSON objects, fallback to full comparison + return bodyBefore != bodyAfter + } + + val objBefore = jsonBefore.asJsonObject + val objAfter = jsonAfter.asJsonObject + + for(field in fieldNames){ + val valueBefore = objBefore.get(field) + val valueAfter = objAfter.get(field) + + if(valueBefore != valueAfter){ + return true + } + } + + return false + } catch (e: Exception) { + // JSON parsing failed, fallback to full comparison + return bodyBefore != bodyAfter + } + } +} diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt index cad8de5945..af38757328 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt @@ -6,6 +6,7 @@ import org.evomaster.core.problem.rest.* import org.evomaster.core.problem.rest.builder.RestIndividualSelectorUtils import org.evomaster.core.problem.rest.data.HttpVerb import org.evomaster.core.problem.rest.data.RestCallAction +import org.evomaster.core.problem.rest.data.RestCallResult import org.evomaster.core.problem.rest.data.RestIndividual import org.evomaster.core.problem.rest.service.fitness.RestFitness import org.evomaster.core.problem.rest.service.sampler.AbstractRestSampler @@ -45,6 +46,9 @@ class HttpSemanticsService { @Inject private lateinit var idMapper: IdMapper + @Inject + private lateinit var builder: RestIndividualBuilder + /** * All actions that can be defined from the OpenAPI schema */ @@ -202,9 +206,10 @@ class HttpSemanticsService { /** * Checking bugs like: - * GET /X 2xx (before state) - * PUT|PATCH /X 4xx (failed modification) - * GET /X 2xx (after state - should be same as before) + * POST|PUT /X 2xx (create resource) + * GET /X 2xx (before state) + * PUT|PATCH /X 4xx (failed modification) + * GET /X 2xx (after state - should be same as before) * * If a PUT/PATCH fails with 4xx, it should have no side-effects. * A GET before and after should return the same resource state. @@ -223,53 +228,68 @@ class HttpSemanticsService { val getDef = actionDefinitions.find { it.verb == HttpVerb.GET && it.path == modOp.path } ?: return@forEach - // check that a 2xx GET exists in the solution for this path - val successGet = RestIndividualSelectorUtils.findAction( - individualsInSolution, - HttpVerb.GET, - modOp.path, - statusGroup = StatusGroup.G_2xx - ) ?: return@forEach - - // find individuals where this PUT/PATCH returned 4xx - val failedModifications = RestIndividualSelectorUtils.findAndSlice( + // find individuals that have a 4xx PUT/PATCH on this path + val failedModifyIndividuals = RestIndividualSelectorUtils.findIndividuals( individualsInSolution, verb, modOp.path, statusGroup = StatusGroup.G_4xx ) - if (failedModifications.isEmpty()) { + if (failedModifyIndividuals.isEmpty()) { return@forEach } - val ind = failedModifications.minBy { it.size() } - val actions = ind.seeMainExecutableActions() - val last = actions[actions.size - 1] // the failed PUT/PATCH - - // check if there is already a 2xx GET right before the failed PUT/PATCH - val hasPreviousGet = ind.size() > 1 - && actions[actions.size - 2].let { - it.verb == HttpVerb.GET && it.path == modOp.path - && it.usingSameResolvedPath(last) - && !it.auth.isDifferentFrom(last.auth) + // among those, find one that also has a successful creation step + // (POST 2xx on parent path, or PUT 201 on same path) + val parentPath = if (!modOp.path.isRoot()) modOp.path.parentPath() else null + + val withCreation = failedModifyIndividuals.filter { ind -> + ind.evaluatedMainActions().any { ea -> + val action = ea.action as RestCallAction + val result = ea.result as RestCallResult + (parentPath != null + && action.verb == HttpVerb.POST + && action.path.isEquivalent(parentPath) + && StatusGroup.G_2xx.isInGroup(result.getStatusCode())) + || + (action.verb == HttpVerb.PUT + && action.path.isEquivalent(modOp.path) + && result.getStatusCode() == 201) + } + } + if (withCreation.isEmpty()) { + return@forEach } - val previous = if (!hasPreviousGet) { - val getOp = getDef.copy() as RestCallAction - getOp.doInitialize(randomness) - getOp.forceNewTaints() - getOp.bindToSamePathResolution(last) - getOp.auth = last.auth - ind.addMainActionInEmptyEnterpriseGroup(actions.size - 1, getOp) - getOp - } else { - actions[actions.size - 2] + val selected = withCreation.minBy { it.individual.size() } + + // slice up to the 4xx PUT/PATCH + val failedIndex = RestIndividualSelectorUtils.findIndexOfAction( + selected, verb, modOp.path, statusGroup = StatusGroup.G_4xx + ) + if (failedIndex < 0) { + return@forEach } - // add GET after the failed PUT/PATCH to verify no side-effects - val after = previous.copy() as RestCallAction - after.resetLocalIdRecursively() - ind.addMainActionInEmptyEnterpriseGroup(-1, after) + val ind = RestIndividualBuilder.sliceAllCallsInIndividualAfterAction( + selected.individual, failedIndex + ) + + val actions = ind.seeMainExecutableActions() + val last = actions[actions.size - 1] // the 4xx PUT/PATCH + + // add GET before the failed PUT/PATCH + val getBefore = getDef.copy() as RestCallAction + getBefore.doInitialize(randomness) + getBefore.forceNewTaints() + getBefore.bindToSamePathResolution(last) + getBefore.auth = last.auth + ind.addMainActionInEmptyEnterpriseGroup(actions.size - 1, getBefore) + + // add GET after the failed PUT/PATCH + val getAfter = getBefore.copy() as RestCallAction + getAfter.resetLocalIdRecursively() + ind.addMainActionInEmptyEnterpriseGroup(-1, getAfter) prepareEvaluateAndSave(ind) } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt index 4cc118384d..0455e35ba6 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt @@ -1230,7 +1230,23 @@ abstract class AbstractRestFitness : HttpWsFitness() { actionResults: List, fv: FitnessValue ) { + val issues = HttpSemanticsOracle.hasSideEffectFailedModification(individual,actionResults) + if(!issues){ + return + } + + val putOrPatch = individual.seeMainExecutableActions().filter { + it.verb == HttpVerb.PUT || it.verb == HttpVerb.PATCH + }.last() + val category = ExperimentalFaultCategory.HTTP_SIDE_EFFECTS_FAILED_MODIFICATION + val scenarioId = idMapper.handleLocalTarget(idMapper.getFaultDescriptiveId(category, putOrPatch.getName()) + ) + fv.updateTarget(scenarioId, 1.0, individual.seeMainExecutableActions().lastIndex) + + val ar = actionResults.find { it.sourceLocalId == putOrPatch.getLocalId() } as RestCallResult? + ?: return + ar.addFault(DetectedFault(category, putOrPatch.getName(), null)) } private fun handleRepeatedCreatePut( From 3823ee2ddfad6953a080aa02e95a4772746d735d Mon Sep 17 00:00:00 2001 From: Omur Date: Thu, 19 Feb 2026 10:37:11 +0300 Subject: [PATCH 03/13] add more tests --- .../rest/oracle/HttpSemanticsOracle.kt | 2 +- .../rest/oracle/HttpSemanticsOracleTest.kt | 116 ++++++++++++++++++ 2 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 core/src/test/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracleTest.kt diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracle.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracle.kt index bfef646911..30cf97315d 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracle.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracle.kt @@ -211,7 +211,7 @@ object HttpSemanticsOracle { * Compare only the fields that were sent in the PUT/PATCH request. * Returns true if any of those fields changed between before and after GET responses. */ - private fun hasChangedModifiedFields( + internal fun hasChangedModifiedFields( bodyBefore: String, bodyAfter: String, fieldNames: Set diff --git a/core/src/test/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracleTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracleTest.kt new file mode 100644 index 0000000000..9919464bb7 --- /dev/null +++ b/core/src/test/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracleTest.kt @@ -0,0 +1,116 @@ +package org.evomaster.core.problem.rest.oracle + +import org.junit.jupiter.api.Assertions.* +import org.junit.jupiter.api.Test + +class HttpSemanticsOracleTest { + + @Test + fun testUnchangedModifiedFieldReturnsFalse() { + assertFalse(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = """{"name":"Doe","ts":"2026-01-01"}""", + bodyAfter = """{"name":"Doe","ts":"2026-01-02"}""", + fieldNames = setOf("name") + )) + } + + @Test + fun testNoModifiedFieldChangedReturnsFalse() { + assertFalse(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = """{"name":"Doe","email":"a@a.com","age":30}""", + bodyAfter = """{"name":"Doe","email":"a@a.com","age":31}""", + fieldNames = setOf("name", "email") + )) + } + + @Test + fun testModifiedFieldAbsentInBothBodiesReturnsFalse() { + assertFalse(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = """{"age":30}""", + bodyAfter = """{"age":31}""", + fieldNames = setOf("name") + )) + } + + @Test + fun testUnchangedIntegerModifiedFieldReturnsFalse() { + assertFalse(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = """{"count":42,"label":"test"}""", + bodyAfter = """{"count":42,"label":"changed"}""", + fieldNames = setOf("count") + )) + } + + // hasChangedModifiedFields — field changed -> true + @Test + fun testChangedModifiedFieldReturnsTrue() { + assertTrue(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = """{"name":"Doe","age":42}""", + bodyAfter = """{"name":"Bob","age":42}""", + fieldNames = setOf("name") + )) + } + + @Test + fun testOneOfMultipleModifiedFieldsChangedReturnsTrue() { + assertTrue(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = """{"name":"Doe","email":"a@a.com","age":42}""", + bodyAfter = """{"name":"Doe","email":"b@b.com","age":42}""", + fieldNames = setOf("name", "email") + )) + } + + @Test + fun testModifiedFieldPresentInBeforeButAbsentInAfterReturnsTrue() { + assertTrue(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = """{"name":"Doe"}""", + bodyAfter = """{"age":42}""", + fieldNames = setOf("name") + )) + } + + @Test + fun testChangedIntegerModifiedFieldReturnsTrue() { + assertTrue(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = """{"count":42,"label":"test"}""", + bodyAfter = """{"count":44,"label":"test"}""", + fieldNames = setOf("count") + )) + } + + @Test + fun testInvalidJsonDifferentBodiesFallbackReturnsTrue() { + assertTrue(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = "not valid json", + bodyAfter = "also not valid json", + fieldNames = setOf("name") + )) + } + + @Test + fun testInvalidJsonSameBodiesFallbackReturnsFalse() { + assertFalse(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = "not valid json", + bodyAfter = "not valid json", + fieldNames = setOf("name") + )) + } + + @Test + fun testJsonArrayDifferentBodiesFallbackReturnsTrue() { + assertTrue(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = """[{"name":"Doe"}]""", + bodyAfter = """[{"name":"Bob"}]""", + fieldNames = setOf("name") + )) + } + + @Test + fun testJsonArraySameBodiesFallbackReturnsFalse() { + assertFalse(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = """[{"name":"Doe"}]""", + bodyAfter = """[{"name":"Doe"}]""", + fieldNames = setOf("name") + )) + } +} From 418948f604748f0e278200e16e9f378f62f91197 Mon Sep 17 00:00:00 2001 From: Omur Sahin Date: Sun, 1 Mar 2026 11:58:16 +0300 Subject: [PATCH 04/13] minor updates --- .../problem/rest/oracle/HttpSemanticsOracle.kt | 15 +++++++++------ .../problem/rest/service/HttpSemanticsService.kt | 3 +++ 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracle.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracle.kt index 30cf97315d..8d9908c16b 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracle.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracle.kt @@ -183,8 +183,7 @@ object HttpSemanticsOracle { return hasChangedModifiedFields(bodyBefore, bodyAfter, modifiedFieldNames) } - // otherwise compare entire bodies - return bodyBefore != bodyAfter + return false } /** @@ -208,8 +207,12 @@ object HttpSemanticsOracle { } /** - * Compare only the fields that were sent in the PUT/PATCH request. - * Returns true if any of those fields changed between before and after GET responses. + * Compares only the fields that were sent in the PUT/PATCH request. + * Returns true if any of those fields changed between the before and after GET responses. + * + * NOTE: This only works when the request payload is a JSON object that directly + * matches the resource structure. It does NOT support operation-based payloads + * such as JSON Patch (RFC 6902). */ internal fun hasChangedModifiedFields( bodyBefore: String, @@ -223,7 +226,7 @@ object HttpSemanticsOracle { if(!jsonBefore.isJsonObject || !jsonAfter.isJsonObject){ // not JSON objects, fallback to full comparison - return bodyBefore != bodyAfter + return false } val objBefore = jsonBefore.asJsonObject @@ -241,7 +244,7 @@ object HttpSemanticsOracle { return false } catch (e: Exception) { // JSON parsing failed, fallback to full comparison - return bodyBefore != bodyAfter + return false } } } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt index af38757328..a0eafb4f06 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt @@ -213,6 +213,9 @@ class HttpSemanticsService { * * If a PUT/PATCH fails with 4xx, it should have no side-effects. * A GET before and after should return the same resource state. + * + * NOTE: When comparing states, non-deterministic fields (e.g. timestamps, + * generated IDs/UUIDs, version counters) must be excluded from the comparison. */ private fun sideEffectsOfFailedModification() { From be9571e3d0f307cc85c0fe1f02e6955fc3758f22 Mon Sep 17 00:00:00 2001 From: Omur Date: Mon, 2 Mar 2026 11:24:38 +0300 Subject: [PATCH 05/13] checking modified body --- .../rest/oracle/HttpSemanticsOracle.kt | 14 +++++--- .../rest/oracle/HttpSemanticsOracleTest.kt | 32 +++++++------------ 2 files changed, 22 insertions(+), 24 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracle.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracle.kt index 8d9908c16b..9136e94ff6 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracle.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracle.kt @@ -166,6 +166,7 @@ object HttpSemanticsOracle { } val bodyBefore = resBefore.getBody() + val bodyModify = resModify.getBody() val bodyAfter = resAfter.getBody() // if both are null/empty, no side-effect detected @@ -179,8 +180,9 @@ object HttpSemanticsOracle { // if we can identify specific fields, compare only those to avoid false positives from timestamps etc. if(modifiedFieldNames.isNotEmpty() && !bodyBefore.isNullOrEmpty() - && !bodyAfter.isNullOrEmpty()) { - return hasChangedModifiedFields(bodyBefore, bodyAfter, modifiedFieldNames) + && !bodyAfter.isNullOrEmpty() + && !bodyModify.isNullOrEmpty()) { + return hasChangedModifiedFields(bodyBefore, bodyAfter, bodyModify, modifiedFieldNames) } return false @@ -217,26 +219,30 @@ object HttpSemanticsOracle { internal fun hasChangedModifiedFields( bodyBefore: String, bodyAfter: String, + bodyModify: String, fieldNames: Set ): Boolean { try { val jsonBefore = JsonParser.parseString(bodyBefore) val jsonAfter = JsonParser.parseString(bodyAfter) + val jsonModify = JsonParser.parseString(bodyModify) - if(!jsonBefore.isJsonObject || !jsonAfter.isJsonObject){ + if(!jsonBefore.isJsonObject || !jsonAfter.isJsonObject || !jsonModify.isJsonObject){ // not JSON objects, fallback to full comparison return false } val objBefore = jsonBefore.asJsonObject val objAfter = jsonAfter.asJsonObject + val objModify = jsonModify.asJsonObject for(field in fieldNames){ val valueBefore = objBefore.get(field) val valueAfter = objAfter.get(field) + val valueModify = objModify.get(field) - if(valueBefore != valueAfter){ + if(valueBefore != valueAfter && valueModify == valueAfter){ return true } } diff --git a/core/src/test/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracleTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracleTest.kt index 9919464bb7..23a86473e7 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracleTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracleTest.kt @@ -10,6 +10,7 @@ class HttpSemanticsOracleTest { assertFalse(HttpSemanticsOracle.hasChangedModifiedFields( bodyBefore = """{"name":"Doe","ts":"2026-01-01"}""", bodyAfter = """{"name":"Doe","ts":"2026-01-02"}""", + bodyModify = """{"name":"Test"}""", fieldNames = setOf("name") )) } @@ -19,6 +20,7 @@ class HttpSemanticsOracleTest { assertFalse(HttpSemanticsOracle.hasChangedModifiedFields( bodyBefore = """{"name":"Doe","email":"a@a.com","age":30}""", bodyAfter = """{"name":"Doe","email":"a@a.com","age":31}""", + bodyModify = """{"age":31}""", fieldNames = setOf("name", "email") )) } @@ -28,6 +30,7 @@ class HttpSemanticsOracleTest { assertFalse(HttpSemanticsOracle.hasChangedModifiedFields( bodyBefore = """{"age":30}""", bodyAfter = """{"age":31}""", + bodyModify = """{"age":31}""", fieldNames = setOf("name") )) } @@ -37,6 +40,7 @@ class HttpSemanticsOracleTest { assertFalse(HttpSemanticsOracle.hasChangedModifiedFields( bodyBefore = """{"count":42,"label":"test"}""", bodyAfter = """{"count":42,"label":"changed"}""", + bodyModify = """{"count":42,"label":"changed"}""", fieldNames = setOf("count") )) } @@ -47,6 +51,7 @@ class HttpSemanticsOracleTest { assertTrue(HttpSemanticsOracle.hasChangedModifiedFields( bodyBefore = """{"name":"Doe","age":42}""", bodyAfter = """{"name":"Bob","age":42}""", + bodyModify = """{"name":"Bob"}""", fieldNames = setOf("name") )) } @@ -56,6 +61,7 @@ class HttpSemanticsOracleTest { assertTrue(HttpSemanticsOracle.hasChangedModifiedFields( bodyBefore = """{"name":"Doe","email":"a@a.com","age":42}""", bodyAfter = """{"name":"Doe","email":"b@b.com","age":42}""", + bodyModify = """{"name":"Doe","email":"b@b.com","age":42}""", fieldNames = setOf("name", "email") )) } @@ -65,6 +71,7 @@ class HttpSemanticsOracleTest { assertTrue(HttpSemanticsOracle.hasChangedModifiedFields( bodyBefore = """{"name":"Doe"}""", bodyAfter = """{"age":42}""", + bodyModify = """{"age":42}""", fieldNames = setOf("name") )) } @@ -74,15 +81,17 @@ class HttpSemanticsOracleTest { assertTrue(HttpSemanticsOracle.hasChangedModifiedFields( bodyBefore = """{"count":42,"label":"test"}""", bodyAfter = """{"count":44,"label":"test"}""", + bodyModify = """{"count":44,"label":"test"}""", fieldNames = setOf("count") )) } @Test - fun testInvalidJsonDifferentBodiesFallbackReturnsTrue() { - assertTrue(HttpSemanticsOracle.hasChangedModifiedFields( + fun testInvalidJsonDifferentBodiesFallbackReturnsFalse() { + assertFalse(HttpSemanticsOracle.hasChangedModifiedFields( bodyBefore = "not valid json", bodyAfter = "also not valid json", + bodyModify = "{}", fieldNames = setOf("name") )) } @@ -92,24 +101,7 @@ class HttpSemanticsOracleTest { assertFalse(HttpSemanticsOracle.hasChangedModifiedFields( bodyBefore = "not valid json", bodyAfter = "not valid json", - fieldNames = setOf("name") - )) - } - - @Test - fun testJsonArrayDifferentBodiesFallbackReturnsTrue() { - assertTrue(HttpSemanticsOracle.hasChangedModifiedFields( - bodyBefore = """[{"name":"Doe"}]""", - bodyAfter = """[{"name":"Bob"}]""", - fieldNames = setOf("name") - )) - } - - @Test - fun testJsonArraySameBodiesFallbackReturnsFalse() { - assertFalse(HttpSemanticsOracle.hasChangedModifiedFields( - bodyBefore = """[{"name":"Doe"}]""", - bodyAfter = """[{"name":"Doe"}]""", + bodyModify = "{}", fieldNames = setOf("name") )) } From d5d244aa9a283e1d76d860a07cd105eacd7fe81e Mon Sep 17 00:00:00 2001 From: Omur Date: Tue, 3 Mar 2026 16:50:28 +0300 Subject: [PATCH 06/13] body extract --- .../core/problem/rest/oracle/HttpSemanticsOracle.kt | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracle.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracle.kt index 9136e94ff6..68cbfff517 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracle.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracle.kt @@ -9,6 +9,7 @@ import org.evomaster.core.problem.rest.param.BodyParam import org.evomaster.core.problem.rest.StatusGroup import org.evomaster.core.search.action.ActionResult import org.evomaster.core.search.gene.ObjectGene +import org.evomaster.core.search.gene.utils.GeneUtils object HttpSemanticsOracle { @@ -166,7 +167,7 @@ object HttpSemanticsOracle { } val bodyBefore = resBefore.getBody() - val bodyModify = resModify.getBody() + val bodyModify = extractRequestBody(modify) val bodyAfter = resAfter.getBody() // if both are null/empty, no side-effect detected @@ -188,6 +189,12 @@ object HttpSemanticsOracle { return false } + private fun extractRequestBody(modify: RestCallAction): String? { + val bodyParam = modify.parameters.find { it is BodyParam } as BodyParam? + ?: return null + return bodyParam.getValueAsPrintableString(mode = GeneUtils.EscapeMode.JSON) + } + /** * Extract field names from the PUT/PATCH request body. * These are the fields that the client attempted to modify. From 81e48f2bd4504cf9c7295978000a6f57ba8413f6 Mon Sep 17 00:00:00 2001 From: Omur Date: Fri, 6 Mar 2026 14:03:51 +0300 Subject: [PATCH 07/13] updated algorithm --- .../FailModificationApplication.kt | 92 --------- .../base/FailModificationApplication.kt | 148 +++++++++++++ .../FailModificationForbiddenApplication.kt | 78 +++++++ .../FailModificationController.kt | 1 + .../FailModificationForbiddenController.kt | 21 ++ .../FailModificationUnauthorizedController.kt | 21 ++ .../FailModificationEMTest.kt | 9 +- .../FailModificationForbiddenEMTest.kt | 46 +++++ .../rest/oracle/HttpSemanticsOracle.kt | 44 +++- .../rest/service/HttpSemanticsService.kt | 195 ++++++++++++------ .../service/fitness/AbstractRestFitness.kt | 11 +- 11 files changed, 496 insertions(+), 170 deletions(-) delete mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationApplication.kt create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/base/FailModificationApplication.kt create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/forbidden/FailModificationForbiddenApplication.kt create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationForbiddenController.kt create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationUnauthorizedController.kt create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationForbiddenEMTest.kt diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationApplication.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationApplication.kt deleted file mode 100644 index afeb1edb24..0000000000 --- a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationApplication.kt +++ /dev/null @@ -1,92 +0,0 @@ -package com.foo.rest.examples.spring.openapi.v3.httporacle.failmodification - -import org.springframework.boot.SpringApplication -import org.springframework.boot.autoconfigure.SpringBootApplication -import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration -import org.springframework.http.ResponseEntity -import org.springframework.web.bind.annotation.* - - -@SpringBootApplication(exclude = [SecurityAutoConfiguration::class]) -@RequestMapping(path = ["/api/resources"]) -@RestController -open class FailModificationApplication { - - companion object { - @JvmStatic - fun main(args: Array) { - SpringApplication.run(FailModificationApplication::class.java, *args) - } - - private val data = mutableMapOf() - - fun reset(){ - data.clear() - } - } - - data class ResourceData( - var name: String, - var value: Int - ) - - data class UpdateRequest( - val name: String?, - val value: Int? - ) - - - @PostMapping - open fun create(@RequestBody body: ResourceData): ResponseEntity { - val id = data.size + 1 - data[id] = body.copy() - return ResponseEntity.status(201).body(data[id]) - } - - @GetMapping(path = ["/{id}"]) - open fun get(@PathVariable("id") id: Int): ResponseEntity { - val resource = data[id] - ?: return ResponseEntity.status(404).build() - return ResponseEntity.status(200).body(resource) - } - - @PutMapping(path = ["/{id}"]) - open fun put( - @PathVariable("id") id: Int, - @RequestBody body: UpdateRequest - ): ResponseEntity { - - val resource = data[id] - ?: return ResponseEntity.status(404).build() - - // bug: modifies data even though it will return 4xx - if(body.name != null) { - resource.name = body.name - } - if(body.value != null) { - resource.value = body.value - } - - // returns 400 Bad Request, but the data was already modified above - return ResponseEntity.status(400).body("Invalid request") - } - - @PatchMapping(path = ["/{id}"]) - open fun patch( - @PathVariable("id") id: Int, - @RequestBody body: UpdateRequest - ): ResponseEntity { - - val resource = data[id] - ?: return ResponseEntity.status(404).build() - - // correct: validation first, reject without modifying - if(body.name == null && body.value == null) { - return ResponseEntity.status(400).body("No fields to update") - } - - // correct: does NOT modify data, just returns 4xx - return ResponseEntity.status(403).body("Forbidden") - } - -} diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/base/FailModificationApplication.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/base/FailModificationApplication.kt new file mode 100644 index 0000000000..1ebc3143e5 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/base/FailModificationApplication.kt @@ -0,0 +1,148 @@ +package com.foo.rest.examples.spring.openapi.v3.httporacle.failmodification.base + +import org.springframework.boot.SpringApplication +import org.springframework.boot.autoconfigure.SpringBootApplication +import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PatchMapping +import org.springframework.web.bind.annotation.PathVariable +import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.PutMapping +import org.springframework.web.bind.annotation.RequestBody +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController + +@SpringBootApplication(exclude = [SecurityAutoConfiguration::class]) +@RequestMapping(path = ["/api/resources"]) +@RestController +open class FailModificationApplication { + + companion object { + @JvmStatic + fun main(args: Array) { + SpringApplication.run(FailModificationApplication::class.java, *args) + } + + private val data = mutableMapOf() + private val dataAlreadyExists = mutableMapOf() + + fun reset(){ + data.clear() + dataAlreadyExists.clear() + dataAlreadyExists[0] = ResourceData("existing", 42) + } + } + + data class ResourceData( + var name: String, + var value: Int + ) + + data class UpdateRequest( + val name: String, + val value: Int + ) + + + @PostMapping(path = ["/empty"]) + open fun create(@RequestBody body: ResourceData): ResponseEntity { + val id = data.size + 1 + data[id] = body.copy() + return ResponseEntity.status(201).body(data[id]) + } + + @GetMapping(path = ["/empty/{id}"]) + open fun get(@PathVariable("id") id: Int): ResponseEntity { + val resource = data[id] + ?: return ResponseEntity.status(404).build() + return ResponseEntity.status(200).body(resource) + } + + @PutMapping(path = ["/empty/{id}"]) + open fun put( + @PathVariable("id") id: Int, + @RequestBody body: UpdateRequest + ): ResponseEntity { + + val resource = data[id] + ?: return ResponseEntity.status(404).build() + + // bug: modifies data even though it will return 4xx + if(body.name != null) { + resource.name = body.name + } + if(body.value != null) { + resource.value = body.value + } + + // returns 400 Bad Request, but the data was already modified above + return ResponseEntity.status(400).body("Invalid request") + } + + @PatchMapping(path = ["/empty/{id}"]) + open fun patch( + @PathVariable("id") id: Int, + @RequestBody body: UpdateRequest + ): ResponseEntity { + + val resource = data[id] + ?: return ResponseEntity.status(404).build() + + // correct: validation first, reject without modifying + if(body.name == null && body.value == null) { + return ResponseEntity.status(400).body("No fields to update") + } + + // correct: does NOT modify data, just returns 4xx + return ResponseEntity.status(403).body("Forbidden") + } + + // pre-populated resource to test that it is not modified by failed PUT + + @PostMapping(path = ["/notempty"]) + open fun createnotempty(@RequestBody body: ResourceData): ResponseEntity { + val id = dataAlreadyExists.size + 1 + data[id] = body.copy() + return ResponseEntity.status(201).body(data[id]) + } + + @GetMapping(path = ["/notempty/{id}"]) + open fun getnotempty(@PathVariable("id") id: Int): ResponseEntity { + val resource = dataAlreadyExists[id] + ?: return ResponseEntity.status(404).build() + return ResponseEntity.status(200).body(resource) + } + + @PutMapping(path = ["/notempty/{id}"]) + open fun putnotempty( + @PathVariable("id") id: Int, + @RequestBody body: UpdateRequest + ): ResponseEntity { + + val resource = dataAlreadyExists[id] + ?: return ResponseEntity.status(404).build() + + resource.name = body.name + resource.value = body.value + + // returns 400 Bad Request, but the data was already modified above + return ResponseEntity.status(400).body("Invalid request") + } + + @PatchMapping(path = ["/notempty/{id}"]) + open fun patchnotempty( + @PathVariable("id") id: Int, + @RequestBody body: UpdateRequest + ): ResponseEntity { + + val resource = dataAlreadyExists[id] + ?: return ResponseEntity.status(404).build() + + // correct: validation first, reject without modifying + return ResponseEntity.status(400).body("No fields to update") + + // correct: does NOT modify data, just returns 4xx + return ResponseEntity.status(403).body("Forbidden") + } +} diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/forbidden/FailModificationForbiddenApplication.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/forbidden/FailModificationForbiddenApplication.kt new file mode 100644 index 0000000000..d6944d0da4 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/forbidden/FailModificationForbiddenApplication.kt @@ -0,0 +1,78 @@ +package com.foo.rest.examples.spring.openapi.v3.httporacle.failmodification.forbidden + +import org.springframework.boot.SpringApplication +import org.springframework.boot.autoconfigure.SpringBootApplication +import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.* + + +@SpringBootApplication(exclude = [SecurityAutoConfiguration::class]) +@RequestMapping("/api/resources") +@RestController +open class FailModificationForbiddenApplication { + + companion object { + @JvmStatic + fun main(args: Array) { + SpringApplication.run(FailModificationForbiddenApplication::class.java, *args) + } + + val USERS = setOf("FOO", "BAR") + + private val data = mutableMapOf() + + fun reset() { + data.clear() + } + } + + data class ResourceData( + val name: String, + var value: String + ) + + data class UpdateRequest( + val value: String + ) + + private fun isValidUser(auth: String?) = auth != null && USERS.contains(auth) + + @PostMapping + open fun create( + @RequestHeader(value = "Authorization", required = false) auth: String?, + @RequestBody body: UpdateRequest + ): ResponseEntity { + if (!isValidUser(auth)) return ResponseEntity.status(401).build() + val id = data.size + 1 + data[id] = ResourceData(name = auth!!, value = body.value) + return ResponseEntity.status(201).body(data[id]) + } + + @GetMapping("/{id}") + open fun get( + @RequestHeader(value = "Authorization", required = false) auth: String?, + @PathVariable("id") id: Int + ): ResponseEntity { + if (!isValidUser(auth)) return ResponseEntity.status(401).build() + val resource = data[id] ?: return ResponseEntity.status(404).build() + return ResponseEntity.status(200).body(resource) + } + + @PatchMapping("/{id}") + open fun patch( + @RequestHeader(value = "Authorization", required = false) auth: String?, + @PathVariable("id") id: Int, + @RequestBody body: UpdateRequest + ): ResponseEntity { + if (!isValidUser(auth)) return ResponseEntity.status(403).build() + + val resource = data[id] ?: return ResponseEntity.status(404).build() + + // BUG: side-effect before ownership check + resource.value = body.value + + if (resource.name != auth) return ResponseEntity.status(403).build() + return ResponseEntity.status(200).build() + } +} diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationController.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationController.kt index a507132fb1..d37a1ff075 100644 --- a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationController.kt +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationController.kt @@ -1,6 +1,7 @@ package com.foo.rest.examples.spring.openapi.v3.httporacle.failmodification import com.foo.rest.examples.spring.openapi.v3.SpringController +import com.foo.rest.examples.spring.openapi.v3.httporacle.failmodification.base.FailModificationApplication class FailModificationController: SpringController(FailModificationApplication::class.java){ diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationForbiddenController.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationForbiddenController.kt new file mode 100644 index 0000000000..9eef90b31c --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationForbiddenController.kt @@ -0,0 +1,21 @@ +package com.foo.rest.examples.spring.openapi.v3.httporacle.failmodification + +import com.foo.rest.examples.spring.openapi.v3.SpringController +import com.foo.rest.examples.spring.openapi.v3.httporacle.failmodification.forbidden.FailModificationForbiddenApplication +import org.evomaster.client.java.controller.AuthUtils +import org.evomaster.client.java.controller.api.dto.auth.AuthenticationDto + + +class FailModificationForbiddenController: SpringController(FailModificationForbiddenApplication::class.java){ + + override fun getInfoForAuthentication(): List { + return listOf( + AuthUtils.getForAuthorizationHeader("FOO","FOO"), + AuthUtils.getForAuthorizationHeader("BAR","BAR"), + ) + } + + override fun resetStateOfSUT() { + FailModificationForbiddenApplication.reset() + } +} diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationUnauthorizedController.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationUnauthorizedController.kt new file mode 100644 index 0000000000..1230bf44ce --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationUnauthorizedController.kt @@ -0,0 +1,21 @@ +package com.foo.rest.examples.spring.openapi.v3.httporacle.failmodification + +import com.foo.rest.examples.spring.openapi.v3.SpringController +import com.foo.rest.examples.spring.openapi.v3.httporacle.failmodification.unauthorized.FailModificationUnauthApplication +import org.evomaster.client.java.controller.AuthUtils +import org.evomaster.client.java.controller.api.dto.auth.AuthenticationDto + + +class FailModificationUnauthorizedController: SpringController(FailModificationUnauthApplication::class.java){ + + override fun getInfoForAuthentication(): List { + return listOf( + AuthUtils.getForAuthorizationHeader("FOO","FOO"), + AuthUtils.getForAuthorizationHeader("BAR","BAR"), + ) + } + + override fun resetStateOfSUT() { + FailModificationUnauthApplication.reset() + } +} diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationEMTest.kt index 5fa5a8b617..cb48e1d0a9 100644 --- a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationEMTest.kt +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationEMTest.kt @@ -26,7 +26,7 @@ class FailModificationEMTest : SpringTestBase(){ runTestHandlingFlakyAndCompilation( "FailedModificationEM", - 500 + 2000 ) { args: MutableList -> setOption(args, "security", "false") @@ -38,9 +38,10 @@ class FailModificationEMTest : SpringTestBase(){ assertTrue(solution.individuals.size >= 1) - val faults = DetectedFaultUtils.getDetectedFaultCategories(solution) - assertEquals(1, faults.size) - assertEquals(ExperimentalFaultCategory.HTTP_SIDE_EFFECTS_FAILED_MODIFICATION, faults.first()) + val faults = DetectedFaultUtils.getDetectedFaults(solution) + + assertEquals(2, faults.size) + assertEquals(ExperimentalFaultCategory.HTTP_SIDE_EFFECTS_FAILED_MODIFICATION, faults.first().category) } } } diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationForbiddenEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationForbiddenEMTest.kt new file mode 100644 index 0000000000..9e82c49a86 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationForbiddenEMTest.kt @@ -0,0 +1,46 @@ +package org.evomaster.e2etests.spring.openapi.v3.httporacle.failmodification + +import com.foo.rest.examples.spring.openapi.v3.httporacle.failmodification.FailModificationForbiddenController +import org.evomaster.core.problem.enterprise.DetectedFaultUtils +import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory +import org.evomaster.e2etests.spring.openapi.v3.SpringTestBase +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test + +class FailModificationForbiddenEMTest : SpringTestBase(){ + + companion object { + @BeforeAll + @JvmStatic + fun init() { + initClass(FailModificationForbiddenController()) + } + } + + + @Test + fun testRunEM() { + + runTestHandlingFlakyAndCompilation( + "FailedModificationForbiddenEM", + 2000 + ) { args: MutableList -> + + setOption(args, "security", "false") + setOption(args, "schemaOracles", "false") + setOption(args, "httpOracles", "true") + setOption(args, "useExperimentalOracles", "true") + + val solution = initAndRun(args) + + assertTrue(solution.individuals.size >= 1) + + val faults = DetectedFaultUtils.getDetectedFaults(solution) + + assertEquals(1, faults.size) + assertEquals(ExperimentalFaultCategory.HTTP_SIDE_EFFECTS_FAILED_MODIFICATION, faults.first().category) + } + } +} diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracle.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracle.kt index 68cbfff517..438472b383 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracle.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracle.kt @@ -139,8 +139,8 @@ object HttpSemanticsOracle { return false } - // auth should be consistent - if(before.auth.isDifferentFrom(modify.auth) || after.auth.isDifferentFrom(modify.auth)) { + // the two GETs must use the same auth so the state comparison is meaningful. + if(before.auth.isDifferentFrom(after.auth)) { return false } @@ -195,6 +195,44 @@ object HttpSemanticsOracle { return bodyParam.getValueAsPrintableString(mode = GeneUtils.EscapeMode.JSON) } + /** + * Checks the special K==404 side-effect pattern: + * + * GET /path → 404 (resource does not exist before the call) + * PUT|PATCH /path → 404 (failed modification - resource still not found) + * GET /path → ??? (should STILL be 404; anything else is a side-effect) + */ + fun hasSideEffectIn404Modification( + individual: RestIndividual, + actionResults: List + ): Boolean { + + if(individual.size() < 3) return false + + val actions = individual.seeMainExecutableActions() + val before = actions[actions.size - 3] // GET (should be 404) + val modify = actions[actions.size - 2] // PUT or PATCH (should be 404) + val after = actions[actions.size - 1] // GET (oracle target) + + if(before.verb != HttpVerb.GET) return false + if(modify.verb != HttpVerb.PUT && modify.verb != HttpVerb.PATCH) return false + if(after.verb != HttpVerb.GET) return false + + if(!before.usingSameResolvedPath(modify) || !after.usingSameResolvedPath(modify)) return false + + val resBefore = actionResults.find { it.sourceLocalId == before.getLocalId() } as RestCallResult? + ?: return false + val resModify = actionResults.find { it.sourceLocalId == modify.getLocalId() } as RestCallResult? + ?: return false + val resAfter = actionResults.find { it.sourceLocalId == after.getLocalId() } as RestCallResult? + ?: return false + + if(resBefore.getStatusCode() != 404) return false + if(resModify.getStatusCode() != 404) return false + + return resAfter.getStatusCode() != 404 + } + /** * Extract field names from the PUT/PATCH request body. * These are the fields that the client attempted to modify. @@ -236,7 +274,6 @@ object HttpSemanticsOracle { val jsonModify = JsonParser.parseString(bodyModify) if(!jsonBefore.isJsonObject || !jsonAfter.isJsonObject || !jsonModify.isJsonObject){ - // not JSON objects, fallback to full comparison return false } @@ -256,7 +293,6 @@ object HttpSemanticsOracle { return false } catch (e: Exception) { - // JSON parsing failed, fallback to full comparison return false } } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt index a0eafb4f06..6b7269c400 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt @@ -2,12 +2,15 @@ package org.evomaster.core.problem.rest.service import com.google.inject.Inject import org.evomaster.core.problem.enterprise.SampleType +import org.evomaster.core.problem.httpws.auth.HttpWsAuthenticationInfo +import org.evomaster.core.problem.httpws.auth.HttpWsNoAuth import org.evomaster.core.problem.rest.* import org.evomaster.core.problem.rest.builder.RestIndividualSelectorUtils import org.evomaster.core.problem.rest.data.HttpVerb import org.evomaster.core.problem.rest.data.RestCallAction import org.evomaster.core.problem.rest.data.RestCallResult import org.evomaster.core.problem.rest.data.RestIndividual +import org.evomaster.core.problem.rest.data.RestPath import org.evomaster.core.problem.rest.service.fitness.RestFitness import org.evomaster.core.problem.rest.service.sampler.AbstractRestSampler import org.evomaster.core.search.EvaluatedIndividual @@ -204,19 +207,7 @@ class HttpSemanticsService { } } - /** - * Checking bugs like: - * POST|PUT /X 2xx (create resource) - * GET /X 2xx (before state) - * PUT|PATCH /X 4xx (failed modification) - * GET /X 2xx (after state - should be same as before) - * - * If a PUT/PATCH fails with 4xx, it should have no side-effects. - * A GET before and after should return the same resource state. - * - * NOTE: When comparing states, non-deterministic fields (e.g. timestamps, - * generated IDs/UUIDs, version counters) must be excluded from the comparison. - */ + private fun sideEffectsOfFailedModification() { val verbs = listOf(HttpVerb.PUT, HttpVerb.PATCH) @@ -227,76 +218,148 @@ class HttpSemanticsService { modifyOperations.forEach { modOp -> - // check that a GET definition exists for this same path val getDef = actionDefinitions.find { it.verb == HttpVerb.GET && it.path == modOp.path } ?: return@forEach - // find individuals that have a 4xx PUT/PATCH on this path - val failedModifyIndividuals = RestIndividualSelectorUtils.findIndividuals( + val failedModifyEvals = RestIndividualSelectorUtils.findIndividuals( individualsInSolution, verb, modOp.path, statusGroup = StatusGroup.G_4xx ) - if (failedModifyIndividuals.isEmpty()) { - return@forEach - } + if (failedModifyEvals.isEmpty()) return@forEach + + // gather distinct 4xx status codes observed on this verb+path + val distinctCodes = failedModifyEvals.flatMap { ei -> + ei.evaluatedMainActions().mapNotNull { ea -> + val a = ea.action as? RestCallAction ?: return@mapNotNull null + val r = ea.result as? RestCallResult ?: return@mapNotNull null + if (a.verb == verb && a.path.isEquivalent(modOp.path) + && StatusGroup.G_4xx.isInGroup(r.getStatusCode()) + ) r.getStatusCode() else null + } + }.distinct() - // among those, find one that also has a successful creation step - // (POST 2xx on parent path, or PUT 201 on same path) - val parentPath = if (!modOp.path.isRoot()) modOp.path.parentPath() else null - - val withCreation = failedModifyIndividuals.filter { ind -> - ind.evaluatedMainActions().any { ea -> - val action = ea.action as RestCallAction - val result = ea.result as RestCallResult - (parentPath != null - && action.verb == HttpVerb.POST - && action.path.isEquivalent(parentPath) - && StatusGroup.G_2xx.isInGroup(result.getStatusCode())) - || - (action.verb == HttpVerb.PUT - && action.path.isEquivalent(modOp.path) - && result.getStatusCode() == 201) + for (k in distinctCodes) { + when (k) { + 401, 403 -> handle401Or403SideEffect(verb, k, modOp.path) + else -> addGetAroundFailedModification(verb, k, modOp.path, getDef, failedModifyEvals) } } - if (withCreation.isEmpty()) { - return@forEach - } - - val selected = withCreation.minBy { it.individual.size() } + } + } + } - // slice up to the 4xx PUT/PATCH - val failedIndex = RestIndividualSelectorUtils.findIndexOfAction( - selected, verb, modOp.path, statusGroup = StatusGroup.G_4xx - ) - if (failedIndex < 0) { - return@forEach + /** + * Handles K==401 and K==403. + * + * 1. Find T — smallest individual ending with a clean GET 2xx on [path] + * 2. Find a 2xx PUT/PATCH action as the body template or fall back to the K action if no 2xx exists + * 3. Copy the template and override auth: + * K==401 → NoAuth (expected to trigger 401) + * K==403 → a different authenticated user + * 4. Append the modified PUT/PATCH after the GET in T, then append another GET + */ + private fun handle401Or403SideEffect(verb: HttpVerb, k: Int, path: RestPath) { + + // T: smallest clean individual ending with GET 2xx (no prior PUT/PATCH on same path) + val T = RestIndividualSelectorUtils.findAndSlice( + individualsInSolution, HttpVerb.GET, path, statusGroup = StatusGroup.G_2xx + ).filter { ind -> + val actions = ind.seeMainExecutableActions() + actions.subList(0, actions.size - 1).none { + (it.verb == HttpVerb.PUT || it.verb == HttpVerb.PATCH) && it.path.isEquivalent(path) + } + }.minByOrNull { it.size() } ?: return + + // find a 2xx PUT/PATCH action to use as the body template + // 401/403 action itself if no 2xx exists + val successAction = RestIndividualSelectorUtils.findIndividuals( + individualsInSolution, verb, path, statusGroup = StatusGroup.G_2xx + ).flatMap { ei -> + ei.evaluatedMainActions().mapNotNull { ea -> + val a = ea.action as? RestCallAction ?: return@mapNotNull null + val r = ea.result as? RestCallResult ?: return@mapNotNull null + if (a.verb == verb && a.path.isEquivalent(path) && StatusGroup.G_2xx.isInGroup(r.getStatusCode())) + a else null + } + }.firstOrNull() + ?: RestIndividualSelectorUtils.findIndividuals( + individualsInSolution, verb, path, status = k + ).flatMap { ei -> + ei.evaluatedMainActions().mapNotNull { ea -> + (ea.action as? RestCallAction) + ?.takeIf { it.verb == verb && it.path.isEquivalent(path) } } + }.firstOrNull() + ?: return + + val ind = T.copy() as RestIndividual + val getAction = ind.seeMainExecutableActions().last() // the GET 2xx at the end of T + + val modifyCopy = successAction.copy() as RestCallAction + modifyCopy.resetLocalIdRecursively() + modifyCopy.forceNewTaints() + modifyCopy.bindToSamePathResolution(getAction) + + when (k) { + 401 -> modifyCopy.auth = HttpWsNoAuth() + 403 -> { + val otherAuths = sampler.authentications + .getAllOthers(getAction.auth.name, HttpWsAuthenticationInfo::class.java) + if (otherAuths.isEmpty()) return + modifyCopy.auth = otherAuths.first() + } + } - val ind = RestIndividualBuilder.sliceAllCallsInIndividualAfterAction( - selected.individual, failedIndex - ) - - val actions = ind.seeMainExecutableActions() - val last = actions[actions.size - 1] // the 4xx PUT/PATCH + ind.addMainActionInEmptyEnterpriseGroup(-1, modifyCopy) - // add GET before the failed PUT/PATCH - val getBefore = getDef.copy() as RestCallAction - getBefore.doInitialize(randomness) - getBefore.forceNewTaints() - getBefore.bindToSamePathResolution(last) - getBefore.auth = last.auth - ind.addMainActionInEmptyEnterpriseGroup(actions.size - 1, getBefore) + val getAfter = getAction.copy() as RestCallAction + getAfter.resetLocalIdRecursively() + ind.addMainActionInEmptyEnterpriseGroup(-1, getAfter) - // add GET after the failed PUT/PATCH - val getAfter = getBefore.copy() as RestCallAction - getAfter.resetLocalIdRecursively() - ind.addMainActionInEmptyEnterpriseGroup(-1, getAfter) + prepareEvaluateAndSave(ind) + } - prepareEvaluateAndSave(ind) - } - } + /** + * Takes the smallest individual in [candidates] where [verb] on [path] returned [k], + * slices it at that action, then inserts a GET immediately before it and appends + * another GET immediately after it — both on the same resolved path and with the + * same auth as the PUT/PATCH: + * + * GET /path (same auth as PUT/PATCH) + * PUT|PATCH /path [k] + * GET /path (same auth) + */ + private fun addGetAroundFailedModification( + verb: HttpVerb, + k: Int, + path: RestPath, + getDef: RestCallAction, + candidates: List> + ) { + val kEval = RestIndividualSelectorUtils.findIndividuals(candidates, verb, path, status = k) + .minByOrNull { it.individual.size() } ?: return + + val ind = RestIndividualBuilder.sliceAllCallsInIndividualAfterAction(kEval, verb, path, status = k) + + val actions = ind.seeMainExecutableActions() + val last = actions.last() // the PUT/PATCH [k] + + // insert GET before the PUT/PATCH + val getBefore = getDef.copy() as RestCallAction + getBefore.doInitialize(randomness) + getBefore.forceNewTaints() + getBefore.bindToSamePathResolution(last) + getBefore.auth = last.auth + ind.addMainActionInEmptyEnterpriseGroup(actions.size - 1, getBefore) + + // append GET after the PUT/PATCH + val getAfter = getBefore.copy() as RestCallAction + getAfter.resetLocalIdRecursively() + ind.addMainActionInEmptyEnterpriseGroup(-1, getAfter) + + prepareEvaluateAndSave(ind) } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt index 67222e086b..f8d1e3e6b2 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt @@ -1229,8 +1229,12 @@ abstract class AbstractRestFitness : HttpWsFitness() { actionResults: List, fv: FitnessValue ) { - val issues = HttpSemanticsOracle.hasSideEffectFailedModification(individual,actionResults) - if(!issues){ + // covers normal / 401 / 403 cases: GET 2xx → PUT|PATCH 4xx → GET 2xx (fields unchanged) + val hasSideEffect = HttpSemanticsOracle.hasSideEffectFailedModification(individual, actionResults) + // covers the 404 special case: GET 404 → PUT|PATCH 404 → GET (must still be 404) + val hasSideEffect404 = HttpSemanticsOracle.hasSideEffectIn404Modification(individual, actionResults) + + if (!hasSideEffect && !hasSideEffect404) { return } @@ -1239,8 +1243,7 @@ abstract class AbstractRestFitness : HttpWsFitness() { }.last() val category = ExperimentalFaultCategory.HTTP_SIDE_EFFECTS_FAILED_MODIFICATION - val scenarioId = idMapper.handleLocalTarget(idMapper.getFaultDescriptiveId(category, putOrPatch.getName()) - ) + val scenarioId = idMapper.handleLocalTarget(idMapper.getFaultDescriptiveId(category, putOrPatch.getName())) fv.updateTarget(scenarioId, 1.0, individual.seeMainExecutableActions().lastIndex) val ar = actionResults.find { it.sourceLocalId == putOrPatch.getLocalId() } as RestCallResult? From b59881c030f702cf0e9a510229792df9b77a55a5 Mon Sep 17 00:00:00 2001 From: Omur Date: Fri, 6 Mar 2026 15:10:48 +0300 Subject: [PATCH 08/13] not found detection --- .../FailModificationNotFoundApplication.kt | 52 +++++++++++++++++++ .../FailModificationNotFoundController.kt | 11 ++++ .../FailModificationUnauthorizedController.kt | 21 -------- .../FailModificationNotFoundEMTest.kt | 47 +++++++++++++++++ 4 files changed, 110 insertions(+), 21 deletions(-) create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/notfound/FailModificationNotFoundApplication.kt create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationNotFoundController.kt delete mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationUnauthorizedController.kt create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationNotFoundEMTest.kt diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/notfound/FailModificationNotFoundApplication.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/notfound/FailModificationNotFoundApplication.kt new file mode 100644 index 0000000000..c355556243 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/notfound/FailModificationNotFoundApplication.kt @@ -0,0 +1,52 @@ +package com.foo.rest.examples.spring.openapi.v3.httporacle.failmodification.notfound + +import org.springframework.boot.SpringApplication +import org.springframework.boot.autoconfigure.SpringBootApplication +import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.* + + +@SpringBootApplication(exclude = [SecurityAutoConfiguration::class]) +@RequestMapping("/api/resources") +@RestController +open class FailModificationNotFoundApplication { + + companion object { + @JvmStatic + fun main(args: Array) { + SpringApplication.run(FailModificationNotFoundApplication::class.java, *args) + } + + private val data = mutableMapOf() + + fun reset() { + data.clear() + } + } + + data class ResourceData(val name: String, val value: Int) + + data class UpdateRequest(val name: String, val value: Int) + + + @GetMapping("/{id}") + open fun get(@PathVariable("id") id: Int): ResponseEntity { + val resource = data[id] ?: return ResponseEntity.status(404).build() + return ResponseEntity.ok(resource) + } + + @PutMapping("/{id}") + open fun put( + @PathVariable("id") id: Int, + @RequestBody body: UpdateRequest + ): ResponseEntity { + if (!data.containsKey(id)) { + // BUG: stores the resource before returning 404 + data[id] = ResourceData(body.name, body.value) + return ResponseEntity.status(404).build() + } + data[id] = ResourceData(body.name, body.value) + return ResponseEntity.ok().build() + } +} diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationNotFoundController.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationNotFoundController.kt new file mode 100644 index 0000000000..1c3012deaf --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationNotFoundController.kt @@ -0,0 +1,11 @@ +package com.foo.rest.examples.spring.openapi.v3.httporacle.failmodification + +import com.foo.rest.examples.spring.openapi.v3.SpringController +import com.foo.rest.examples.spring.openapi.v3.httporacle.failmodification.notfound.FailModificationNotFoundApplication + + +class FailModificationNotFoundController: SpringController(FailModificationNotFoundApplication::class.java){ + override fun resetStateOfSUT() { + FailModificationNotFoundApplication.reset() + } +} diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationUnauthorizedController.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationUnauthorizedController.kt deleted file mode 100644 index 1230bf44ce..0000000000 --- a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationUnauthorizedController.kt +++ /dev/null @@ -1,21 +0,0 @@ -package com.foo.rest.examples.spring.openapi.v3.httporacle.failmodification - -import com.foo.rest.examples.spring.openapi.v3.SpringController -import com.foo.rest.examples.spring.openapi.v3.httporacle.failmodification.unauthorized.FailModificationUnauthApplication -import org.evomaster.client.java.controller.AuthUtils -import org.evomaster.client.java.controller.api.dto.auth.AuthenticationDto - - -class FailModificationUnauthorizedController: SpringController(FailModificationUnauthApplication::class.java){ - - override fun getInfoForAuthentication(): List { - return listOf( - AuthUtils.getForAuthorizationHeader("FOO","FOO"), - AuthUtils.getForAuthorizationHeader("BAR","BAR"), - ) - } - - override fun resetStateOfSUT() { - FailModificationUnauthApplication.reset() - } -} diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationNotFoundEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationNotFoundEMTest.kt new file mode 100644 index 0000000000..ea0dd7e999 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationNotFoundEMTest.kt @@ -0,0 +1,47 @@ +package org.evomaster.e2etests.spring.openapi.v3.httporacle.failmodification + +import com.foo.rest.examples.spring.openapi.v3.httporacle.failmodification.FailModificationForbiddenController +import com.foo.rest.examples.spring.openapi.v3.httporacle.failmodification.FailModificationNotFoundController +import org.evomaster.core.problem.enterprise.DetectedFaultUtils +import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory +import org.evomaster.e2etests.spring.openapi.v3.SpringTestBase +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test + +class FailModificationNotFoundEMTest : SpringTestBase(){ + + companion object { + @BeforeAll + @JvmStatic + fun init() { + initClass(FailModificationNotFoundController()) + } + } + + + @Test + fun testRunEM() { + + runTestHandlingFlakyAndCompilation( + "FailedModificationNotFoundEM", + 2000 + ) { args: MutableList -> + + setOption(args, "security", "false") + setOption(args, "schemaOracles", "false") + setOption(args, "httpOracles", "true") + setOption(args, "useExperimentalOracles", "true") + + val solution = initAndRun(args) + + assertTrue(solution.individuals.size >= 1) + + val faults = DetectedFaultUtils.getDetectedFaults(solution) + + assertEquals(1, faults.size) + assertEquals(ExperimentalFaultCategory.HTTP_SIDE_EFFECTS_FAILED_MODIFICATION, faults.first().category) + } + } +} From cc9850b2a91a2757181e23c1259602bbae8f1f12 Mon Sep 17 00:00:00 2001 From: Omur Date: Fri, 6 Mar 2026 16:23:41 +0300 Subject: [PATCH 09/13] failed modification 403 case --- .../FailModificationForbiddenApplication.kt | 2 +- .../FailModificationEMTest.kt | 1 - .../FailModificationForbiddenEMTest.kt | 4 ++-- .../FailModificationNotFoundEMTest.kt | 7 ++---- .../main/kotlin/org/evomaster/core/Main.kt | 2 +- .../rest/service/HttpSemanticsService.kt | 24 +++++++------------ 6 files changed, 15 insertions(+), 25 deletions(-) diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/forbidden/FailModificationForbiddenApplication.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/forbidden/FailModificationForbiddenApplication.kt index d6944d0da4..e7b7acb821 100644 --- a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/forbidden/FailModificationForbiddenApplication.kt +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/forbidden/FailModificationForbiddenApplication.kt @@ -65,7 +65,7 @@ open class FailModificationForbiddenApplication { @PathVariable("id") id: Int, @RequestBody body: UpdateRequest ): ResponseEntity { - if (!isValidUser(auth)) return ResponseEntity.status(403).build() + if (!isValidUser(auth)) return ResponseEntity.status(401).build() val resource = data[id] ?: return ResponseEntity.status(404).build() diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationEMTest.kt index cb48e1d0a9..f3127a2038 100644 --- a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationEMTest.kt +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationEMTest.kt @@ -29,7 +29,6 @@ class FailModificationEMTest : SpringTestBase(){ 2000 ) { args: MutableList -> - setOption(args, "security", "false") setOption(args, "schemaOracles", "false") setOption(args, "httpOracles", "true") setOption(args, "useExperimentalOracles", "true") diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationForbiddenEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationForbiddenEMTest.kt index 9e82c49a86..c405da8d8a 100644 --- a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationForbiddenEMTest.kt +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationForbiddenEMTest.kt @@ -25,10 +25,10 @@ class FailModificationForbiddenEMTest : SpringTestBase(){ runTestHandlingFlakyAndCompilation( "FailedModificationForbiddenEM", - 2000 + 4000 ) { args: MutableList -> - setOption(args, "security", "false") + setOption(args, "security", "true") setOption(args, "schemaOracles", "false") setOption(args, "httpOracles", "true") setOption(args, "useExperimentalOracles", "true") diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationNotFoundEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationNotFoundEMTest.kt index ea0dd7e999..c7fbd75d72 100644 --- a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationNotFoundEMTest.kt +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationNotFoundEMTest.kt @@ -29,7 +29,6 @@ class FailModificationNotFoundEMTest : SpringTestBase(){ 2000 ) { args: MutableList -> - setOption(args, "security", "false") setOption(args, "schemaOracles", "false") setOption(args, "httpOracles", "true") setOption(args, "useExperimentalOracles", "true") @@ -38,10 +37,8 @@ class FailModificationNotFoundEMTest : SpringTestBase(){ assertTrue(solution.individuals.size >= 1) - val faults = DetectedFaultUtils.getDetectedFaults(solution) - - assertEquals(1, faults.size) - assertEquals(ExperimentalFaultCategory.HTTP_SIDE_EFFECTS_FAILED_MODIFICATION, faults.first().category) + val faultsCategories = DetectedFaultUtils.getDetectedFaultCategories(solution) + assertTrue(ExperimentalFaultCategory.HTTP_SIDE_EFFECTS_FAILED_MODIFICATION in faultsCategories) } } } diff --git a/core/src/main/kotlin/org/evomaster/core/Main.kt b/core/src/main/kotlin/org/evomaster/core/Main.kt index c967064558..bc8852e928 100644 --- a/core/src/main/kotlin/org/evomaster/core/Main.kt +++ b/core/src/main/kotlin/org/evomaster/core/Main.kt @@ -262,8 +262,8 @@ class Main { logTimeSearchInfo(injector, config) //apply new phases - solution = phaseHttpOracle(injector, config, epc, solution) solution = phaseSecurity(injector, config, epc, solution) + solution = phaseHttpOracle(injector, config, epc, solution) solution = phaseFlaky(injector, config, epc, solution) epc.startWriteOutput() diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt index 6b7269c400..7067fa5744 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt @@ -262,6 +262,10 @@ class HttpSemanticsService { */ private fun handle401Or403SideEffect(verb: HttpVerb, k: Int, path: RestPath) { + // GET schema definition — needed to create the GET after via builder + val getDef = actionDefinitions.find { it.verb == HttpVerb.GET && it.path.isEquivalent(path) } + ?: return + // T: smallest clean individual ending with GET 2xx (no prior PUT/PATCH on same path) val T = RestIndividualSelectorUtils.findAndSlice( individualsInSolution, HttpVerb.GET, path, statusGroup = StatusGroup.G_2xx @@ -297,11 +301,8 @@ class HttpSemanticsService { val ind = T.copy() as RestIndividual val getAction = ind.seeMainExecutableActions().last() // the GET 2xx at the end of T - val modifyCopy = successAction.copy() as RestCallAction - modifyCopy.resetLocalIdRecursively() - modifyCopy.forceNewTaints() - modifyCopy.bindToSamePathResolution(getAction) - + // we override auth afterwards to achieve no-auth (401) or different-user (403) + val modifyCopy = builder.createBoundActionFor(successAction, getAction) when (k) { 401 -> modifyCopy.auth = HttpWsNoAuth() 403 -> { @@ -311,11 +312,9 @@ class HttpSemanticsService { modifyCopy.auth = otherAuths.first() } } - ind.addMainActionInEmptyEnterpriseGroup(-1, modifyCopy) - val getAfter = getAction.copy() as RestCallAction - getAfter.resetLocalIdRecursively() + val getAfter = builder.createBoundActionFor(getDef, getAction) ind.addMainActionInEmptyEnterpriseGroup(-1, getAfter) prepareEvaluateAndSave(ind) @@ -347,16 +346,11 @@ class HttpSemanticsService { val last = actions.last() // the PUT/PATCH [k] // insert GET before the PUT/PATCH - val getBefore = getDef.copy() as RestCallAction - getBefore.doInitialize(randomness) - getBefore.forceNewTaints() - getBefore.bindToSamePathResolution(last) - getBefore.auth = last.auth + val getBefore = builder.createBoundActionFor(getDef, last) ind.addMainActionInEmptyEnterpriseGroup(actions.size - 1, getBefore) // append GET after the PUT/PATCH - val getAfter = getBefore.copy() as RestCallAction - getAfter.resetLocalIdRecursively() + val getAfter = builder.createBoundActionFor(getDef, last) ind.addMainActionInEmptyEnterpriseGroup(-1, getAfter) prepareEvaluateAndSave(ind) From 7099e920b24fe743f7481193e71c3d4fe622d31d Mon Sep 17 00:00:00 2001 From: Omur Sahin Date: Mon, 9 Mar 2026 00:31:33 +0300 Subject: [PATCH 10/13] fix local id bug --- .../FailModificationForbiddenEMTest.kt | 2 +- .../rest/service/HttpSemanticsService.kt | 22 +++++++++++++++---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationForbiddenEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationForbiddenEMTest.kt index c405da8d8a..bad8c57b13 100644 --- a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationForbiddenEMTest.kt +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationForbiddenEMTest.kt @@ -25,7 +25,7 @@ class FailModificationForbiddenEMTest : SpringTestBase(){ runTestHandlingFlakyAndCompilation( "FailedModificationForbiddenEM", - 4000 + 3000 ) { args: MutableList -> setOption(args, "security", "true") diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt index 7067fa5744..bf56595ad7 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt @@ -299,10 +299,15 @@ class HttpSemanticsService { ?: return val ind = T.copy() as RestIndividual - val getAction = ind.seeMainExecutableActions().last() // the GET 2xx at the end of T + val getAction = ind.seeMainExecutableActions().last().copy() as RestCallAction // the GET 2xx at the end of T + val successCopy = successAction.copy() as RestCallAction + + successCopy.forceNewTaints() + successCopy.resetLocalIdRecursively() + // we override auth afterwards to achieve no-auth (401) or different-user (403) - val modifyCopy = builder.createBoundActionFor(successAction, getAction) + val modifyCopy = builder.createBoundActionFor(successCopy, getAction) when (k) { 401 -> modifyCopy.auth = HttpWsNoAuth() 403 -> { @@ -312,10 +317,19 @@ class HttpSemanticsService { modifyCopy.auth = otherAuths.first() } } - ind.addMainActionInEmptyEnterpriseGroup(-1, modifyCopy) + getAction.forceNewTaints() + getAction.resetLocalIdRecursively() val getAfter = builder.createBoundActionFor(getDef, getAction) - ind.addMainActionInEmptyEnterpriseGroup(-1, getAfter) + + + ind.addMainActionInEmptyEnterpriseGroup(action = modifyCopy) + ind.addMainActionInEmptyEnterpriseGroup(action = getAfter) + + + + ind.ensureFlattenedStructure() + org.evomaster.core.Lazy.assert { ind.verifyValidity(); true } prepareEvaluateAndSave(ind) } From 6cd618c6f23a60890db92d8439499d4018eea5b2 Mon Sep 17 00:00:00 2001 From: Omur Date: Fri, 27 Mar 2026 17:35:34 +0300 Subject: [PATCH 11/13] fixing failed modification --- .../base/FailModificationApplication.kt | 4 +- .../UrlencodedFailModificationApplication.kt | 94 +++++++ .../xml/XmlFailModificationApplication.kt | 155 +++++++++++ .../FailModificationURLEncodedController.kt | 12 + .../FailModificationXMLController.kt | 11 + .../FailModificationEMTest.kt | 2 +- .../FailModificationForbiddenEMTest.kt | 2 +- .../FailModificationNotFoundEMTest.kt | 2 +- .../URLEncodedFailModificationEMTest.kt | 45 ++++ .../XMLFailModificationEMTest.kt | 47 ++++ .../main/kotlin/org/evomaster/core/Main.kt | 9 +- .../core/output/formatter/OutputFormatter.kt | 68 ++++- .../rest/oracle/HttpSemanticsOracle.kt | 132 ++++++++-- .../rest/service/HttpSemanticsService.kt | 178 ++++++------- .../service/fitness/AbstractRestFitness.kt | 12 +- .../rest/oracle/HttpSemanticsOracleTest.kt | 242 ++++++++++++++++++ 16 files changed, 880 insertions(+), 135 deletions(-) create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/urlencoded/UrlencodedFailModificationApplication.kt create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/xml/XmlFailModificationApplication.kt create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationURLEncodedController.kt create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationXMLController.kt create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/URLEncodedFailModificationEMTest.kt create mode 100644 core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/XMLFailModificationEMTest.kt diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/base/FailModificationApplication.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/base/FailModificationApplication.kt index 1ebc3143e5..b5b69a6f68 100644 --- a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/base/FailModificationApplication.kt +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/base/FailModificationApplication.kt @@ -103,8 +103,8 @@ open class FailModificationApplication { @PostMapping(path = ["/notempty"]) open fun createnotempty(@RequestBody body: ResourceData): ResponseEntity { val id = dataAlreadyExists.size + 1 - data[id] = body.copy() - return ResponseEntity.status(201).body(data[id]) + dataAlreadyExists[id] = body.copy() + return ResponseEntity.status(201).body(dataAlreadyExists[id]) } @GetMapping(path = ["/notempty/{id}"]) diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/urlencoded/UrlencodedFailModificationApplication.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/urlencoded/UrlencodedFailModificationApplication.kt new file mode 100644 index 0000000000..7f574b0883 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/urlencoded/UrlencodedFailModificationApplication.kt @@ -0,0 +1,94 @@ +package com.foo.rest.examples.spring.openapi.v3.httporacle.failmodification.urlencoded + +import org.springframework.boot.SpringApplication +import org.springframework.boot.autoconfigure.SpringBootApplication +import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PatchMapping +import org.springframework.web.bind.annotation.PathVariable +import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.PutMapping +import org.springframework.web.bind.WebDataBinder +import org.springframework.web.bind.annotation.InitBinder +import org.springframework.web.bind.annotation.ModelAttribute +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController +import java.beans.PropertyEditorSupport + +@SpringBootApplication(exclude = [SecurityAutoConfiguration::class]) +@RequestMapping(path = ["/api/resources"]) +@RestController +open class UrlencodedFailModificationApplication { + + companion object { + @JvmStatic + fun main(args: Array) { + SpringApplication.run(UrlencodedFailModificationApplication::class.java, *args) + } + + private val dataAlreadyExists = mutableMapOf() + + fun reset(){ + dataAlreadyExists.clear() + dataAlreadyExists[0] = ResourceData("existing", 42) + } + } + + open class ResourceData( + var name: String = "", + var value: Int = 0 + ) + + open class UpdateRequest( + var name: String = "", + var value: Int = 0 + ) + + + @PostMapping(path = ["/notempty"], consumes = ["application/x-www-form-urlencoded"], produces = ["application/json"]) + open fun createnotempty(@ModelAttribute body: ResourceData): ResponseEntity { + val id = dataAlreadyExists.size + 1 + dataAlreadyExists[id] = ResourceData(body.name, body.value) + return ResponseEntity.status(201).body(dataAlreadyExists[id]) + } + + @GetMapping(path = ["/notempty/{id}"], produces = ["application/json"]) + open fun getnotempty(@PathVariable("id") id: Int): ResponseEntity { + val resource = dataAlreadyExists[id] + ?: return ResponseEntity.status(404).build() + return ResponseEntity.status(200).body(resource) + } + + @PutMapping(path = ["/notempty/{id}"], consumes = ["application/x-www-form-urlencoded"], produces = ["text/plain"]) + open fun putnotempty( + @PathVariable("id") id: Int, + @ModelAttribute body: UpdateRequest + ): ResponseEntity { + + val resource = dataAlreadyExists[id] + ?: return ResponseEntity.status(404).build() + + resource.name = body.name + resource.value = body.value + + // returns 400 Bad Request, but the data was already modified above + return ResponseEntity.status(400).body("Invalid request") + } + + @PatchMapping(path = ["/notempty/{id}"], consumes = ["application/x-www-form-urlencoded"], produces = ["text/plain"]) + open fun patchnotempty( + @PathVariable("id") id: Int, + @ModelAttribute body: UpdateRequest + ): ResponseEntity { + + val resource = dataAlreadyExists[id] + ?: return ResponseEntity.status(404).build() + + // correct: validation first, reject without modifying + return ResponseEntity.status(400).body("No fields to update") + + // correct: does NOT modify data, just returns 4xx + return ResponseEntity.status(403).body("Forbidden") + } +} diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/xml/XmlFailModificationApplication.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/xml/XmlFailModificationApplication.kt new file mode 100644 index 0000000000..d8cce3087c --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/main/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/xml/XmlFailModificationApplication.kt @@ -0,0 +1,155 @@ +package com.foo.rest.examples.spring.openapi.v3.httporacle.failmodification.xml + +import org.springframework.boot.SpringApplication +import org.springframework.boot.autoconfigure.SpringBootApplication +import org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PatchMapping +import org.springframework.web.bind.annotation.PathVariable +import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.PutMapping +import org.springframework.web.bind.annotation.RequestBody +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController +import javax.xml.bind.annotation.XmlAccessType +import javax.xml.bind.annotation.XmlAccessorType +import javax.xml.bind.annotation.XmlRootElement + +@SpringBootApplication(exclude = [SecurityAutoConfiguration::class]) +@RequestMapping(path = ["/api/resources"]) +@RestController +open class XmlFailModificationApplication { + + companion object { + @JvmStatic + fun main(args: Array) { + SpringApplication.run(XmlFailModificationApplication::class.java, *args) + } + + private val data = mutableMapOf() + private val dataAlreadyExists = mutableMapOf() + + fun reset(){ + data.clear() + dataAlreadyExists.clear() + dataAlreadyExists[0] = ResourceData("existing", 42) + } + } + + @XmlRootElement(name = "resourceData") + @XmlAccessorType(XmlAccessType.FIELD) + open class ResourceData( + var name: String = "", + var value: Int = 0 + ) + + @XmlRootElement(name = "updateRequest") + @XmlAccessorType(XmlAccessType.FIELD) + open class UpdateRequest( + var name: String = "", + var value: Int = 0 + ) + + + @PostMapping(path = ["/empty"], consumes = ["application/xml"], produces = ["application/xml"]) + open fun create(@RequestBody body: ResourceData): ResponseEntity { + val id = data.size + 1 + data[id] = ResourceData(body.name, body.value) + return ResponseEntity.status(201).body(data[id]) + } + + @GetMapping(path = ["/empty/{id}"], produces = ["application/xml"]) + open fun get(@PathVariable("id") id: Int): ResponseEntity { + val resource = data[id] + ?: return ResponseEntity.status(404).build() + return ResponseEntity.status(200).body(resource) + } + + @PutMapping(path = ["/empty/{id}"], consumes = ["application/xml"], produces = ["text/plain"]) + open fun put( + @PathVariable("id") id: Int, + @RequestBody body: UpdateRequest + ): ResponseEntity { + + val resource = data[id] + ?: return ResponseEntity.status(404).build() + + // bug: modifies data even though it will return 4xx + if(body.name != null) { + resource.name = body.name + } + if(body.value != null) { + resource.value = body.value + } + + // returns 400 Bad Request, but the data was already modified above + return ResponseEntity.status(400).body("Invalid request") + } + + @PatchMapping(path = ["/empty/{id}"], consumes = ["application/xml"], produces = ["text/plain"]) + open fun patch( + @PathVariable("id") id: Int, + @RequestBody body: UpdateRequest + ): ResponseEntity { + + val resource = data[id] + ?: return ResponseEntity.status(404).build() + + // correct: validation first, reject without modifying + if(body.name == null && body.value == null) { + return ResponseEntity.status(400).body("No fields to update") + } + + // correct: does NOT modify data, just returns 4xx + return ResponseEntity.status(403).body("Forbidden") + } + + // pre-populated resource to test that it is not modified by failed PUT + + @PostMapping(path = ["/notempty"], consumes = ["application/xml"], produces = ["application/xml"]) + open fun createnotempty(@RequestBody body: ResourceData): ResponseEntity { + val id = dataAlreadyExists.size + 1 + dataAlreadyExists[id] = ResourceData(body.name, body.value) + return ResponseEntity.status(201).body(dataAlreadyExists[id]) + } + + @GetMapping(path = ["/notempty/{id}"], produces = ["application/xml"]) + open fun getnotempty(@PathVariable("id") id: Int): ResponseEntity { + val resource = dataAlreadyExists[id] + ?: return ResponseEntity.status(404).build() + return ResponseEntity.status(200).body(resource) + } + + @PutMapping(path = ["/notempty/{id}"], consumes = ["application/xml"], produces = ["text/plain"]) + open fun putnotempty( + @PathVariable("id") id: Int, + @RequestBody body: UpdateRequest + ): ResponseEntity { + + val resource = dataAlreadyExists[id] + ?: return ResponseEntity.status(404).build() + + resource.name = body.name + resource.value = body.value + + // returns 400 Bad Request, but the data was already modified above + return ResponseEntity.status(400).body("Invalid request") + } + + @PatchMapping(path = ["/notempty/{id}"], consumes = ["application/xml"], produces = ["text/plain"]) + open fun patchnotempty( + @PathVariable("id") id: Int, + @RequestBody body: UpdateRequest + ): ResponseEntity { + + val resource = dataAlreadyExists[id] + ?: return ResponseEntity.status(404).build() + + // correct: validation first, reject without modifying + return ResponseEntity.status(400).body("No fields to update") + + // correct: does NOT modify data, just returns 4xx + return ResponseEntity.status(403).body("Forbidden") + } +} diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationURLEncodedController.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationURLEncodedController.kt new file mode 100644 index 0000000000..35e00cebec --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationURLEncodedController.kt @@ -0,0 +1,12 @@ +package com.foo.rest.examples.spring.openapi.v3.httporacle.failmodification + +import com.foo.rest.examples.spring.openapi.v3.SpringController +import com.foo.rest.examples.spring.openapi.v3.httporacle.failmodification.urlencoded.UrlencodedFailModificationApplication +import com.foo.rest.examples.spring.openapi.v3.httporacle.failmodification.xml.XmlFailModificationApplication + + +class FailModificationURLEncodedController: SpringController(UrlencodedFailModificationApplication::class.java){ + override fun resetStateOfSUT() { + UrlencodedFailModificationApplication.reset() + } +} diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationXMLController.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationXMLController.kt new file mode 100644 index 0000000000..5a1af347cf --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/com/foo/rest/examples/spring/openapi/v3/httporacle/failmodification/FailModificationXMLController.kt @@ -0,0 +1,11 @@ +package com.foo.rest.examples.spring.openapi.v3.httporacle.failmodification + +import com.foo.rest.examples.spring.openapi.v3.SpringController +import com.foo.rest.examples.spring.openapi.v3.httporacle.failmodification.xml.XmlFailModificationApplication + + +class FailModificationXMLController: SpringController(XmlFailModificationApplication::class.java){ + override fun resetStateOfSUT() { + XmlFailModificationApplication.reset() + } +} diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationEMTest.kt index f3127a2038..21f4f8e10a 100644 --- a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationEMTest.kt +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationEMTest.kt @@ -26,7 +26,7 @@ class FailModificationEMTest : SpringTestBase(){ runTestHandlingFlakyAndCompilation( "FailedModificationEM", - 2000 + 1000 ) { args: MutableList -> setOption(args, "schemaOracles", "false") diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationForbiddenEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationForbiddenEMTest.kt index bad8c57b13..5f2da7aebe 100644 --- a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationForbiddenEMTest.kt +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationForbiddenEMTest.kt @@ -25,7 +25,7 @@ class FailModificationForbiddenEMTest : SpringTestBase(){ runTestHandlingFlakyAndCompilation( "FailedModificationForbiddenEM", - 3000 + 2500 ) { args: MutableList -> setOption(args, "security", "true") diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationNotFoundEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationNotFoundEMTest.kt index c7fbd75d72..3d908ba5cd 100644 --- a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationNotFoundEMTest.kt +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/FailModificationNotFoundEMTest.kt @@ -26,7 +26,7 @@ class FailModificationNotFoundEMTest : SpringTestBase(){ runTestHandlingFlakyAndCompilation( "FailedModificationNotFoundEM", - 2000 + 50 ) { args: MutableList -> setOption(args, "schemaOracles", "false") diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/URLEncodedFailModificationEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/URLEncodedFailModificationEMTest.kt new file mode 100644 index 0000000000..b08f92c1b2 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/URLEncodedFailModificationEMTest.kt @@ -0,0 +1,45 @@ +package org.evomaster.e2etests.spring.openapi.v3.httporacle.failmodification + +import com.foo.rest.examples.spring.openapi.v3.httporacle.failmodification.FailModificationURLEncodedController +import org.evomaster.core.problem.enterprise.DetectedFaultUtils +import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory +import org.evomaster.e2etests.spring.openapi.v3.SpringTestBase +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test + +class URLEncodedFailModificationEMTest : SpringTestBase(){ + + companion object { + @BeforeAll + @JvmStatic + fun init() { + initClass(FailModificationURLEncodedController()) + } + } + + + @Test + fun testRunEM() { + + runTestHandlingFlakyAndCompilation( + "URLEncodedFailedModificationEM", + 100 + ) { args: MutableList -> + + setOption(args, "schemaOracles", "false") + setOption(args, "httpOracles", "true") + setOption(args, "useExperimentalOracles", "true") + + val solution = initAndRun(args) + + assertTrue(solution.individuals.size >= 1) + + val faults = DetectedFaultUtils.getDetectedFaults(solution) + + assertEquals(1, faults.size) + assertEquals(ExperimentalFaultCategory.HTTP_SIDE_EFFECTS_FAILED_MODIFICATION, faults.first().category) + } + } +} diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/XMLFailModificationEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/XMLFailModificationEMTest.kt new file mode 100644 index 0000000000..a491da04c4 --- /dev/null +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/XMLFailModificationEMTest.kt @@ -0,0 +1,47 @@ +package org.evomaster.e2etests.spring.openapi.v3.httporacle.failmodification + +import com.foo.rest.examples.spring.openapi.v3.httporacle.failmodification.FailModificationController +import com.foo.rest.examples.spring.openapi.v3.httporacle.failmodification.FailModificationXMLController +import org.evomaster.core.problem.enterprise.DetectedFaultUtils +import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory +import org.evomaster.core.problem.rest.data.HttpVerb +import org.evomaster.e2etests.spring.openapi.v3.SpringTestBase +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeAll +import org.junit.jupiter.api.Test + +class XMLFailModificationEMTest : SpringTestBase(){ + + companion object { + @BeforeAll + @JvmStatic + fun init() { + initClass(FailModificationXMLController()) + } + } + + + @Test + fun testRunEM() { + + runTestHandlingFlakyAndCompilation( + "XMLFailedModificationEM", + 500 + ) { args: MutableList -> + + setOption(args, "schemaOracles", "false") + setOption(args, "httpOracles", "true") + setOption(args, "useExperimentalOracles", "true") + + val solution = initAndRun(args) + + assertTrue(solution.individuals.size >= 1) + + val faults = DetectedFaultUtils.getDetectedFaults(solution) + + assertEquals(2, faults.size) + assertEquals(ExperimentalFaultCategory.HTTP_SIDE_EFFECTS_FAILED_MODIFICATION, faults.first().category) + } + } +} diff --git a/core/src/main/kotlin/org/evomaster/core/Main.kt b/core/src/main/kotlin/org/evomaster/core/Main.kt index bc8852e928..2151e16e11 100644 --- a/core/src/main/kotlin/org/evomaster/core/Main.kt +++ b/core/src/main/kotlin/org/evomaster/core/Main.kt @@ -262,6 +262,9 @@ class Main { logTimeSearchInfo(injector, config) //apply new phases + // 403 are usually not found during search, but created on purpose during security phase by + // mixing different users in the same test. as some http oracles might depend on 403s, + // we make sure to run security phase first solution = phaseSecurity(injector, config, epc, solution) solution = phaseHttpOracle(injector, config, epc, solution) solution = phaseFlaky(injector, config, epc, solution) @@ -674,7 +677,7 @@ class Main { Key.get(object : TypeLiteral>() {}) EMConfig.Algorithm.MuPlusLambdaEA -> Key.get(object : TypeLiteral>() {}) - + EMConfig.Algorithm.MuLambdaEA -> Key.get(object : TypeLiteral>(){}) EMConfig.Algorithm.BreederGA -> @@ -713,7 +716,7 @@ class Main { Key.get(object : TypeLiteral>() {}) EMConfig.Algorithm.LIPS -> Key.get(object : TypeLiteral>() {}) - + EMConfig.Algorithm.MuPlusLambdaEA -> Key.get(object : TypeLiteral>() {}) EMConfig.Algorithm.MuLambdaEA -> @@ -753,7 +756,7 @@ class Main { Key.get(object : TypeLiteral>() {}) EMConfig.Algorithm.LIPS -> Key.get(object : TypeLiteral>() {}) - + EMConfig.Algorithm.MuPlusLambdaEA -> Key.get(object : TypeLiteral>() {}) EMConfig.Algorithm.MuLambdaEA -> diff --git a/core/src/main/kotlin/org/evomaster/core/output/formatter/OutputFormatter.kt b/core/src/main/kotlin/org/evomaster/core/output/formatter/OutputFormatter.kt index 9daad11fa9..8b43438744 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/formatter/OutputFormatter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/formatter/OutputFormatter.kt @@ -3,10 +3,13 @@ package org.evomaster.core.output.formatter import com.fasterxml.jackson.core.JsonProcessingException import com.fasterxml.jackson.databind.DeserializationFeature import com.fasterxml.jackson.databind.ObjectMapper -import com.google.gson.GsonBuilder -import com.google.gson.JsonParser -import javax.ws.rs.client.Entity.json - +import java.io.ByteArrayInputStream +import java.io.StringWriter +import javax.xml.parsers.DocumentBuilderFactory +import javax.xml.transform.OutputKeys +import javax.xml.transform.TransformerFactory +import javax.xml.transform.dom.DOMSource +import javax.xml.transform.stream.StreamResult /** * @javatypes: manzhang @@ -64,9 +67,65 @@ open abstract class OutputFormatter (val name: String) { } throw MismatchedFormatException(this, content) } + + override fun readFields(content: String, fieldNames: Set): Map? { + return try { + val node = objectMapper.readTree(content) + if (!node.isObject) return null + fieldNames.mapNotNull { field -> + val value = node.get(field) ?: return@mapNotNull null + field to value.asText() + }.toMap() + } catch (e: Exception) { + null + } + } } + + + val XML_FORMATTER = object : OutputFormatter("XML_FORMATTER") { + private val xmlFactory = DocumentBuilderFactory.newInstance() + + override fun isValid(content: String): Boolean { + return try { + xmlFactory.newDocumentBuilder() + .parse(ByteArrayInputStream(content.toByteArray(Charsets.UTF_8))) + true + } catch (e: Exception) { + false + } + } + + override fun getFormatted(content: String): String { + if (!isValid(content)) throw MismatchedFormatException(this, content) + val doc = xmlFactory.newDocumentBuilder() + .parse(ByteArrayInputStream(content.toByteArray(Charsets.UTF_8))) + val transformer = TransformerFactory.newInstance().newTransformer() + transformer.setOutputProperty(OutputKeys.INDENT, "yes") + transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "2") + val writer = StringWriter() + transformer.transform(DOMSource(doc), StreamResult(writer)) + return writer.toString().replace("\r\n", "\n") + } + + override fun readFields(content: String, fieldNames: Set): Map? { + return try { + val doc = xmlFactory.newDocumentBuilder() + .parse(ByteArrayInputStream(content.toByteArray(Charsets.UTF_8))) + doc.documentElement.normalize() + fieldNames.mapNotNull { field -> + val nodes = doc.getElementsByTagName(field) + if (nodes.length > 0) field to nodes.item(0).textContent else null + }.toMap() + } catch (e: Exception) { + null + } + } + } + init { registerFormatter(JSON_FORMATTER) + registerFormatter(XML_FORMATTER) } @@ -74,6 +133,7 @@ open abstract class OutputFormatter (val name: String) { abstract fun isValid(content: String): Boolean abstract fun getFormatted(content: String): String + abstract fun readFields(content: String, fieldNames: Set): Map? } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracle.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracle.kt index 438472b383..d4630dc56d 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracle.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracle.kt @@ -1,6 +1,6 @@ package org.evomaster.core.problem.rest.oracle -import com.google.gson.JsonParser +import org.evomaster.core.output.formatter.OutputFormatter import org.evomaster.core.problem.rest.data.HttpVerb import org.evomaster.core.problem.rest.data.RestCallAction import org.evomaster.core.problem.rest.data.RestCallResult @@ -14,6 +14,7 @@ import org.evomaster.core.search.gene.utils.GeneUtils object HttpSemanticsOracle { + fun hasRepeatedCreatePut(individual: RestIndividual, actionResults: List ): Boolean{ @@ -161,6 +162,13 @@ object HttpSemanticsOracle { return false } + // this oracle only supports JSON, XML, and form-urlencoded request bodies; + // other content types (e.g. text/plain, multipart, binary) are not handled + val bodyParam = modify.parameters.find { it is BodyParam } as BodyParam? + if (bodyParam != null && !bodyParam.isJson() && !bodyParam.isXml() && !bodyParam.isForm()) { + return false + } + // after GET must be 2xx if(!StatusGroup.G_2xx.isInGroup(resAfter.getStatusCode())) { return false @@ -183,7 +191,7 @@ object HttpSemanticsOracle { && !bodyBefore.isNullOrEmpty() && !bodyAfter.isNullOrEmpty() && !bodyModify.isNullOrEmpty()) { - return hasChangedModifiedFields(bodyBefore, bodyAfter, bodyModify, modifiedFieldNames) + return hasChangedModifiedFields(bodyBefore, bodyAfter, bodyModify, modifiedFieldNames, bodyParam) } return false @@ -192,7 +200,13 @@ object HttpSemanticsOracle { private fun extractRequestBody(modify: RestCallAction): String? { val bodyParam = modify.parameters.find { it is BodyParam } as BodyParam? ?: return null - return bodyParam.getValueAsPrintableString(mode = GeneUtils.EscapeMode.JSON) + val mode = when { + bodyParam.isJson() -> GeneUtils.EscapeMode.JSON + bodyParam.isXml() -> GeneUtils.EscapeMode.XML + bodyParam.isForm() -> GeneUtils.EscapeMode.X_WWW_FORM_URLENCODED + else -> null + } + return bodyParam.getValueAsPrintableString(mode = mode) } /** @@ -257,43 +271,109 @@ object HttpSemanticsOracle { * Compares only the fields that were sent in the PUT/PATCH request. * Returns true if any of those fields changed between the before and after GET responses. * - * NOTE: This only works when the request payload is a JSON object that directly - * matches the resource structure. It does NOT support operation-based payloads - * such as JSON Patch (RFC 6902). + * Dispatches to a format-specific comparison based on [bodyParam] content type: + * - JSON : field-by-field comparison via [OutputFormatter.JSON_FORMATTER] + * - XML : field-by-field comparison via XML DOM parsing + * - form-encoded : GET responses parsed as JSON or XML, request parsed as key=value pairs + * - other : returns false (incl. text/plain, which may cause too many false positives) + * + * NOTE: Does not support operation-based payloads such as JSON Patch (RFC 6902). */ internal fun hasChangedModifiedFields( + bodyBefore: String, + bodyAfter: String, + bodyModify: String, + fieldNames: Set, + bodyParam: BodyParam? = null + ): Boolean { + return when { + bodyParam == null || bodyParam.isJson() -> + hasChangedModifiedFieldsStructured(OutputFormatter.JSON_FORMATTER, bodyBefore, bodyAfter, bodyModify, fieldNames) + bodyParam.isXml() -> + hasChangedModifiedFieldsStructured(OutputFormatter.XML_FORMATTER, bodyBefore, bodyAfter, bodyModify, fieldNames) + bodyParam.isForm() -> + hasChangedModifiedFieldsForm(bodyBefore, bodyAfter, bodyModify, fieldNames) + else -> false + } + } + + /** + * Generic field-level comparison for structured formats (JSON, XML). + * Uses [formatter]'s [OutputFormatter.readFields] to extract field values from all three bodies. + */ + private fun hasChangedModifiedFieldsStructured( + formatter: OutputFormatter, bodyBefore: String, bodyAfter: String, bodyModify: String, fieldNames: Set ): Boolean { + val fieldsBefore = formatter.readFields(bodyBefore, fieldNames) ?: return false + val fieldsAfter = formatter.readFields(bodyAfter, fieldNames) ?: return false + val fieldsModify = formatter.readFields(bodyModify, fieldNames) ?: return false - try { - val jsonBefore = JsonParser.parseString(bodyBefore) - val jsonAfter = JsonParser.parseString(bodyAfter) - val jsonModify = JsonParser.parseString(bodyModify) + for (field in fieldNames) { + val valueBefore = fieldsBefore[field] ?: continue + val valueAfter = fieldsAfter[field] - if(!jsonBefore.isJsonObject || !jsonAfter.isJsonObject || !jsonModify.isJsonObject){ - return false - } + // field existed before but disappeared after the failed modification + if (valueAfter == null) return true - val objBefore = jsonBefore.asJsonObject - val objAfter = jsonAfter.asJsonObject - val objModify = jsonModify.asJsonObject + val valueModify = fieldsModify[field] ?: continue - for(field in fieldNames){ - val valueBefore = objBefore.get(field) - val valueAfter = objAfter.get(field) - val valueModify = objModify.get(field) + // checking valueModify==valueAfter (not just valueAfter!=valueBefore) is done to + // deal with possible flakiness issues (e.g. timestamps changing between the two GETs) + if (valueBefore != valueAfter && valueModify == valueAfter) return true + } + return false + } - if(valueBefore != valueAfter && valueModify == valueAfter){ - return true - } - } + /** + * Handles the case where the PUT/PATCH request body is form-encoded. + * GET responses (bodyBefore / bodyAfter) can be JSON or XML; the format is auto-detected + * by trying each formatter in order and using the first one that parses both responses. + * Values are compared as strings since form-encoded values are always strings. + */ + private fun hasChangedModifiedFieldsForm( + bodyBefore: String, + bodyAfter: String, + bodyModify: String, + fieldNames: Set + ): Boolean { + val formFields = parseFormBody(bodyModify) + if (formFields.isEmpty()) return false - return false - } catch (e: Exception) { + for (formatter in listOf(OutputFormatter.JSON_FORMATTER, OutputFormatter.XML_FORMATTER)) { + val fieldsBefore = formatter.readFields(bodyBefore, fieldNames) ?: continue + val fieldsAfter = formatter.readFields(bodyAfter, fieldNames) ?: continue + + for (field in fieldNames) { + val valueBefore = fieldsBefore[field] ?: continue + val valueAfter = fieldsAfter[field] + + // field existed before but disappeared after the failed modification + if (valueAfter == null) return true + + val valueModify = formFields[field] ?: continue + + // checking valueModify==valueAfter (not just valueAfter!=valueBefore) is done to + // deal with possible flakiness issues (e.g. timestamps changing between the two GETs) + if (valueBefore != valueAfter && valueModify == valueAfter) return true + } return false } + return false + } + + private fun parseFormBody(body: String): Map { + return body.split("&").mapNotNull { pair -> + val parts = pair.split("=", limit = 2) + if (parts.size == 2) { + try { + java.net.URLDecoder.decode(parts[0], "UTF-8") to + java.net.URLDecoder.decode(parts[1], "UTF-8") + } catch (e: Exception) { null } + } else null + }.toMap() } } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt index bf56595ad7..872b53a531 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/HttpSemanticsService.kt @@ -208,6 +208,15 @@ class HttpSemanticsService { } + /** + * A failed PUT/PATCH (4xx) must not mutate the resource state. + * For each path with PUT/PATCH and GET, and for each distinct K in 4xx: + * - find T (smallest individual ending with GET 2xx on same path), slice after it + * - append a PUT/PATCH targeting K, then a final GET to check state is unchanged + * - K==401: use a 2xx PUT/PATCH with no auth; K==403: different auth than the GET + * - K==404: special - no T, prepend GET->(PUT/PATCH 404)->GET, all expecting 404 + * - otherwise (400, 409, ...): copy a 4xx action with same auth as the GET + */ private fun sideEffectsOfFailedModification() { val verbs = listOf(HttpVerb.PUT, HttpVerb.PATCH) @@ -242,8 +251,8 @@ class HttpSemanticsService { for (k in distinctCodes) { when (k) { - 401, 403 -> handle401Or403SideEffect(verb, k, modOp.path) - else -> addGetAroundFailedModification(verb, k, modOp.path, getDef, failedModifyEvals) + 404 -> handle404SideEffect(verb, modOp.path, getDef) + else -> handleSideEffectOfFailedModification(verb, k, modOp.path, getDef) } } } @@ -251,63 +260,93 @@ class HttpSemanticsService { } /** - * Handles K==401 and K==403. + * Handles K==404: the resource did not exist before the call, PUT/PATCH also returned 404, + * and the final GET must still return 404 — anything else is a side effect. * - * 1. Find T — smallest individual ending with a clean GET 2xx on [path] - * 2. Find a 2xx PUT/PATCH action as the body template or fall back to the K action if no 2xx exists - * 3. Copy the template and override auth: - * K==401 → NoAuth (expected to trigger 401) - * K==403 → a different authenticated user - * 4. Append the modified PUT/PATCH after the GET in T, then append another GET + * GET /path → 404 (resource absent) + * PUT|PATCH /path → 404 + * GET /path → ??? (oracle: must still be 404) */ - private fun handle401Or403SideEffect(verb: HttpVerb, k: Int, path: RestPath) { + private fun handle404SideEffect(verb: HttpVerb, path: RestPath, getDef: RestCallAction) { + + val kEval = RestIndividualSelectorUtils.findIndividuals(individualsInSolution, verb, path, status = 404) + .minByOrNull { it.individual.size() } ?: return + + val ind = RestIndividualBuilder.sliceAllCallsInIndividualAfterAction(kEval, verb, path, status = 404) + + val actions = ind.seeMainExecutableActions() + val last = actions.last() // the PUT/PATCH [404] + + val getBefore = builder.createBoundActionFor(getDef, last) + ind.addMainActionInEmptyEnterpriseGroup(actions.size - 1, getBefore) + + val getAfter = builder.createBoundActionFor(getDef, last) + ind.addMainActionInEmptyEnterpriseGroup(-1, getAfter) - // GET schema definition — needed to create the GET after via builder - val getDef = actionDefinitions.find { it.verb == HttpVerb.GET && it.path.isEquivalent(path) } - ?: return + prepareEvaluateAndSave(ind) + } + + /** + * Handles all non-404 4xx cases (K==400, K==401, K==403, K==409, …). + * Starts from T (smallest clean GET 2xx individual), appends the K-returning PUT/PATCH + * bound to that GET's resolved path, then appends a final GET to verify state is unchanged. + * + * Action template: + * - K==401/403 : prefer a 2xx PUT/PATCH (valid body, wrong auth); fall back to the K action + * - otherwise : use the K-returning action directly (correct auth, body that causes K) + * + * Auth override: + * - K==401 → NoAuth + * - K==403 → a different authenticated user + * - otherwise → same auth as the GET (failure is due to body content, not access rights) + */ + private fun handleSideEffectOfFailedModification(verb: HttpVerb, k: Int, path: RestPath, getDef: RestCallAction) { - // T: smallest clean individual ending with GET 2xx (no prior PUT/PATCH on same path) + // T: smallest individual ending with GET 2xx on the same path val T = RestIndividualSelectorUtils.findAndSlice( individualsInSolution, HttpVerb.GET, path, statusGroup = StatusGroup.G_2xx - ).filter { ind -> - val actions = ind.seeMainExecutableActions() - actions.subList(0, actions.size - 1).none { - (it.verb == HttpVerb.PUT || it.verb == HttpVerb.PATCH) && it.path.isEquivalent(path) - } - }.minByOrNull { it.size() } ?: return - - // find a 2xx PUT/PATCH action to use as the body template - // 401/403 action itself if no 2xx exists - val successAction = RestIndividualSelectorUtils.findIndividuals( - individualsInSolution, verb, path, statusGroup = StatusGroup.G_2xx - ).flatMap { ei -> - ei.evaluatedMainActions().mapNotNull { ea -> - val a = ea.action as? RestCallAction ?: return@mapNotNull null - val r = ea.result as? RestCallResult ?: return@mapNotNull null - if (a.verb == verb && a.path.isEquivalent(path) && StatusGroup.G_2xx.isInGroup(r.getStatusCode())) - a else null - } - }.firstOrNull() - ?: RestIndividualSelectorUtils.findIndividuals( - individualsInSolution, verb, path, status = k - ).flatMap { ei -> - ei.evaluatedMainActions().mapNotNull { ea -> - (ea.action as? RestCallAction) - ?.takeIf { it.verb == verb && it.path.isEquivalent(path) } - } - }.firstOrNull() - ?: return + ).minByOrNull { it.size() } ?: return + + val actionTemplate = when { + k == 401 || k == 403 -> + // prefer a 2xx action (valid body); fall back to the K action if no 2xx exists + RestIndividualSelectorUtils.findIndividuals( + individualsInSolution, verb, path, statusGroup = StatusGroup.G_2xx + ).flatMap { ei -> + ei.evaluatedMainActions().mapNotNull { ea -> + val a = ea.action as? RestCallAction ?: return@mapNotNull null + val r = ea.result as? RestCallResult ?: return@mapNotNull null + if (a.verb == verb && a.path.isEquivalent(path) && StatusGroup.G_2xx.isInGroup(r.getStatusCode())) + a else null + } + }.firstOrNull() + ?: RestIndividualSelectorUtils.findIndividuals( + individualsInSolution, verb, path, status = k + ).flatMap { ei -> + ei.evaluatedMainActions().mapNotNull { ea -> + (ea.action as? RestCallAction) + ?.takeIf { it.verb == verb && it.path.isEquivalent(path) } + } + }.firstOrNull() + else -> + RestIndividualSelectorUtils.findIndividuals( + individualsInSolution, verb, path, status = k + ).flatMap { ei -> + ei.evaluatedMainActions().mapNotNull { ea -> + (ea.action as? RestCallAction) + ?.takeIf { it.verb == verb && it.path.isEquivalent(path) } + } + }.firstOrNull() + } ?: return val ind = T.copy() as RestIndividual - val getAction = ind.seeMainExecutableActions().last().copy() as RestCallAction // the GET 2xx at the end of T - val successCopy = successAction.copy() as RestCallAction + val getAction = ind.seeMainExecutableActions().last().copy() as RestCallAction - successCopy.forceNewTaints() - successCopy.resetLocalIdRecursively() + val templateCopy = actionTemplate.copy() as RestCallAction + templateCopy.forceNewTaints() + templateCopy.resetLocalIdRecursively() - - // we override auth afterwards to achieve no-auth (401) or different-user (403) - val modifyCopy = builder.createBoundActionFor(successCopy, getAction) + val modifyCopy = builder.createBoundActionFor(templateCopy, getAction) when (k) { 401 -> modifyCopy.auth = HttpWsNoAuth() 403 -> { @@ -316,59 +355,22 @@ class HttpSemanticsService { if (otherAuths.isEmpty()) return modifyCopy.auth = otherAuths.first() } + else -> modifyCopy.auth = getAction.auth } + getAction.forceNewTaints() getAction.resetLocalIdRecursively() val getAfter = builder.createBoundActionFor(getDef, getAction) - ind.addMainActionInEmptyEnterpriseGroup(action = modifyCopy) ind.addMainActionInEmptyEnterpriseGroup(action = getAfter) - - ind.ensureFlattenedStructure() org.evomaster.core.Lazy.assert { ind.verifyValidity(); true } prepareEvaluateAndSave(ind) } - /** - * Takes the smallest individual in [candidates] where [verb] on [path] returned [k], - * slices it at that action, then inserts a GET immediately before it and appends - * another GET immediately after it — both on the same resolved path and with the - * same auth as the PUT/PATCH: - * - * GET /path (same auth as PUT/PATCH) - * PUT|PATCH /path [k] - * GET /path (same auth) - */ - private fun addGetAroundFailedModification( - verb: HttpVerb, - k: Int, - path: RestPath, - getDef: RestCallAction, - candidates: List> - ) { - val kEval = RestIndividualSelectorUtils.findIndividuals(candidates, verb, path, status = k) - .minByOrNull { it.individual.size() } ?: return - - val ind = RestIndividualBuilder.sliceAllCallsInIndividualAfterAction(kEval, verb, path, status = k) - - val actions = ind.seeMainExecutableActions() - val last = actions.last() // the PUT/PATCH [k] - - // insert GET before the PUT/PATCH - val getBefore = builder.createBoundActionFor(getDef, last) - ind.addMainActionInEmptyEnterpriseGroup(actions.size - 1, getBefore) - - // append GET after the PUT/PATCH - val getAfter = builder.createBoundActionFor(getDef, last) - ind.addMainActionInEmptyEnterpriseGroup(-1, getAfter) - - prepareEvaluateAndSave(ind) - } - } diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt index f8d1e3e6b2..cc5ffdb03f 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt @@ -1205,21 +1205,15 @@ abstract class AbstractRestFitness : HttpWsFitness() { } private fun analyzeHttpSemantics(individual: RestIndividual, actionResults: List, fv: FitnessValue) { - if(!config.isEnabledFaultCategory(ExperimentalFaultCategory.HTTP_NONWORKING_DELETE)) { - LoggingUtil.uniqueUserInfo("Skipping experimental security test for non-working DELETE, as it has been disabled via configuration") - } else { + if(config.isEnabledFaultCategory(ExperimentalFaultCategory.HTTP_NONWORKING_DELETE)) { handleDeleteShouldDelete(individual, actionResults, fv) } - if(!config.isEnabledFaultCategory(ExperimentalFaultCategory.HTTP_REPEATED_CREATE_PUT)) { - LoggingUtil.uniqueUserInfo("Skipping experimental security test for repeated PUT after CREATE, as it has been disabled via configuration") - } else { + if(config.isEnabledFaultCategory(ExperimentalFaultCategory.HTTP_REPEATED_CREATE_PUT)) { handleRepeatedCreatePut(individual, actionResults, fv) } - if(!config.isEnabledFaultCategory(ExperimentalFaultCategory.HTTP_SIDE_EFFECTS_FAILED_MODIFICATION)) { - LoggingUtil.uniqueUserInfo("Skipping experimental security test for repeated PUT after CREATE, as it has been disabled via configuration") - } else { + if(config.isEnabledFaultCategory(ExperimentalFaultCategory.HTTP_SIDE_EFFECTS_FAILED_MODIFICATION)) { handleFailedModification(individual, actionResults, fv) } } diff --git a/core/src/test/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracleTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracleTest.kt index 23a86473e7..5af0801637 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracleTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/rest/oracle/HttpSemanticsOracleTest.kt @@ -1,10 +1,23 @@ package org.evomaster.core.problem.rest.oracle +import org.evomaster.core.problem.rest.param.BodyParam +import org.evomaster.core.search.gene.BooleanGene +import org.evomaster.core.search.gene.collection.EnumGene import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.Test class HttpSemanticsOracleTest { + private fun xmlBodyParam() = BodyParam( + BooleanGene("body"), + EnumGene("body", listOf("application/xml")) + ) + + private fun formBodyParam() = BodyParam( + BooleanGene("body"), + EnumGene("body", listOf("application/x-www-form-urlencoded")) + ) + @Test fun testUnchangedModifiedFieldReturnsFalse() { assertFalse(HttpSemanticsOracle.hasChangedModifiedFields( @@ -105,4 +118,233 @@ class HttpSemanticsOracleTest { fieldNames = setOf("name") )) } + + // ------------------------------------------------------------------------- + // XML variants + // ------------------------------------------------------------------------- + + @Test + fun testUnchangedModifiedFieldReturnsFalse_Xml() { + assertFalse(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = "Doe2026-01-01", + bodyAfter = "Doe2026-01-02", + bodyModify = "Test", + fieldNames = setOf("name"), + bodyParam = xmlBodyParam() + )) + } + + @Test + fun testNoModifiedFieldChangedReturnsFalse_Xml() { + assertFalse(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = "Doea@a.com30", + bodyAfter = "Doea@a.com31", + bodyModify = "31", + fieldNames = setOf("name", "email"), + bodyParam = xmlBodyParam() + )) + } + + @Test + fun testModifiedFieldAbsentInBothBodiesReturnsFalse_Xml() { + assertFalse(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = "30", + bodyAfter = "31", + bodyModify = "31", + fieldNames = setOf("name"), + bodyParam = xmlBodyParam() + )) + } + + @Test + fun testUnchangedIntegerModifiedFieldReturnsFalse_Xml() { + assertFalse(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = "42", + bodyAfter = "42", + bodyModify = "42", + fieldNames = setOf("count"), + bodyParam = xmlBodyParam() + )) + } + + @Test + fun testChangedModifiedFieldReturnsTrue_Xml() { + assertTrue(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = "Doe42", + bodyAfter = "Bob42", + bodyModify = "Bob", + fieldNames = setOf("name"), + bodyParam = xmlBodyParam() + )) + } + + @Test + fun testOneOfMultipleModifiedFieldsChangedReturnsTrue_Xml() { + assertTrue(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = "Doea@a.com42", + bodyAfter = "Doeb@b.com42", + bodyModify = "Doeb@b.com42", + fieldNames = setOf("name", "email"), + bodyParam = xmlBodyParam() + )) + } + + @Test + fun testModifiedFieldPresentInBeforeButAbsentInAfterReturnsTrue_Xml() { + assertTrue(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = "Doe", + bodyAfter = "42", + bodyModify = "42", + fieldNames = setOf("name"), + bodyParam = xmlBodyParam() + )) + } + + @Test + fun testChangedIntegerModifiedFieldReturnsTrue_Xml() { + assertTrue(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = "42", + bodyAfter = "44", + bodyModify = "44", + fieldNames = setOf("count"), + bodyParam = xmlBodyParam() + )) + } + + @Test + fun testInvalidXmlDifferentBodiesFallbackReturnsFalse_Xml() { + assertFalse(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = "not valid xml", + bodyAfter = "also not valid xml", + bodyModify = "", + fieldNames = setOf("name"), + bodyParam = xmlBodyParam() + )) + } + + @Test + fun testInvalidXmlSameBodiesFallbackReturnsFalse_Xml() { + assertFalse(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = "not valid xml", + bodyAfter = "not valid xml", + bodyModify = "", + fieldNames = setOf("name"), + bodyParam = xmlBodyParam() + )) + } + + // ------------------------------------------------------------------------- + // Form (application/x-www-form-urlencoded) variants + // bodyBefore / bodyAfter are JSON GET responses; bodyModify is form-encoded + // ------------------------------------------------------------------------- + + @Test + fun testUnchangedModifiedFieldReturnsFalse_Form() { + assertFalse(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = """{"name":"Doe","ts":"2026-01-01"}""", + bodyAfter = """{"name":"Doe","ts":"2026-01-02"}""", + bodyModify = "name=Test", + fieldNames = setOf("name"), + bodyParam = formBodyParam() + )) + } + + @Test + fun testNoModifiedFieldChangedReturnsFalse_Form() { + assertFalse(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = """{"name":"Doe","email":"a@a.com","age":30}""", + bodyAfter = """{"name":"Doe","email":"a@a.com","age":31}""", + bodyModify = "age=31", + fieldNames = setOf("name", "email"), + bodyParam = formBodyParam() + )) + } + + @Test + fun testModifiedFieldAbsentInBothBodiesReturnsFalse_Form() { + assertFalse(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = """{"age":30}""", + bodyAfter = """{"age":31}""", + bodyModify = "age=31", + fieldNames = setOf("name"), + bodyParam = formBodyParam() + )) + } + + @Test + fun testUnchangedIntegerModifiedFieldReturnsFalse_Form() { + assertFalse(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = """{"count":42,"label":"test"}""", + bodyAfter = """{"count":42,"label":"changed"}""", + bodyModify = "count=42&label=changed", + fieldNames = setOf("count"), + bodyParam = formBodyParam() + )) + } + + @Test + fun testChangedModifiedFieldReturnsTrue_Form() { + assertTrue(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = """{"name":"Doe","age":42}""", + bodyAfter = """{"name":"Bob","age":42}""", + bodyModify = "name=Bob", + fieldNames = setOf("name"), + bodyParam = formBodyParam() + )) + } + + @Test + fun testOneOfMultipleModifiedFieldsChangedReturnsTrue_Form() { + assertTrue(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = """{"name":"Doe","email":"a@a.com","age":42}""", + bodyAfter = """{"name":"Doe","email":"b@b.com","age":42}""", + bodyModify = "name=Doe&email=b%40b.com&age=42", + fieldNames = setOf("name", "email"), + bodyParam = formBodyParam() + )) + } + + @Test + fun testModifiedFieldPresentInBeforeButAbsentInAfterReturnsTrue_Form() { + assertTrue(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = """{"name":"Doe"}""", + bodyAfter = """{"age":42}""", + bodyModify = "age=42", + fieldNames = setOf("name"), + bodyParam = formBodyParam() + )) + } + + @Test + fun testChangedIntegerModifiedFieldReturnsTrue_Form() { + assertTrue(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = """{"count":42,"label":"test"}""", + bodyAfter = """{"count":44,"label":"test"}""", + bodyModify = "count=44&label=test", + fieldNames = setOf("count"), + bodyParam = formBodyParam() + )) + } + + @Test + fun testInvalidBodiesDifferentReturnsFalse_Form() { + assertFalse(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = "not valid json", + bodyAfter = "also not valid json", + bodyModify = "name=Test", + fieldNames = setOf("name"), + bodyParam = formBodyParam() + )) + } + + @Test + fun testInvalidBodiesSameReturnsFalse_Form() { + assertFalse(HttpSemanticsOracle.hasChangedModifiedFields( + bodyBefore = "not valid json", + bodyAfter = "not valid json", + bodyModify = "name=Test", + fieldNames = setOf("name"), + bodyParam = formBodyParam() + )) + } } From 97375edc710ddb1d93f4477871eedc074d489bd5 Mon Sep 17 00:00:00 2001 From: Omur Sahin Date: Fri, 27 Mar 2026 23:53:45 +0300 Subject: [PATCH 12/13] update brittle tests and extend --- .../output/formatter/OutputFormatterTest.kt | 250 +++++++++++++++++- 1 file changed, 241 insertions(+), 9 deletions(-) diff --git a/core/src/test/kotlin/org/evomaster/core/output/formatter/OutputFormatterTest.kt b/core/src/test/kotlin/org/evomaster/core/output/formatter/OutputFormatterTest.kt index 825d4befe7..4cdb6d76ee 100644 --- a/core/src/test/kotlin/org/evomaster/core/output/formatter/OutputFormatterTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/output/formatter/OutputFormatterTest.kt @@ -13,7 +13,6 @@ class OutputFormatterTest { @Test fun test(){ - assertTrue(OutputFormatter.getFormatters()?.size == 1) val body = """ { "authorId": "VZyJz8z_Eu2", @@ -28,7 +27,6 @@ class OutputFormatterTest { @Test fun testMismatched(){ - assertTrue(OutputFormatter.getFormatters()?.size == 1) val body = """ "authorId": "VZyJz8z_Eu2", @@ -43,7 +41,6 @@ class OutputFormatterTest { } @Test fun testEscapes(){ - assertTrue(OutputFormatter.getFormatters()?.size == 1) val body = """ { "name":"T\"" @@ -56,7 +53,6 @@ class OutputFormatterTest { @Test fun testEscapes2(){ - assertTrue(OutputFormatter.getFormatters()?.size == 1) val string = """{"id":"9d8UV_=e1T0eWTlc", "value":"93${'$'}v98g"}""" OutputFormatter.JSON_FORMATTER.getFormatted(string) @@ -64,7 +60,6 @@ class OutputFormatterTest { @Test fun testEscapes3(){ - assertTrue(OutputFormatter.getFormatters()?.size == 1) val string = """ {"id":"19r\"l_", "value":""} @@ -74,7 +69,6 @@ class OutputFormatterTest { @Test fun testEscapes4(){ - assertTrue(OutputFormatter.getFormatters()?.size == 1) val testGene = StringGene("QuoteGene", "Test For the quotes${'"'}escape") OutputFormatter.JSON_FORMATTER.getFormatted(testGene.getValueAsPrintableString(mode = GeneUtils.EscapeMode.JSON, targetFormat = OutputFormat.KOTLIN_JUNIT_5)) @@ -82,7 +76,6 @@ class OutputFormatterTest { @Test fun testEscapes6(){ - assertTrue(OutputFormatter.getFormatters()?.size == 1) val string = """ {"id":"19r\\l_"} @@ -92,7 +85,6 @@ class OutputFormatterTest { @Test fun testEscapes7(){ - assertTrue(OutputFormatter.getFormatters()?.size == 1) val string = """ {"id":"Ot${'$'}Ag", "value":"Q"} @@ -102,7 +94,6 @@ class OutputFormatterTest { @Test fun testEscapes8(){ - assertTrue(OutputFormatter.getFormatters()?.size == 1) val testGene = StringGene("DollarGene", "Test For the dollar${'$'}escape") OutputFormatter.JSON_FORMATTER.getFormatted(testGene.getValueAsPrintableString(mode = GeneUtils.EscapeMode.JSON, targetFormat = OutputFormat.KOTLIN_JUNIT_5)) @@ -165,5 +156,246 @@ class OutputFormatterTest { assertTrue(isValid) } + @Test + fun testXml(){ + assertTrue(OutputFormatter.getFormatters()?.size == 2) + val body = """ + + VZyJz8z_Eu2 + 1921-3-13T10:18:56.000Z + L + + """.trimIndent() + + // should throw no exception + OutputFormatter.XML_FORMATTER.getFormatted(body) + } + + @Test + fun testXmlMismatched(){ + assertTrue(OutputFormatter.getFormatters()?.size == 2) + val body = """ + + VZyJz8z_Eu2 + 1921-3-13T10:18:56.000Z + L + """.trimIndent() + + assertThrows { + OutputFormatter.XML_FORMATTER.getFormatted(body) + } + } + + @Test + fun testValidXml() { + val xml = "Hello World" + val isValid = OutputFormatter.XML_FORMATTER.isValid(xml) + assertTrue(isValid) + } + + @Test + fun testInvalidXml() { + val xml = "Hello World" + val isValid = OutputFormatter.XML_FORMATTER.isValid(xml) + assertFalse(isValid) + } + + @Test + fun testXmlScientificNotationLikeValues() { + val body = """ + + 1e10 + 2.5e-3 + + """.trimIndent() + + val formatted = OutputFormatter.XML_FORMATTER.getFormatted(body) + assertNotNull(formatted) + } + + @Test + fun testXmlWithAttributes() { + val body = """ + + Content + + """.trimIndent() + + val formatted = OutputFormatter.XML_FORMATTER.getFormatted(body) + assertNotNull(formatted) + } + + @Test + fun testXmlWithSpecialCharacters() { + val body = """ + + <test> & "quote" + + """.trimIndent() + + val formatted = OutputFormatter.XML_FORMATTER.getFormatted(body) + assertNotNull(formatted) + } + + @Test + fun testXmlNestedElements() { + val body = """ + + + + value + + + + """.trimIndent() + + val formatted = OutputFormatter.XML_FORMATTER.getFormatted(body) + assertNotNull(formatted) + } + + @Test + fun testXmlSelfClosingTag() { + val body = """ + + + + """.trimIndent() + + val formatted = OutputFormatter.XML_FORMATTER.getFormatted(body) + assertNotNull(formatted) + } + @Test + fun testXmlInvalidEscape() { + val body = """ + + &invalid; + + """.trimIndent() + + assertThrows { + OutputFormatter.XML_FORMATTER.getFormatted(body) + } + } + + @Test + fun testXmlUnclosedTag() { + val body = """ + + value + + """.trimIndent() + + assertThrows { + OutputFormatter.XML_FORMATTER.getFormatted(body) + } + } + + @Test + fun testJsonReadFields() { + val body = """ + { + "authorId": "VZyJz8z_Eu2", + "creationTime": "1921-3-13T10:18:56.000Z", + "newsId": "L", + "title": "Hello" + } + """.trimIndent() + + val result = OutputFormatter.JSON_FORMATTER.readFields( + body, + setOf("authorId", "newsId") + ) + + assertNotNull(result) + assertEquals("VZyJz8z_Eu2", result?.get("authorId")) + assertEquals("L", result?.get("newsId")) + assertEquals(2, result?.size) + } + + @Test + fun testJsonReadFieldsMissingAndInvalid() { + val body = """ + { + "authorId": "VZyJz8z_Eu2", + "title": "Hello" + } + """.trimIndent() + + val result = OutputFormatter.JSON_FORMATTER.readFields( + body, + setOf("authorId", "newsId") + ) + + assertNotNull(result) + assertEquals("VZyJz8z_Eu2", result?.get("authorId")) + assertFalse(result?.containsKey("newsId") ?: true) + + val invalidBody = """ + { + "authorId": "VZyJz8z_Eu2", + "title": "Hello" + """.trimIndent() + + val invalidResult = OutputFormatter.JSON_FORMATTER.readFields( + invalidBody, + setOf("authorId", "title") + ) + + assertNull(invalidResult) + } + + @Test + fun testXmlReadFields() { + val body = """ + + VZyJz8z_Eu2 + 1921-3-13T10:18:56.000Z + L + Hello + + """.trimIndent() + + val result = OutputFormatter.XML_FORMATTER.readFields( + body, + setOf("authorId", "newsId") + ) + + assertNotNull(result) + assertEquals("VZyJz8z_Eu2", result?.get("authorId")) + assertEquals("L", result?.get("newsId")) + assertEquals(2, result?.size) + } + + @Test + fun testXmlReadFieldsMissingAndInvalid() { + val body = """ + + VZyJz8z_Eu2 + Hello + + """.trimIndent() + + val result = OutputFormatter.XML_FORMATTER.readFields( + body, + setOf("authorId", "newsId") + ) + + assertNotNull(result) + assertEquals("VZyJz8z_Eu2", result?.get("authorId")) + assertFalse(result?.containsKey("newsId") ?: true) + + val invalidBody = """ + + VZyJz8z_Eu2 + Hello + """.trimIndent() + + val invalidResult = OutputFormatter.XML_FORMATTER.readFields( + invalidBody, + setOf("authorId", "title") + ) + + assertNull(invalidResult) + } } From 99d0898372901020c05b05775fbf1d76b03eb5d1 Mon Sep 17 00:00:00 2001 From: Omur Sahin Date: Sat, 28 Mar 2026 16:04:58 +0300 Subject: [PATCH 13/13] increase iteration --- .../v3/httporacle/failmodification/XMLFailModificationEMTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/XMLFailModificationEMTest.kt b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/XMLFailModificationEMTest.kt index a491da04c4..f962b3e60d 100644 --- a/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/XMLFailModificationEMTest.kt +++ b/core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/httporacle/failmodification/XMLFailModificationEMTest.kt @@ -27,7 +27,7 @@ class XMLFailModificationEMTest : SpringTestBase(){ runTestHandlingFlakyAndCompilation( "XMLFailedModificationEM", - 500 + 2000 ) { args: MutableList -> setOption(args, "schemaOracles", "false")