From e719cefaafb77c08788578b2f88562882f5bc156 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Fri, 29 May 2026 22:08:38 +0200 Subject: [PATCH 01/11] refactoring --- .../core/output/TestSuiteOrganizer.kt | 290 +----------------- .../core/output/naming/NamingHelper.kt | 99 ++++++ ...ingHelperNumberedTestCaseNamingStrategy.kt | 1 - .../core/output/naming/NamingStrategy.kt | 9 +- .../core/output/sorting/SortingHelper.kt | 201 ++++++++++++ .../core/output/TestSuiteOrganizerTest.kt | 1 + 6 files changed, 308 insertions(+), 293 deletions(-) create mode 100644 core/src/main/kotlin/org/evomaster/core/output/naming/NamingHelper.kt create mode 100644 core/src/main/kotlin/org/evomaster/core/output/sorting/SortingHelper.kt diff --git a/core/src/main/kotlin/org/evomaster/core/output/TestSuiteOrganizer.kt b/core/src/main/kotlin/org/evomaster/core/output/TestSuiteOrganizer.kt index 0fd305be57..cf42514dd5 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/TestSuiteOrganizer.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/TestSuiteOrganizer.kt @@ -1,22 +1,9 @@ package org.evomaster.core.output -import org.evomaster.core.Lazy import org.evomaster.core.output.naming.TestCaseNamingStrategy +import org.evomaster.core.output.sorting.SortingHelper import org.evomaster.core.output.sorting.SortingStrategy -import org.evomaster.core.problem.graphql.GraphQLAction -import org.evomaster.core.problem.graphql.GraphQLIndividual -import org.evomaster.core.problem.httpws.HttpWsCallResult -import org.evomaster.core.problem.rest.data.HttpVerb -import org.evomaster.core.problem.rest.data.RestCallAction -import org.evomaster.core.problem.rest.data.RestIndividual -import org.evomaster.core.problem.rpc.RPCCallAction -import org.evomaster.core.problem.rpc.RPCIndividual -import org.evomaster.core.problem.webfrontend.WebIndividual -import org.evomaster.core.search.EvaluatedIndividual import org.evomaster.core.search.Solution -import org.slf4j.Logger -import org.slf4j.LoggerFactory -import kotlin.reflect.KFunction1 /** * This class is responsible to decide the order in which @@ -32,287 +19,12 @@ class TestSuiteOrganizer { private val sortingHelper = SortingHelper() - private val defaultSorting = listOf(0, 1) fun sortTests(solution: Solution<*>, namingStrategy: TestCaseNamingStrategy, testCaseSortingStrategy: SortingStrategy): List { - //sortingHelper.selectCriteriaByIndex(defaultSorting) - //TODO here in the future we will have something a bit smarter return sortingHelper.sort(solution, namingStrategy, testCaseSortingStrategy) } } -class NamingHelper { - /** - * The presence of a call with a 500 status code will be added to the test name. - */ - private fun criterion1_500 (individual: EvaluatedIndividual<*>): String{ - if (individual.seeResults().filterIsInstance().any{ it.getStatusCode() == 500 }){ - return "_with500" - } - return "" - } - - private fun criterion2_hasPost (individual: EvaluatedIndividual<*>): String{ - if(individual.individual.seeAllActions().filterIsInstance().any{it.verb == HttpVerb.POST} ){ - return "_hasPost" - } - - return "" - } - - /** - * The type of sample is added to the name. This is tied to the the [RestIndividual] and will change with a new problem. - */ - private fun criterion3_sampling(individual: EvaluatedIndividual<*>): String{ - if(individual.individual is RestIndividual) - return "_" + (individual.individual as RestIndividual).sampleType - else return "" - } - - /** - * The presence of separate steps for DB initialization will be added to the test name. This is currently tied to - * the [RestIndividual] and will change with a new problem - */ - private fun criterion4_dbInit(individual: EvaluatedIndividual<*>): String{ - if ((individual.individual is RestIndividual) && (individual.individual as RestIndividual).seeInitializingActions().isNotEmpty()){ - return "_" + "hasDbInit" - } - else return "" - } - -// private fun criterion5_partialOracle(individual: EvaluatedIndividual<*>): String{ -// var name = "" -// partialOracles.adjustName().forEach { -// if(!it.adjustName().isNullOrBlank() -// && it.generatesExpectation(individual)){ -// name = name + it.adjustName() -// } -// } -// return name -// } - -// fun setPartialOracles(partialOracles: PartialOracles){ -// this.partialOracles = partialOracles -// } - -// private var partialOracles = PartialOracles() - private var namingCriteria = listOf(::criterion1_500 ) //, ::criterion5_partialOracle) - private val availableCriteria = listOf(::criterion1_500, ::criterion2_hasPost, ::criterion3_sampling, ::criterion4_dbInit) //, ::criterion5_partialOracle) - - - fun suggestName(individual: EvaluatedIndividual<*>): String{ - return namingCriteria.map { it(individual) }.joinToString("") - } - - fun getAvailableCriteria(): List, String>> { - return availableCriteria - } - - fun selectCriteria(selected: List, String>>){ - if (availableCriteria.containsAll(selected)){ - namingCriteria = selected - } - else { - throw UnsupportedOperationException("The naming criteria chosen appear to not be supported at the moment.") - } - } - - fun selectCriteriaByIndex(selected: List){ - if (availableCriteria.indices.toList().containsAll(selected)){ - for (i in selected) - namingCriteria = availableCriteria.filterIndexed{ index, _ -> - selected.contains(index) - } as List, String>> - } - else { - throw UnsupportedOperationException("The naming criteria chosen appear to not be supported at the moment.") - } - } - - -} - - -class SortingHelper { - - companion object { - private val log: Logger = LoggerFactory.getLogger(SortingHelper::class.java) - } - - /** [maxStatusCode] sorts Evaluated individuals based on the highest status code (e.g., 500s are first). - * - * **/ - private val maxStatusCode: Comparator> = compareBy>{ ind -> - val max = ind.seeResults().filterIsInstance().maxByOrNull { it.getStatusCode()?:0 } - (max as HttpWsCallResult).getStatusCode() ?: 0 - }.reversed() - - /** - * [statusCode] sorts Evaluated individuals based on the status code, as follows: - * - first: 5xx - * - second: 2xx - * - third: 4xx - */ - - - private val statusCode: Comparator> = compareBy { ind -> - val min = ind.seeResults().filterIsInstance().minByOrNull { - it.getStatusCode()?.rem(500) ?: 0 - } - (min?.getStatusCode())?.rem(500) ?: 0 - } - - /** [maxActions] sorts Evaluated individuals based on the number of actions (most actions first). - */ - private val maxActions: Comparator> = compareBy>{ ind -> - ind.individual.seeAllActions().size - }.reversed() - - /** [minActions] sorts Evaluated individuals based on the number of actions (most actions first). - */ - private val minActions: Comparator> = compareBy { ind -> - ind.individual.seeAllActions().size - } - - /** - * dbInitSize sorts [EvaluatedIndividual] objects on the basis of the presence (and number) of db initialization actions. - * Currently, this is only supported for [RestIndividual]. - * Note, writing the comparator as [EvaluatedIndividual>] seems to break the .sortWith() later on. - */ - private val dbInitSize: Comparator> = compareBy>{ ind -> - if(ind.individual is RestIndividual) { - ind.individual.seeInitializingActions().size - } - else 0 - }.reversed() - - /** - * [coveredTargets] sorts [EvaluatedIndividual] objects on based on the higher number of covered targets. - * The purpose is to give an example of sorting based on fitness information. - */ - private val coveredTargets: Comparator> = compareBy { - it.fitness.numberOfCoveredTargets() - } - - /** - * [comparatorList] holds those comparators that are currently selected for sorting - * Note that the order of the comparators is the order their importance/priority. - */ - - var comparatorList = listOf(statusCode, coveredTargets) - - val restComparator: Comparator> = compareBy> { ind -> - (ind.evaluatedMainActions().last().action as RestCallAction).path.levels() - } - .thenBy { ind -> - val min = ind.seeResults().filterIsInstance().minByOrNull { - it.getStatusCode()?.rem(500) ?: 0 - } - (min?.getStatusCode())?.rem(500) ?: 0 - } - .thenBy { ind -> - (ind.evaluatedMainActions().last().action as RestCallAction).verb - } - - val graphQLComparator: Comparator> = compareBy> { ind -> - (ind.evaluatedMainActions().last().action as GraphQLAction).methodName - } - .thenBy { ind -> - (ind.evaluatedMainActions().last().action as GraphQLAction).methodType - } - .thenBy { ind -> - (ind.evaluatedMainActions().last().action as GraphQLAction).parameters.size - } - - val rpcComparator: Comparator> = compareBy> { ind -> - (ind.evaluatedMainActions().last().action as RPCCallAction).getSimpleClassName() - } - .thenBy { ind -> - (ind.evaluatedMainActions().last().action as RPCCallAction).getExecutedFunctionName() - } - .thenBy { ind -> - (ind.evaluatedMainActions().last().action as RPCCallAction).parameters.size - } - - private val availableSortCriteria = listOf(statusCode, minActions, coveredTargets, maxStatusCode, maxActions, dbInitSize) - - fun getAvailableCriteria(): List>> { - return availableSortCriteria - } - - fun selectCriteria(selected: List>>){ - if (availableSortCriteria.containsAll(selected)){ - comparatorList = selected - } - else { - throw UnsupportedOperationException("The sorting criteria chosen appear to not be supported at the moment.") - } - } - - fun selectCriteriaByIndex(selected: List){ - if (availableSortCriteria.indices.toList().containsAll(selected)){ - comparatorList = availableSortCriteria.filterIndexed{ index, _ -> - selected.contains(index) - } - } - else { - throw UnsupportedOperationException("The sorting criteria chosen appear to not be supported at the moment.") - } - } - - /** - *Sorting is done according to the comparator list. If no list is provided, individuals are sorted by max status. - */ - private fun sortByComparatorList (comparators: List>> = listOf(statusCode), - namingStrategy: TestCaseNamingStrategy - ): List { - /** - * Comparisons, as far as I understand them, are done as follows: - * First, the list is sorted based on the first criterion. - * Then, the (now sorted) list, is sorted based on the second criterion. - * Where two values have equal priority with respect to the most recent sort, they maintain the order (and, thus, - * are still sorted according to the first criterion). - * - * So, a criterion with more priority overrides most other criteria, unless elements have the same value. - * If too many criteria are used, the ones that are lower on the priority list will not really have a chance to manifest. - * - * An example of how this approach is used: - * = first priority (thus, last to be executed and most likely to be observed) is the [statusCode]. Thus, every - * test case that contains a 500 code is at the top. - * = second priority (thus, second to last to be executed), is the [coveredTargets]. Thus, among those test cases - * that have the same code, the ones with the most covered targets will be at the top (among their sub-group). - */ - - return namingStrategy.getSortedTestCases(comparators) - } - - private fun sortByTargetIncremental(solution: Solution<*>, namingStrategy: TestCaseNamingStrategy): List { - val individuals = solution.individuals - val comparator = when { - individuals.any { it.individual is RestIndividual } -> restComparator - individuals.any { it.individual is GraphQLIndividual } -> graphQLComparator - individuals.any { it.individual is RPCIndividual } -> rpcComparator - individuals.any { it.individual is WebIndividual } -> { - log.warn("Web individuals do not have action based test case naming yet. Defaulting to Numbered strategy.") - statusCode - } - else -> throw IllegalStateException("Unrecognized test individuals with no target incremental based sorting strategy set.") - } - - return namingStrategy.getSortedTestCases(comparator) - } - - fun sort(solution: Solution<*>, namingStrategy: TestCaseNamingStrategy, testCaseSortingStrategy: SortingStrategy): List { - val newSort = when (testCaseSortingStrategy) { - SortingStrategy.COVERED_TARGETS -> sortByComparatorList(comparatorList, namingStrategy) - SortingStrategy.TARGET_INCREMENTAL -> sortByTargetIncremental(solution, namingStrategy) - else -> throw IllegalStateException("Unrecognized sorting strategy $testCaseSortingStrategy") - } - - Lazy.assert { solution.individuals.toSet() == newSort.map { it.test }.toSet()} - return newSort - } -} diff --git a/core/src/main/kotlin/org/evomaster/core/output/naming/NamingHelper.kt b/core/src/main/kotlin/org/evomaster/core/output/naming/NamingHelper.kt new file mode 100644 index 0000000000..6cf3d497d4 --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/output/naming/NamingHelper.kt @@ -0,0 +1,99 @@ +package org.evomaster.core.output.naming + +import org.evomaster.core.problem.httpws.HttpWsCallResult +import org.evomaster.core.problem.rest.data.HttpVerb +import org.evomaster.core.problem.rest.data.RestCallAction +import org.evomaster.core.problem.rest.data.RestIndividual +import org.evomaster.core.search.EvaluatedIndividual +import kotlin.reflect.KFunction1 + +class NamingHelper { + /** + * The presence of a call with a 500 status code will be added to the test name. + */ + private fun criterion1_500 (individual: EvaluatedIndividual<*>): String{ + if (individual.seeResults().filterIsInstance().any{ it.getStatusCode() == 500 }){ + return "_with500" + } + return "" + } + + private fun criterion2_hasPost (individual: EvaluatedIndividual<*>): String{ + if(individual.individual.seeAllActions().filterIsInstance().any{it.verb == HttpVerb.POST} ){ + return "_hasPost" + } + + return "" + } + + /** + * The type of sample is added to the name. This is tied to the the [RestIndividual] and will change with a new problem. + */ + private fun criterion3_sampling(individual: EvaluatedIndividual<*>): String{ + if(individual.individual is RestIndividual) + return "_" + (individual.individual as RestIndividual).sampleType + else return "" + } + + /** + * The presence of separate steps for DB initialization will be added to the test name. This is currently tied to + * the [RestIndividual] and will change with a new problem + */ + private fun criterion4_dbInit(individual: EvaluatedIndividual<*>): String{ + if ((individual.individual is RestIndividual) && (individual.individual as RestIndividual).seeInitializingActions().isNotEmpty()){ + return "_" + "hasDbInit" + } + else return "" + } + +// private fun criterion5_partialOracle(individual: EvaluatedIndividual<*>): String{ +// var name = "" +// partialOracles.adjustName().forEach { +// if(!it.adjustName().isNullOrBlank() +// && it.generatesExpectation(individual)){ +// name = name + it.adjustName() +// } +// } +// return name +// } + +// fun setPartialOracles(partialOracles: PartialOracles){ +// this.partialOracles = partialOracles +// } + + // private var partialOracles = PartialOracles() + private var namingCriteria = listOf(::criterion1_500 ) //, ::criterion5_partialOracle) + private val availableCriteria = listOf(::criterion1_500, ::criterion2_hasPost, ::criterion3_sampling, ::criterion4_dbInit) //, ::criterion5_partialOracle) + + + fun suggestName(individual: EvaluatedIndividual<*>): String{ + return namingCriteria.map { it(individual) }.joinToString("") + } + + fun getAvailableCriteria(): List, String>> { + return availableCriteria + } + + fun selectCriteria(selected: List, String>>){ + if (availableCriteria.containsAll(selected)){ + namingCriteria = selected + } + else { + throw UnsupportedOperationException("The naming criteria chosen appear to not be supported at the moment.") + } + } + + fun selectCriteriaByIndex(selected: List){ + if (availableCriteria.indices.toList().containsAll(selected)){ + for (i in selected) + namingCriteria = availableCriteria.filterIndexed{ index, _ -> + selected.contains(index) + } as List, String>> + } + else { + throw UnsupportedOperationException("The naming criteria chosen appear to not be supported at the moment.") + } + } + + +} diff --git a/core/src/main/kotlin/org/evomaster/core/output/naming/NamingHelperNumberedTestCaseNamingStrategy.kt b/core/src/main/kotlin/org/evomaster/core/output/naming/NamingHelperNumberedTestCaseNamingStrategy.kt index 42bfa2d9a9..962c8b2575 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/naming/NamingHelperNumberedTestCaseNamingStrategy.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/naming/NamingHelperNumberedTestCaseNamingStrategy.kt @@ -1,6 +1,5 @@ package org.evomaster.core.output.naming -import org.evomaster.core.output.NamingHelper import org.evomaster.core.search.EvaluatedIndividual import org.evomaster.core.search.Solution diff --git a/core/src/main/kotlin/org/evomaster/core/output/naming/NamingStrategy.kt b/core/src/main/kotlin/org/evomaster/core/output/naming/NamingStrategy.kt index fbb023d6a0..cf90e56d20 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/naming/NamingStrategy.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/naming/NamingStrategy.kt @@ -3,10 +3,13 @@ package org.evomaster.core.output.naming enum class NamingStrategy { NUMBERED, - ACTION + ACTION, + LLM ; - fun isNumbered() = this.name.startsWith("numbered", true) + fun isNumbered() = this == NUMBERED - fun isAction() = this.name.startsWith("action", true) + fun isAction() = this == ACTION + + fun isLLM() = this == LLM } diff --git a/core/src/main/kotlin/org/evomaster/core/output/sorting/SortingHelper.kt b/core/src/main/kotlin/org/evomaster/core/output/sorting/SortingHelper.kt new file mode 100644 index 0000000000..cd8d93fe05 --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/output/sorting/SortingHelper.kt @@ -0,0 +1,201 @@ +package org.evomaster.core.output.sorting + +import org.evomaster.core.Lazy +import org.evomaster.core.output.TestCase +import org.evomaster.core.output.naming.TestCaseNamingStrategy +import org.evomaster.core.problem.graphql.GraphQLAction +import org.evomaster.core.problem.graphql.GraphQLIndividual +import org.evomaster.core.problem.httpws.HttpWsCallResult +import org.evomaster.core.problem.rest.data.RestCallAction +import org.evomaster.core.problem.rest.data.RestIndividual +import org.evomaster.core.problem.rpc.RPCCallAction +import org.evomaster.core.problem.rpc.RPCIndividual +import org.evomaster.core.problem.webfrontend.WebIndividual +import org.evomaster.core.search.EvaluatedIndividual +import org.evomaster.core.search.Solution +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +class SortingHelper { + + companion object { + private val log: Logger = LoggerFactory.getLogger(SortingHelper::class.java) + } + + /** [maxStatusCode] sorts Evaluated individuals based on the highest status code (e.g., 500s are first). + * + * **/ + private val maxStatusCode: Comparator> = compareBy>{ ind -> + val max = ind.seeResults().filterIsInstance().maxByOrNull { it.getStatusCode()?:0 } + (max as HttpWsCallResult).getStatusCode() ?: 0 + }.reversed() + + /** + * [statusCode] sorts Evaluated individuals based on the status code, as follows: + * - first: 5xx + * - second: 2xx + * - third: 4xx + */ + + + private val statusCode: Comparator> = compareBy { ind -> + val min = ind.seeResults().filterIsInstance().minByOrNull { + it.getStatusCode()?.rem(500) ?: 0 + } + (min?.getStatusCode())?.rem(500) ?: 0 + } + + /** [maxActions] sorts Evaluated individuals based on the number of actions (most actions first). + */ + private val maxActions: Comparator> = compareBy>{ ind -> + ind.individual.seeAllActions().size + }.reversed() + + /** [minActions] sorts Evaluated individuals based on the number of actions (most actions first). + */ + private val minActions: Comparator> = compareBy { ind -> + ind.individual.seeAllActions().size + } + + /** + * dbInitSize sorts [EvaluatedIndividual] objects on the basis of the presence (and number) of db initialization actions. + * Currently, this is only supported for [RestIndividual]. + * Note, writing the comparator as [EvaluatedIndividual>] seems to break the .sortWith() later on. + */ + private val dbInitSize: Comparator> = compareBy>{ ind -> + if(ind.individual is RestIndividual) { + ind.individual.seeInitializingActions().size + } + else 0 + }.reversed() + + /** + * [coveredTargets] sorts [EvaluatedIndividual] objects on based on the higher number of covered targets. + * The purpose is to give an example of sorting based on fitness information. + */ + private val coveredTargets: Comparator> = compareBy { + it.fitness.numberOfCoveredTargets() + } + + /** + * [comparatorList] holds those comparators that are currently selected for sorting + * Note that the order of the comparators is the order their importance/priority. + */ + + var comparatorList = listOf(statusCode, coveredTargets) + + val restComparator: Comparator> = compareBy> { ind -> + (ind.evaluatedMainActions().last().action as RestCallAction).path.levels() + } + .thenBy { ind -> + val min = ind.seeResults().filterIsInstance().minByOrNull { + it.getStatusCode()?.rem(500) ?: 0 + } + (min?.getStatusCode())?.rem(500) ?: 0 + } + .thenBy { ind -> + (ind.evaluatedMainActions().last().action as RestCallAction).verb + } + + val graphQLComparator: Comparator> = compareBy> { ind -> + (ind.evaluatedMainActions().last().action as GraphQLAction).methodName + } + .thenBy { ind -> + (ind.evaluatedMainActions().last().action as GraphQLAction).methodType + } + .thenBy { ind -> + (ind.evaluatedMainActions().last().action as GraphQLAction).parameters.size + } + + val rpcComparator: Comparator> = compareBy> { ind -> + (ind.evaluatedMainActions().last().action as RPCCallAction).getSimpleClassName() + } + .thenBy { ind -> + (ind.evaluatedMainActions().last().action as RPCCallAction).getExecutedFunctionName() + } + .thenBy { ind -> + (ind.evaluatedMainActions().last().action as RPCCallAction).parameters.size + } + + private val availableSortCriteria = listOf(statusCode, minActions, coveredTargets, maxStatusCode, maxActions, dbInitSize) + + + + fun getAvailableCriteria(): List>> { + return availableSortCriteria + } + + fun selectCriteria(selected: List>>){ + if (availableSortCriteria.containsAll(selected)){ + comparatorList = selected + } + else { + throw UnsupportedOperationException("The sorting criteria chosen appear to not be supported at the moment.") + } + } + + fun selectCriteriaByIndex(selected: List){ + if (availableSortCriteria.indices.toList().containsAll(selected)){ + comparatorList = availableSortCriteria.filterIndexed{ index, _ -> + selected.contains(index) + } + } + else { + throw UnsupportedOperationException("The sorting criteria chosen appear to not be supported at the moment.") + } + } + + /** + *Sorting is done according to the comparator list. If no list is provided, individuals are sorted by max status. + */ + private fun sortByComparatorList (comparators: List>> = listOf(statusCode), + namingStrategy: TestCaseNamingStrategy + + ): List { + /** + * Comparisons, as far as I understand them, are done as follows: + * First, the list is sorted based on the first criterion. + * Then, the (now sorted) list, is sorted based on the second criterion. + * Where two values have equal priority with respect to the most recent sort, they maintain the order (and, thus, + * are still sorted according to the first criterion). + * + * So, a criterion with more priority overrides most other criteria, unless elements have the same value. + * If too many criteria are used, the ones that are lower on the priority list will not really have a chance to manifest. + * + * An example of how this approach is used: + * = first priority (thus, last to be executed and most likely to be observed) is the [statusCode]. Thus, every + * test case that contains a 500 code is at the top. + * = second priority (thus, second to last to be executed), is the [coveredTargets]. Thus, among those test cases + * that have the same code, the ones with the most covered targets will be at the top (among their sub-group). + */ + + return namingStrategy.getSortedTestCases(comparators) + } + + private fun sortByTargetIncremental(solution: Solution<*>, namingStrategy: TestCaseNamingStrategy): List { + val individuals = solution.individuals + val comparator = when { + individuals.any { it.individual is RestIndividual } -> restComparator + individuals.any { it.individual is GraphQLIndividual } -> graphQLComparator + individuals.any { it.individual is RPCIndividual } -> rpcComparator + individuals.any { it.individual is WebIndividual } -> { + log.warn("Web individuals do not have action based test case naming yet. Defaulting to Numbered strategy.") + statusCode + } + else -> throw IllegalStateException("Unrecognized test individuals with no target incremental based sorting strategy set.") + } + + return namingStrategy.getSortedTestCases(comparator) + } + + fun sort(solution: Solution<*>, namingStrategy: TestCaseNamingStrategy, testCaseSortingStrategy: SortingStrategy): List { + val newSort = when (testCaseSortingStrategy) { + SortingStrategy.COVERED_TARGETS -> sortByComparatorList(comparatorList, namingStrategy) + SortingStrategy.TARGET_INCREMENTAL -> sortByTargetIncremental(solution, namingStrategy) + else -> throw IllegalStateException("Unrecognized sorting strategy $testCaseSortingStrategy") + } + + Lazy.assert { solution.individuals.toSet() == newSort.map { it.test }.toSet()} + return newSort + } +} \ No newline at end of file diff --git a/core/src/test/kotlin/org/evomaster/core/output/TestSuiteOrganizerTest.kt b/core/src/test/kotlin/org/evomaster/core/output/TestSuiteOrganizerTest.kt index 55075ba0e6..8adef7e4c5 100644 --- a/core/src/test/kotlin/org/evomaster/core/output/TestSuiteOrganizerTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/output/TestSuiteOrganizerTest.kt @@ -4,6 +4,7 @@ import org.evomaster.core.TestUtils import org.evomaster.core.output.naming.NumberedTestCaseNamingStrategy import org.evomaster.core.output.naming.RestActionTestCaseUtils.getEvaluatedIndividualWith import org.evomaster.core.output.naming.RestActionTestCaseUtils.getRestCallAction +import org.evomaster.core.output.sorting.SortingHelper import org.evomaster.core.output.sorting.SortingStrategy import org.evomaster.core.problem.api.param.Param import org.evomaster.core.problem.enterprise.EnterpriseActionGroup From 25c79ccadce55b9b799b69d426fbe8fd286f09f2 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Fri, 29 May 2026 22:45:23 +0200 Subject: [PATCH 02/11] cleaning --- .../spring/examples/sort/SortEMTest.java | 2 +- .../core/output/TestSuiteOrganizer.kt | 9 ++- .../naming/NumberedTestCaseNamingStrategy.kt | 12 ---- .../output/naming/TestCaseNamingStrategy.kt | 13 ---- .../core/output/service/TestSuiteWriter.kt | 2 +- .../core/output/sorting/SortingHelper.kt | 62 +++++++++++++------ 6 files changed, 53 insertions(+), 47 deletions(-) diff --git a/core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/sort/SortEMTest.java b/core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/sort/SortEMTest.java index 4c1d1f5402..dcff4e20e4 100644 --- a/core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/sort/SortEMTest.java +++ b/core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/sort/SortEMTest.java @@ -47,7 +47,7 @@ public void testRunEM() throws Throwable { TestCaseNamingStrategy namingStrategy = new NumberedTestCaseNamingStrategy(solution); - List tclist = organizer.sortTests(solution, namingStrategy, SortingStrategy.COVERED_TARGETS); + List tclist = organizer.createSortedTestCases(namingStrategy, SortingStrategy.COVERED_TARGETS); //Iterator iterator = tclist.iterator(); //TestCase current, previous = iterator.next(); diff --git a/core/src/main/kotlin/org/evomaster/core/output/TestSuiteOrganizer.kt b/core/src/main/kotlin/org/evomaster/core/output/TestSuiteOrganizer.kt index cf42514dd5..d762c7dcac 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/TestSuiteOrganizer.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/TestSuiteOrganizer.kt @@ -20,9 +20,14 @@ class TestSuiteOrganizer { private val sortingHelper = SortingHelper() - fun sortTests(solution: Solution<*>, namingStrategy: TestCaseNamingStrategy, testCaseSortingStrategy: SortingStrategy): List { - return sortingHelper.sort(solution, namingStrategy, testCaseSortingStrategy) + fun createSortedTestCases(namingStrategy: TestCaseNamingStrategy, testCaseSortingStrategy: SortingStrategy): List { + + val tests = namingStrategy.getTestCases() + + return sortingHelper.sort(tests, testCaseSortingStrategy) } + + } diff --git a/core/src/main/kotlin/org/evomaster/core/output/naming/NumberedTestCaseNamingStrategy.kt b/core/src/main/kotlin/org/evomaster/core/output/naming/NumberedTestCaseNamingStrategy.kt index da35a4e105..07f1473355 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/naming/NumberedTestCaseNamingStrategy.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/naming/NumberedTestCaseNamingStrategy.kt @@ -13,19 +13,7 @@ open class NumberedTestCaseNamingStrategy( return generateNames(solution.individuals) } - override fun getSortedTestCases(comparator: Comparator>): List { - return getSortedTestCases(singletonList(comparator)) - } - - override fun getSortedTestCases(comparators: List>>): List { - val inds = solution.individuals - comparators.asReversed().forEach { - inds.sortWith(it) - } - - return generateNames(inds) - } // numbered strategy will not expand the name unless it is using the namingHelper override fun expandName(individual: EvaluatedIndividual<*>, nameTokens: MutableList, ambiguitySolvers: List): String { diff --git a/core/src/main/kotlin/org/evomaster/core/output/naming/TestCaseNamingStrategy.kt b/core/src/main/kotlin/org/evomaster/core/output/naming/TestCaseNamingStrategy.kt index fccb6b2b0a..24d997526c 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/naming/TestCaseNamingStrategy.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/naming/TestCaseNamingStrategy.kt @@ -17,19 +17,6 @@ abstract class TestCaseNamingStrategy( */ abstract fun getTestCases(): List - /** - * @param comparator used to sort the test cases - * - * @return the list of sorted TestCase with the generated name given the naming strategy - */ - abstract fun getSortedTestCases(comparator: Comparator>): List - - /** - * @param comparators used to sort the test cases - * - * @return the list of sorted TestCase with the generated name given the naming strategy - */ - abstract fun getSortedTestCases(comparators: List>>): List /** * @param individual containing information for the test about to be named diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt index fe08c169ec..db5673b6ba 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt @@ -156,7 +156,7 @@ class TestSuiteWriter { //catch any sorting problems (see NPE is SortingHelper on Trello) val tests = try { // TODO skip to sort RPC for the moment - testSuiteOrganizer.sortTests(solution, namingStrategy, config.testCaseSortingStrategy) + testSuiteOrganizer.createSortedTestCases(namingStrategy, config.testCaseSortingStrategy) } catch (ex: Exception) { log.warn( "A failure has occurred with the test sorting. Reverting to default settings. \n" diff --git a/core/src/main/kotlin/org/evomaster/core/output/sorting/SortingHelper.kt b/core/src/main/kotlin/org/evomaster/core/output/sorting/SortingHelper.kt index cd8d93fe05..a793889e03 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/sorting/SortingHelper.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/sorting/SortingHelper.kt @@ -15,6 +15,7 @@ import org.evomaster.core.search.EvaluatedIndividual import org.evomaster.core.search.Solution import org.slf4j.Logger import org.slf4j.LoggerFactory +import java.util.Collections.singletonList class SortingHelper { @@ -22,6 +23,8 @@ class SortingHelper { private val log: Logger = LoggerFactory.getLogger(SortingHelper::class.java) } + + /** [maxStatusCode] sorts Evaluated individuals based on the highest status code (e.g., 500s are first). * * **/ @@ -120,6 +123,20 @@ class SortingHelper { private val availableSortCriteria = listOf(statusCode, minActions, coveredTargets, maxStatusCode, maxActions, dbInitSize) + fun sort(tests: List, testCaseSortingStrategy: SortingStrategy): List { + val newSort = when (testCaseSortingStrategy) { + SortingStrategy.COVERED_TARGETS -> sortByComparatorList(comparatorList, tests) + SortingStrategy.TARGET_INCREMENTAL -> sortByTargetIncremental(tests) + else -> throw IllegalStateException("Unrecognized sorting strategy $testCaseSortingStrategy") + } + + return newSort + } + + @Deprecated("Use other version") + fun sort(solution: Solution<*>, namingStrategy: TestCaseNamingStrategy, testCaseSortingStrategy: SortingStrategy) : List { + return sort(namingStrategy.getTestCases(), testCaseSortingStrategy) + } fun getAvailableCriteria(): List>> { return availableSortCriteria @@ -145,11 +162,29 @@ class SortingHelper { } } + private fun getSortedTestCases(tests: List, comparator: Comparator>): List { + return getSortedTestCases(tests, singletonList(comparator)) + } + + private fun getSortedTestCases(tests: List, comparators: List>>): List { + + val copy = tests.toMutableList() + + comparators.asReversed().forEach { comp -> + val w = Comparator{a, b -> comp.compare(a.test,b.test)} + copy.sortWith(w) + } + + return copy + } + + + /** *Sorting is done according to the comparator list. If no list is provided, individuals are sorted by max status. */ private fun sortByComparatorList (comparators: List>> = listOf(statusCode), - namingStrategy: TestCaseNamingStrategy + tests: List ): List { /** @@ -169,33 +204,24 @@ class SortingHelper { * that have the same code, the ones with the most covered targets will be at the top (among their sub-group). */ - return namingStrategy.getSortedTestCases(comparators) + return getSortedTestCases(tests, comparators) } - private fun sortByTargetIncremental(solution: Solution<*>, namingStrategy: TestCaseNamingStrategy): List { - val individuals = solution.individuals + private fun sortByTargetIncremental(tests: List): List { + val comparator = when { - individuals.any { it.individual is RestIndividual } -> restComparator - individuals.any { it.individual is GraphQLIndividual } -> graphQLComparator - individuals.any { it.individual is RPCIndividual } -> rpcComparator - individuals.any { it.individual is WebIndividual } -> { + tests.any { it.test.individual is RestIndividual } -> restComparator + tests.any { it.test.individual is GraphQLIndividual } -> graphQLComparator + tests.any { it.test.individual is RPCIndividual } -> rpcComparator + tests.any { it.test.individual is WebIndividual } -> { log.warn("Web individuals do not have action based test case naming yet. Defaulting to Numbered strategy.") statusCode } else -> throw IllegalStateException("Unrecognized test individuals with no target incremental based sorting strategy set.") } - return namingStrategy.getSortedTestCases(comparator) + return getSortedTestCases(tests, comparator) } - fun sort(solution: Solution<*>, namingStrategy: TestCaseNamingStrategy, testCaseSortingStrategy: SortingStrategy): List { - val newSort = when (testCaseSortingStrategy) { - SortingStrategy.COVERED_TARGETS -> sortByComparatorList(comparatorList, namingStrategy) - SortingStrategy.TARGET_INCREMENTAL -> sortByTargetIncremental(solution, namingStrategy) - else -> throw IllegalStateException("Unrecognized sorting strategy $testCaseSortingStrategy") - } - Lazy.assert { solution.individuals.toSet() == newSort.map { it.test }.toSet()} - return newSort - } } \ No newline at end of file From c74c9081e270fb4cc20aab3729634cd53479aaf0 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Fri, 29 May 2026 22:51:35 +0200 Subject: [PATCH 03/11] more cleaning --- .../core/output/TestSuiteOrganizer.kt | 27 ++++++++------- .../core/output/service/TestSuiteWriter.kt | 14 +------- .../core/output/sorting/SortingHelper.kt | 33 +++---------------- 3 files changed, 20 insertions(+), 54 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/output/TestSuiteOrganizer.kt b/core/src/main/kotlin/org/evomaster/core/output/TestSuiteOrganizer.kt index d762c7dcac..07c33e24dd 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/TestSuiteOrganizer.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/TestSuiteOrganizer.kt @@ -3,23 +3,26 @@ package org.evomaster.core.output import org.evomaster.core.output.naming.TestCaseNamingStrategy import org.evomaster.core.output.sorting.SortingHelper import org.evomaster.core.output.sorting.SortingStrategy -import org.evomaster.core.search.Solution - -/** - * This class is responsible to decide the order in which - * the test cases should be written in the final test suite. - * Ideally, the most "interesting" tests should be written first. - * - *
- * - * Furthermore, this class is also responsible for deciding which - * name each test will have - */ + + class TestSuiteOrganizer { private val sortingHelper = SortingHelper() + /** + * This method is responsible to decide the order in which + * the test cases should be written in the final test suite. + * Ideally, the most "interesting" tests should be written first. + * + *
+ * + * Furthermore, this class is also responsible for deciding which + * name each test will have. + * + *
+ * Note that the 'solution' is stored insider the [namingStrategy] + */ fun createSortedTestCases(namingStrategy: TestCaseNamingStrategy, testCaseSortingStrategy: SortingStrategy): List { val tests = namingStrategy.getTestCases() diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt index db5673b6ba..1f57ffe53d 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt @@ -153,19 +153,7 @@ class TestSuiteWriter { beforeAfterMethods(solution, controllerName, controllerInput, lines, config.outputFormat, testSuiteFileName) - //catch any sorting problems (see NPE is SortingHelper on Trello) - val tests = try { - // TODO skip to sort RPC for the moment - testSuiteOrganizer.createSortedTestCases(namingStrategy, config.testCaseSortingStrategy) - } catch (ex: Exception) { - log.warn( - "A failure has occurred with the test sorting. Reverting to default settings. \n" - + "Exception: ${ex.localizedMessage} \n" - + "At ${ex.stackTrace.joinToString(separator = " \n -> ")}. " - ) - // fallback to numbered naming strategy upon failure - NumberedTestCaseNamingStrategy(solution).getTestCases() - } + val tests = testSuiteOrganizer.createSortedTestCases(namingStrategy, config.testCaseSortingStrategy) val testSuitePath = getTestSuitePath(testSuiteFileName, config) diff --git a/core/src/main/kotlin/org/evomaster/core/output/sorting/SortingHelper.kt b/core/src/main/kotlin/org/evomaster/core/output/sorting/SortingHelper.kt index a793889e03..5a06cc8ecd 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/sorting/SortingHelper.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/sorting/SortingHelper.kt @@ -85,9 +85,9 @@ class SortingHelper { * Note that the order of the comparators is the order their importance/priority. */ - var comparatorList = listOf(statusCode, coveredTargets) + private val comparatorList = listOf(statusCode, coveredTargets) - val restComparator: Comparator> = compareBy> { ind -> + private val restComparator: Comparator> = compareBy> { ind -> (ind.evaluatedMainActions().last().action as RestCallAction).path.levels() } .thenBy { ind -> @@ -100,7 +100,7 @@ class SortingHelper { (ind.evaluatedMainActions().last().action as RestCallAction).verb } - val graphQLComparator: Comparator> = compareBy> { ind -> + private val graphQLComparator: Comparator> = compareBy> { ind -> (ind.evaluatedMainActions().last().action as GraphQLAction).methodName } .thenBy { ind -> @@ -110,7 +110,7 @@ class SortingHelper { (ind.evaluatedMainActions().last().action as GraphQLAction).parameters.size } - val rpcComparator: Comparator> = compareBy> { ind -> + private val rpcComparator: Comparator> = compareBy> { ind -> (ind.evaluatedMainActions().last().action as RPCCallAction).getSimpleClassName() } .thenBy { ind -> @@ -120,8 +120,6 @@ class SortingHelper { (ind.evaluatedMainActions().last().action as RPCCallAction).parameters.size } - private val availableSortCriteria = listOf(statusCode, minActions, coveredTargets, maxStatusCode, maxActions, dbInitSize) - fun sort(tests: List, testCaseSortingStrategy: SortingStrategy): List { val newSort = when (testCaseSortingStrategy) { @@ -138,29 +136,6 @@ class SortingHelper { return sort(namingStrategy.getTestCases(), testCaseSortingStrategy) } - fun getAvailableCriteria(): List>> { - return availableSortCriteria - } - - fun selectCriteria(selected: List>>){ - if (availableSortCriteria.containsAll(selected)){ - comparatorList = selected - } - else { - throw UnsupportedOperationException("The sorting criteria chosen appear to not be supported at the moment.") - } - } - - fun selectCriteriaByIndex(selected: List){ - if (availableSortCriteria.indices.toList().containsAll(selected)){ - comparatorList = availableSortCriteria.filterIndexed{ index, _ -> - selected.contains(index) - } - } - else { - throw UnsupportedOperationException("The sorting criteria chosen appear to not be supported at the moment.") - } - } private fun getSortedTestCases(tests: List, comparator: Comparator>): List { return getSortedTestCases(tests, singletonList(comparator)) From 80045d498a5454f9a6be713eda3d3448e1b75917 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Fri, 29 May 2026 23:01:14 +0200 Subject: [PATCH 04/11] yet more cleaning --- .../spring/examples/sort/SortEMTest.java | 17 ++++++----------- .../evomaster/core/output/TestSuiteOrganizer.kt | 17 ++++++++++------- .../core/output/service/TestSuiteWriter.kt | 5 ++--- 3 files changed, 18 insertions(+), 21 deletions(-) diff --git a/core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/sort/SortEMTest.java b/core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/sort/SortEMTest.java index dcff4e20e4..1b7cf5468a 100644 --- a/core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/sort/SortEMTest.java +++ b/core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/sort/SortEMTest.java @@ -1,10 +1,6 @@ package org.evomaster.e2etests.spring.examples.sort; -import org.evomaster.core.output.TestCase; -import org.evomaster.core.output.TestSuiteOrganizer; -import org.evomaster.core.output.naming.NumberedTestCaseNamingStrategy; -import org.evomaster.core.output.naming.TestCaseNamingStrategy; -import org.evomaster.core.output.sorting.SortingStrategy; + import org.evomaster.core.problem.rest.data.HttpVerb; import org.evomaster.core.problem.rest.data.RestCallResult; import org.evomaster.core.problem.rest.data.RestIndividual; @@ -14,7 +10,6 @@ import org.junit.jupiter.api.Test; import java.util.Iterator; -import java.util.List; import java.util.OptionalInt; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -43,11 +38,11 @@ public void testRunEM() throws Throwable { assertHasAtLeastOne(solution, HttpVerb.PUT, 500); assertHasAtLeastOne(solution, HttpVerb.POST, 500); - TestSuiteOrganizer organizer = new TestSuiteOrganizer(); - - TestCaseNamingStrategy namingStrategy = new NumberedTestCaseNamingStrategy(solution); - - List tclist = organizer.createSortedTestCases(namingStrategy, SortingStrategy.COVERED_TARGETS); +// TestSuiteOrganizer organizer = new TestSuiteOrganizer(); +// +// TestCaseNamingStrategy namingStrategy = new NumberedTestCaseNamingStrategy(solution); +// +// List tclist = organizer.createSortedTestCases(namingStrategy, SortingStrategy.COVERED_TARGETS); //Iterator iterator = tclist.iterator(); //TestCase current, previous = iterator.next(); diff --git a/core/src/main/kotlin/org/evomaster/core/output/TestSuiteOrganizer.kt b/core/src/main/kotlin/org/evomaster/core/output/TestSuiteOrganizer.kt index 07c33e24dd..c4565b85e0 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/TestSuiteOrganizer.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/TestSuiteOrganizer.kt @@ -1,11 +1,14 @@ package org.evomaster.core.output -import org.evomaster.core.output.naming.TestCaseNamingStrategy +import org.evomaster.core.EMConfig +import org.evomaster.core.output.naming.TestCaseNamingStrategyFactory import org.evomaster.core.output.sorting.SortingHelper -import org.evomaster.core.output.sorting.SortingStrategy +import org.evomaster.core.search.Solution -class TestSuiteOrganizer { +class TestSuiteOrganizer( + private val config: EMConfig +) { private val sortingHelper = SortingHelper() @@ -20,14 +23,14 @@ class TestSuiteOrganizer { * Furthermore, this class is also responsible for deciding which * name each test will have. * - *
- * Note that the 'solution' is stored insider the [namingStrategy] */ - fun createSortedTestCases(namingStrategy: TestCaseNamingStrategy, testCaseSortingStrategy: SortingStrategy): List { + fun createSortedTestCases(solution: Solution<*>): List { + + val namingStrategy = TestCaseNamingStrategyFactory(config).create(solution) val tests = namingStrategy.getTestCases() - return sortingHelper.sort(tests, testCaseSortingStrategy) + return sortingHelper.sort(tests, config.testCaseSortingStrategy) } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt index 1f57ffe53d..b2e0ba03f7 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt @@ -137,8 +137,7 @@ class TestSuiteWriter { ): TestSuiteCode { val lines = Lines(config.outputFormat) - val testSuiteOrganizer = TestSuiteOrganizer() - val namingStrategy = TestCaseNamingStrategyFactory(config).create(solution) + val testSuiteOrganizer = TestSuiteOrganizer(config) header(solution, testSuiteFileName, lines, timestamp, controllerName) @@ -153,7 +152,7 @@ class TestSuiteWriter { beforeAfterMethods(solution, controllerName, controllerInput, lines, config.outputFormat, testSuiteFileName) - val tests = testSuiteOrganizer.createSortedTestCases(namingStrategy, config.testCaseSortingStrategy) + val tests = testSuiteOrganizer.createSortedTestCases(solution) val testSuitePath = getTestSuitePath(testSuiteFileName, config) From a2734746531f698398a4b69eb70956979a6aee23 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Sat, 30 May 2026 20:40:15 +0200 Subject: [PATCH 05/11] more fixes. put back try/catch --- .../core/output/service/TestSuiteWriter.kt | 16 +++++++++++++++- .../core/output/sorting/SortingHelper.kt | 4 ++++ docs/options.md | 2 +- 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt index b2e0ba03f7..071f6ef3b3 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt @@ -152,7 +152,21 @@ class TestSuiteWriter { beforeAfterMethods(solution, controllerName, controllerInput, lines, config.outputFormat, testSuiteFileName) - val tests = testSuiteOrganizer.createSortedTestCases(solution) + //FIXME should solve all problems that happen in the EM tests + //catch any sorting problems (see NPE is SortingHelper on Trello) + val tests = try { + // TODO skip to sort RPC for the moment + testSuiteOrganizer.createSortedTestCases(solution) + } catch (ex: Exception) { + log.warn( + "A failure has occurred with the test sorting. Reverting to default settings. \n" + + "Exception: ${ex.localizedMessage} \n" + + "At ${ex.stackTrace.joinToString(separator = " \n -> ")}. " + ) + // fallback to numbered naming strategy upon failure + NumberedTestCaseNamingStrategy(solution).getTestCases() + } + val testSuitePath = getTestSuitePath(testSuiteFileName, config) diff --git a/core/src/main/kotlin/org/evomaster/core/output/sorting/SortingHelper.kt b/core/src/main/kotlin/org/evomaster/core/output/sorting/SortingHelper.kt index 5a06cc8ecd..ec049847e3 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/sorting/SortingHelper.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/sorting/SortingHelper.kt @@ -184,6 +184,10 @@ class SortingHelper { private fun sortByTargetIncremental(tests: List): List { + if(tests.isEmpty()){ + return emptyList() + } + val comparator = when { tests.any { it.test.individual is RestIndividual } -> restComparator tests.any { it.test.individual is GraphQLIndividual } -> graphQLComparator diff --git a/docs/options.md b/docs/options.md index 75c6de0945..ecefec8e7c 100644 --- a/docs/options.md +++ b/docs/options.md @@ -178,7 +178,7 @@ There are 3 types of options: |`muPlusLambdaOffspringSize`| __Int__. Define the number of offspring (λ) generated per generation in (μ+λ) Evolutionary Algorithm. *Constraints*: `min=1.0`. *Default value*: `30`.| |`mutatedGeneFile`| __String__. Specify a path to save mutation details which is useful for debugging mutation. *DEBUG option*. *Default value*: `mutatedGeneInfo.csv`.| |`nameWithQueryParameters`| __Boolean__. Specify if true boolean query parameters are included in the test case name. Used for test case naming disambiguation. Only valid for Action based naming strategy. *Default value*: `true`.| -|`namingStrategy`| __Enum__. Specify the naming strategy for test cases. *Valid values*: `NUMBERED, ACTION`. *Default value*: `ACTION`.| +|`namingStrategy`| __Enum__. Specify the naming strategy for test cases. *Valid values*: `NUMBERED, ACTION, LLM`. *Default value*: `ACTION`.| |`outputExecutedSQL`| __Enum__. Whether to output executed sql info. *DEBUG option*. *Valid values*: `NONE, ALL_AT_END, ONCE_EXECUTED`. *Default value*: `NONE`.| |`overrideAuthExternalEndpointURL`| __String__. Override the value of externalEndpointURL in auth configurations. This is useful when the auth server is running locally on an ephemeral port, or when several instances are run in parallel, to avoid creating/modifying auth configuration files. If what provided is a URL starting with 'http', then full replacement will occur. Otherwise, the input will be treated as a 'hostname:port', and only that info will be updated (e.g., path element of the URL will not change). *Default value*: `null`.| |`populationSize`| __Int__. Define the population size in the search algorithms that use populations (e.g., Genetic Algorithms, but not MIO). *Constraints*: `min=1.0`. *Default value*: `30`.| From a629d1699dcb299958d6e0409e2c25e67acc37b4 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Sat, 30 May 2026 20:52:57 +0200 Subject: [PATCH 06/11] more cleaning --- .../kotlin/org/evomaster/core/EMConfig.kt | 6 ++++- .../core/output/naming/NamingStrategy.kt | 23 +++++++++++++------ .../naming/TestCaseNamingStrategyFactory.kt | 19 +++++++-------- .../kotlin/org/evomaster/core/EMConfigTest.kt | 6 ++--- 4 files changed, 34 insertions(+), 20 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index 6bda393e9d..cc041e3535 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -79,7 +79,7 @@ class EMConfig { private val defaultOutputFormatForBlackBox = OutputFormat.PYTHON_UNITTEST - private val defaultTestCaseNamingStrategy = NamingStrategy.ACTION + private val defaultTestCaseNamingStrategy = NamingStrategy.DETERMINISTIC private val defaultTestCaseSortingStrategy = SortingStrategy.TARGET_INCREMENTAL @@ -870,6 +870,10 @@ class EMConfig { if (sutJarEnvVarName.isEmpty()) throw ConfigProblemException("'sutJarEnvVarName' must be specified if 'useEnvVarsForPathInTests' is enabled.") } + + if(namingStrategy == NamingStrategy.LLM && !llm){ + throw ConfigProblemException("Naming strategy LLM require the setup and use of an LLM") + } } private fun checkPropertyConstraints(m: KMutableProperty<*>) { diff --git a/core/src/main/kotlin/org/evomaster/core/output/naming/NamingStrategy.kt b/core/src/main/kotlin/org/evomaster/core/output/naming/NamingStrategy.kt index cf90e56d20..8ab14b038b 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/naming/NamingStrategy.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/naming/NamingStrategy.kt @@ -1,15 +1,24 @@ package org.evomaster.core.output.naming +/** + * + */ enum class NamingStrategy { + /** + * Standard, naive approach. + * Each test gets a unique, incremental number + */ NUMBERED, - ACTION, - LLM - ; - - fun isNumbered() = this == NUMBERED - fun isAction() = this == ACTION + /** + * Apply a deterministic set of rules based on the actions' content. + */ + DETERMINISTIC, - fun isLLM() = this == LLM + /** + * Call an LLM to create the names based on the tests' content, if available + */ + LLM + ; } diff --git a/core/src/main/kotlin/org/evomaster/core/output/naming/TestCaseNamingStrategyFactory.kt b/core/src/main/kotlin/org/evomaster/core/output/naming/TestCaseNamingStrategyFactory.kt index de5766b513..50d46b7b86 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/naming/TestCaseNamingStrategyFactory.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/naming/TestCaseNamingStrategyFactory.kt @@ -24,26 +24,27 @@ class TestCaseNamingStrategyFactory( } fun create(solution: Solution<*>): TestCaseNamingStrategy { - return when { - namingStrategy.isNumbered() -> NamingHelperNumberedTestCaseNamingStrategy(solution) - namingStrategy.isAction() -> actionBasedNamingStrategy(solution) + return when(namingStrategy) { + NamingStrategy.NUMBERED -> NamingHelperNumberedTestCaseNamingStrategy(solution) + NamingStrategy.DETERMINISTIC -> deterministicActionBasedNamingStrategy(solution) + //TODO LLM else -> throw IllegalStateException("Unrecognized naming strategy $namingStrategy") } } - private fun actionBasedNamingStrategy(solution: Solution<*>): NumberedTestCaseNamingStrategy { + private fun deterministicActionBasedNamingStrategy(solution: Solution<*>): NumberedTestCaseNamingStrategy { val individuals = solution.individuals return when { - individuals.any { it.individual is RestIndividual } -> return RestActionTestCaseNamingStrategy(solution, languageConventionFormatter, nameWithQueryParameters, maxTestCaseNameLength) - individuals.any { it.individual is GraphQLIndividual } -> return GraphQLActionTestCaseNamingStrategy(solution, languageConventionFormatter, maxTestCaseNameLength) - individuals.any { it.individual is RPCIndividual } -> return RPCActionTestCaseNamingStrategy(solution, languageConventionFormatter, maxTestCaseNameLength) + individuals.any { it.individual is RestIndividual } -> RestActionTestCaseNamingStrategy(solution, languageConventionFormatter, nameWithQueryParameters, maxTestCaseNameLength) + individuals.any { it.individual is GraphQLIndividual } -> GraphQLActionTestCaseNamingStrategy(solution, languageConventionFormatter, maxTestCaseNameLength) + individuals.any { it.individual is RPCIndividual } -> RPCActionTestCaseNamingStrategy(solution, languageConventionFormatter, maxTestCaseNameLength) individuals.any { it.individual is WebIndividual } -> { log.warn("Web individuals do not have action based test case naming yet. Defaulting to Numbered strategy.") - return NamingHelperNumberedTestCaseNamingStrategy(solution) + NamingHelperNumberedTestCaseNamingStrategy(solution) } individuals.isEmpty() -> { log.warn("No individuals present in the solution. Defaulting to Numbered strategy.") - return NumberedTestCaseNamingStrategy(solution) + NumberedTestCaseNamingStrategy(solution) } else -> throw IllegalStateException("Unrecognized test individuals with no action based naming strategy set.") } diff --git a/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt b/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt index 0e021a8fd6..3763096a0d 100644 --- a/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt @@ -685,7 +685,7 @@ internal class EMConfigTest{ config.updateProperties(parser.parse()) - assertEquals(NamingStrategy.ACTION, config.namingStrategy) + assertEquals(NamingStrategy.DETERMINISTIC, config.namingStrategy) } @Test @@ -696,7 +696,7 @@ internal class EMConfigTest{ val options = parser.parse("--namingStrategy", "ACTION", "--nameWithQueryParameters", "true") config.updateProperties(options) - assertEquals(NamingStrategy.ACTION, config.namingStrategy) + assertEquals(NamingStrategy.DETERMINISTIC, config.namingStrategy) assertTrue(config.nameWithQueryParameters) } @@ -708,7 +708,7 @@ internal class EMConfigTest{ val options = parser.parse("--namingStrategy", "ACTION", "--nameWithQueryParameters", "false") config.updateProperties(options) - assertEquals(NamingStrategy.ACTION, config.namingStrategy) + assertEquals(NamingStrategy.DETERMINISTIC, config.namingStrategy) assertFalse(config.nameWithQueryParameters) } From 476704917b68fe7dc55a2e123a1d612a3fb85536 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Sat, 30 May 2026 20:59:06 +0200 Subject: [PATCH 07/11] fix after refactoring --- .../main/kotlin/org/evomaster/core/llm/service/LlmService.kt | 3 ++- core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt | 4 ++-- docs/options.md | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/llm/service/LlmService.kt b/core/src/main/kotlin/org/evomaster/core/llm/service/LlmService.kt index a570443bcb..e767eaa46c 100644 --- a/core/src/main/kotlin/org/evomaster/core/llm/service/LlmService.kt +++ b/core/src/main/kotlin/org/evomaster/core/llm/service/LlmService.kt @@ -6,6 +6,7 @@ import org.evomaster.core.EMConfig import org.evomaster.core.config.ConfigProblemException import org.evomaster.core.llm.LlmSupport import java.util.concurrent.CompletableFuture +import java.util.concurrent.Future import javax.annotation.PostConstruct import javax.inject.Inject @@ -44,7 +45,7 @@ class LlmService { } } - fun chatAsync(userMessage: String): CompletableFuture{ + fun chatAsync(userMessage: String): Future { checkUsingLLM() return LlmSupport.chatAsync(model, userMessage) diff --git a/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt b/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt index 3763096a0d..a31343e4a9 100644 --- a/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt @@ -693,7 +693,7 @@ internal class EMConfigTest{ val parser = EMConfig.getOptionParser() val config = EMConfig() - val options = parser.parse("--namingStrategy", "ACTION", "--nameWithQueryParameters", "true") + val options = parser.parse("--namingStrategy", "DETERMINISTIC", "--nameWithQueryParameters", "true") config.updateProperties(options) assertEquals(NamingStrategy.DETERMINISTIC, config.namingStrategy) @@ -705,7 +705,7 @@ internal class EMConfigTest{ val parser = EMConfig.getOptionParser() val config = EMConfig() - val options = parser.parse("--namingStrategy", "ACTION", "--nameWithQueryParameters", "false") + val options = parser.parse("--namingStrategy", "DETERMINISTIC", "--nameWithQueryParameters", "false") config.updateProperties(options) assertEquals(NamingStrategy.DETERMINISTIC, config.namingStrategy) diff --git a/docs/options.md b/docs/options.md index ecefec8e7c..aa6b5227d1 100644 --- a/docs/options.md +++ b/docs/options.md @@ -178,7 +178,7 @@ There are 3 types of options: |`muPlusLambdaOffspringSize`| __Int__. Define the number of offspring (λ) generated per generation in (μ+λ) Evolutionary Algorithm. *Constraints*: `min=1.0`. *Default value*: `30`.| |`mutatedGeneFile`| __String__. Specify a path to save mutation details which is useful for debugging mutation. *DEBUG option*. *Default value*: `mutatedGeneInfo.csv`.| |`nameWithQueryParameters`| __Boolean__. Specify if true boolean query parameters are included in the test case name. Used for test case naming disambiguation. Only valid for Action based naming strategy. *Default value*: `true`.| -|`namingStrategy`| __Enum__. Specify the naming strategy for test cases. *Valid values*: `NUMBERED, ACTION, LLM`. *Default value*: `ACTION`.| +|`namingStrategy`| __Enum__. Specify the naming strategy for test cases. *Valid values*: `NUMBERED, DETERMINISTIC, LLM`. *Default value*: `DETERMINISTIC`.| |`outputExecutedSQL`| __Enum__. Whether to output executed sql info. *DEBUG option*. *Valid values*: `NONE, ALL_AT_END, ONCE_EXECUTED`. *Default value*: `NONE`.| |`overrideAuthExternalEndpointURL`| __String__. Override the value of externalEndpointURL in auth configurations. This is useful when the auth server is running locally on an ephemeral port, or when several instances are run in parallel, to avoid creating/modifying auth configuration files. If what provided is a URL starting with 'http', then full replacement will occur. Otherwise, the input will be treated as a 'hostname:port', and only that info will be updated (e.g., path element of the URL will not change). *Default value*: `null`.| |`populationSize`| __Int__. Define the population size in the search algorithms that use populations (e.g., Genetic Algorithms, but not MIO). *Constraints*: `min=1.0`. *Default value*: `30`.| From cee1b8ae79c15b14e5effe4102a045f9a2c7c98e Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Sun, 31 May 2026 21:03:32 +0200 Subject: [PATCH 08/11] updated E2E test assertions --- .../spring/examples/sort/SortEMTest.java | 94 ++++++++----------- 1 file changed, 39 insertions(+), 55 deletions(-) diff --git a/core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/sort/SortEMTest.java b/core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/sort/SortEMTest.java index 1b7cf5468a..9ffd503617 100644 --- a/core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/sort/SortEMTest.java +++ b/core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/sort/SortEMTest.java @@ -27,7 +27,7 @@ public void testRunEM() throws Throwable { runTestHandlingFlakyAndCompilation( "SortEM", "org.bar.SortEM", - 3_000, + 100, // 3_000, (args) -> { Solution solution = initAndRun(args); @@ -38,63 +38,47 @@ public void testRunEM() throws Throwable { assertHasAtLeastOne(solution, HttpVerb.PUT, 500); assertHasAtLeastOne(solution, HttpVerb.POST, 500); -// TestSuiteOrganizer organizer = new TestSuiteOrganizer(); -// -// TestCaseNamingStrategy namingStrategy = new NumberedTestCaseNamingStrategy(solution); -// -// List tclist = organizer.createSortedTestCases(namingStrategy, SortingStrategy.COVERED_TARGETS); - //Iterator iterator = tclist.iterator(); - //TestCase current, previous = iterator.next(); /* - while(iterator.hasNext()){ - current = iterator.next(); - // Check that a TC with 500 in the name does not follow a TC without a 500 in the name (500 should be first). - - if(current.getName().contains("500")){ - assertTrue(previous.getName().contains("500")); - } - previous = current; - } + TODO the check here were wrong, as checking a side-effect on internal list of solution. + but solution can be split in sub-solutions before sorted (and tests are named) and printed. + the sorting is not stored in the solution object, as it is a side-effect. */ - - Iterator> iterator = solution.getIndividuals().iterator(); - EvaluatedIndividual current, previous = iterator.next(); - - - - while(iterator.hasNext()){ - current = iterator.next(); - - if (current.seeResults(null).stream() - .filter(w -> w instanceof RestCallResult) - .anyMatch(r -> ((RestCallResult) r).getStatusCode() == 500)) { - - assertTrue(previous.seeResults(null).stream() - .filter(w -> w instanceof RestCallResult) - .anyMatch(r -> ((RestCallResult) r).getStatusCode() == 500)); - } - - - - // Check that the current "priority code" is less than the previous priority code - - OptionalInt currentPrioCode = current.seeResults(null).stream() - .filter(w -> w instanceof RestCallResult) - .mapToInt(w -> ((RestCallResult) w).getStatusCode()) - .map(w -> w % 500) - .min(); - - OptionalInt previousPrioCode = previous.seeResults(null).stream() - .filter(w -> w instanceof RestCallResult) - .mapToInt(w -> ((RestCallResult) w).getStatusCode()) - .map(w -> w % 500) - .min(); - - assertTrue(currentPrioCode.getAsInt() >= previousPrioCode.getAsInt()); - previous = current; - - } +// Iterator> iterator = solution.getIndividuals().iterator(); +// EvaluatedIndividual current, previous = iterator.next(); +// +// while(iterator.hasNext()){ +// current = iterator.next(); +// +// if (current.seeResults(null).stream() +// .filter(w -> w instanceof RestCallResult) +// .anyMatch(r -> ((RestCallResult) r).getStatusCode() == 500)) { +// +// assertTrue(previous.seeResults(null).stream() +// .filter(w -> w instanceof RestCallResult) +// .anyMatch(r -> ((RestCallResult) r).getStatusCode() == 500)); +// } +// +// +// +// // Check that the current "priority code" is less than the previous priority code +// +// OptionalInt currentPrioCode = current.seeResults(null).stream() +// .filter(w -> w instanceof RestCallResult) +// .mapToInt(w -> ((RestCallResult) w).getStatusCode()) +// .map(w -> w % 500) +// .min(); +// +// OptionalInt previousPrioCode = previous.seeResults(null).stream() +// .filter(w -> w instanceof RestCallResult) +// .mapToInt(w -> ((RestCallResult) w).getStatusCode()) +// .map(w -> w % 500) +// .min(); +// +// assertTrue(currentPrioCode.getAsInt() >= previousPrioCode.getAsInt()); +// previous = current; +// +// } }); } From 3c17b6300844b6c0a77b07e30d106046616f8258 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Sun, 31 May 2026 22:31:11 +0200 Subject: [PATCH 09/11] fixed assertions in E2E --- .../e2etests/utils/EnterpriseTestBase.java | 28 ++++++++++++- .../spring/examples/sort/SortEMTest.java | 42 +++++++++++++++---- .../naming/ActionTestCaseNamingStrategy.kt | 2 +- .../naming/NumberedTestCaseNamingStrategy.kt | 6 ++- scripts/loopback-aliases-macos.sh | 4 ++ 5 files changed, 72 insertions(+), 10 deletions(-) diff --git a/core-tests/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java b/core-tests/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java index 3ea63d0b34..6bfe262a04 100644 --- a/core-tests/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java +++ b/core-tests/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java @@ -45,8 +45,10 @@ import java.util.List; import java.util.Objects; import java.util.function.Consumer; -import java.util.function.Function; import java.util.function.Predicate; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.*; @@ -639,7 +641,31 @@ protected void assertTextInTests(String outputFolder, String className, Predicat }catch (IOException e){ throw new IllegalStateException("Fail to get the test "+className+" in "+outputFolder+" with error "+ e.getMessage()); } + } + + protected List getFunctionNames(String outputFolder, String className){ + + //this specific on how we create tests + String regex = "\\s*fun\\s.*\\(\\)\\s*\\{\\s*"; + Pattern pattern = Pattern.compile(regex); + String path = outputFolderPath(outputFolder)+ "/"+String.join("/", className.split("\\."))+".kt"; + Path test = Paths.get(path); + try { + return Files.lines(test) + .filter(it -> { + Matcher matcher = pattern.matcher(it); + return matcher.find(); + } ) + .map(it -> { + int start = it.indexOf("fun") + 3; + int end = it.indexOf("()"); + return it.substring(start, end).trim(); + }) + .collect(Collectors.toList()); + }catch (IOException e){ + throw new IllegalStateException("Fail to get the test "+className+" in "+outputFolder+" with error "+ e.getMessage()); + } } /** diff --git a/core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/sort/SortEMTest.java b/core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/sort/SortEMTest.java index 9ffd503617..484644803a 100644 --- a/core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/sort/SortEMTest.java +++ b/core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/sort/SortEMTest.java @@ -1,17 +1,19 @@ package org.evomaster.e2etests.spring.examples.sort; +import org.evomaster.core.output.naming.NamingStrategy; +import org.evomaster.core.output.naming.NumberedTestCaseNamingStrategy; +import org.evomaster.core.output.naming.TestCaseNamingStrategy; import org.evomaster.core.problem.rest.data.HttpVerb; -import org.evomaster.core.problem.rest.data.RestCallResult; import org.evomaster.core.problem.rest.data.RestIndividual; -import org.evomaster.core.search.EvaluatedIndividual; import org.evomaster.core.search.Solution; import org.evomaster.e2etests.spring.examples.namedresource.NRTestBase; import org.junit.jupiter.api.Test; -import java.util.Iterator; -import java.util.OptionalInt; +import java.util.List; +import java.util.stream.Collectors; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -24,11 +26,17 @@ public class SortEMTest extends NRTestBase { @Test public void testRunEM() throws Throwable { + String outputFolderName = "SortEM"; + String className = "org.bar.SortEM"; + runTestHandlingFlakyAndCompilation( - "SortEM", - "org.bar.SortEM", - 100, // 3_000, + outputFolderName, + className, + 3_000, (args) -> { + + setOption(args, "namingStrategy", NamingStrategy.DETERMINISTIC.name()); + Solution solution = initAndRun(args); assertTrue(solution.getIndividuals().size() >= 1); @@ -38,6 +46,26 @@ public void testRunEM() throws Throwable { assertHasAtLeastOne(solution, HttpVerb.PUT, 500); assertHasAtLeastOne(solution, HttpVerb.POST, 500); + List functionNames = getFunctionNames(outputFolderName, className); + // 3 initializations (eg @BeforEach), plus 1 per individuals + int expected = 3 + solution.getIndividuals().size(); + assertEquals(expected, functionNames.size()); + + String prefix = NumberedTestCaseNamingStrategy.TEST_NAME_PREFIX; + List testNames = functionNames.stream() + .filter(it -> it.startsWith(prefix)) + .collect(Collectors.toList()); + assertEquals(solution.getIndividuals().size(), testNames.size()); + + for(int i=0; i < testNames.size(); i++){ + String name = testNames.get(i); + int index = Integer.parseInt( + name.substring(prefix.length(), name.indexOf("_", prefix.length() + 1)) + ); + assertEquals(i, index, + "Wrong numbering." + + " Expected: " + i + ", but found " + index + " for test name " + name); + } /* TODO the check here were wrong, as checking a side-effect on internal list of solution. diff --git a/core/src/main/kotlin/org/evomaster/core/output/naming/ActionTestCaseNamingStrategy.kt b/core/src/main/kotlin/org/evomaster/core/output/naming/ActionTestCaseNamingStrategy.kt index c02b497219..9e5f42a681 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/naming/ActionTestCaseNamingStrategy.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/naming/ActionTestCaseNamingStrategy.kt @@ -95,7 +95,7 @@ abstract class ActionTestCaseNamingStrategy( protected fun namePrefixChars(): Int { val digitsUsedForTestNumbering = testCasesSize.toString().length - return "test_".length + digitsUsedForTestNumbering + 1 + return TEST_NAME_PREFIX.length + digitsUsedForTestNumbering + 1 } protected fun addNameTokensIfAllowed(nameTokens: MutableList, targetStrings: List, remainingNameChars: Int): Int { diff --git a/core/src/main/kotlin/org/evomaster/core/output/naming/NumberedTestCaseNamingStrategy.kt b/core/src/main/kotlin/org/evomaster/core/output/naming/NumberedTestCaseNamingStrategy.kt index 07f1473355..3320bdbf91 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/naming/NumberedTestCaseNamingStrategy.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/naming/NumberedTestCaseNamingStrategy.kt @@ -9,6 +9,10 @@ open class NumberedTestCaseNamingStrategy( solution: Solution<*> ) : TestCaseNamingStrategy(solution) { + companion object{ + const val TEST_NAME_PREFIX = "test_" + } + override fun getTestCases(): List { return generateNames(solution.individuals) } @@ -26,7 +30,7 @@ open class NumberedTestCaseNamingStrategy( } private fun concatName(counter: Int, expandedName: String): String { - return "test_${counter}${expandedName}" + return "$TEST_NAME_PREFIX${counter}${expandedName}" } private fun generateNames(individuals: List>) : List { diff --git a/scripts/loopback-aliases-macos.sh b/scripts/loopback-aliases-macos.sh index 7bc6316324..3295b6186d 100755 --- a/scripts/loopback-aliases-macos.sh +++ b/scripts/loopback-aliases-macos.sh @@ -77,3 +77,7 @@ then else echo "No operation selected to execute" fi + + +### On shell, can also just run: +### for i in {2..255}; do sudo ifconfig lo0 alias 127.0.0.$i up; done \ No newline at end of file From 1906a99e30b1a970dd578b83cd9b0a2e53a0518c Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Mon, 1 Jun 2026 09:58:20 +0200 Subject: [PATCH 10/11] fixed failing tests --- .../spring/examples/sort/SortEMTest.java | 46 ++--------------- .../core/output/TestSuiteOrganizer.kt | 8 ++- .../core/output/sorting/SortingHelper.kt | 50 ++++++++----------- 3 files changed, 33 insertions(+), 71 deletions(-) diff --git a/core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/sort/SortEMTest.java b/core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/sort/SortEMTest.java index 484644803a..bfe54a9471 100644 --- a/core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/sort/SortEMTest.java +++ b/core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/sort/SortEMTest.java @@ -5,12 +5,16 @@ import org.evomaster.core.output.naming.NumberedTestCaseNamingStrategy; import org.evomaster.core.output.naming.TestCaseNamingStrategy; import org.evomaster.core.problem.rest.data.HttpVerb; +import org.evomaster.core.problem.rest.data.RestCallResult; import org.evomaster.core.problem.rest.data.RestIndividual; +import org.evomaster.core.search.EvaluatedIndividual; import org.evomaster.core.search.Solution; import org.evomaster.e2etests.spring.examples.namedresource.NRTestBase; import org.junit.jupiter.api.Test; +import java.util.Iterator; import java.util.List; +import java.util.OptionalInt; import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -66,48 +70,6 @@ public void testRunEM() throws Throwable { "Wrong numbering." + " Expected: " + i + ", but found " + index + " for test name " + name); } - - /* - TODO the check here were wrong, as checking a side-effect on internal list of solution. - but solution can be split in sub-solutions before sorted (and tests are named) and printed. - the sorting is not stored in the solution object, as it is a side-effect. - */ -// Iterator> iterator = solution.getIndividuals().iterator(); -// EvaluatedIndividual current, previous = iterator.next(); -// -// while(iterator.hasNext()){ -// current = iterator.next(); -// -// if (current.seeResults(null).stream() -// .filter(w -> w instanceof RestCallResult) -// .anyMatch(r -> ((RestCallResult) r).getStatusCode() == 500)) { -// -// assertTrue(previous.seeResults(null).stream() -// .filter(w -> w instanceof RestCallResult) -// .anyMatch(r -> ((RestCallResult) r).getStatusCode() == 500)); -// } -// -// -// -// // Check that the current "priority code" is less than the previous priority code -// -// OptionalInt currentPrioCode = current.seeResults(null).stream() -// .filter(w -> w instanceof RestCallResult) -// .mapToInt(w -> ((RestCallResult) w).getStatusCode()) -// .map(w -> w % 500) -// .min(); -// -// OptionalInt previousPrioCode = previous.seeResults(null).stream() -// .filter(w -> w instanceof RestCallResult) -// .mapToInt(w -> ((RestCallResult) w).getStatusCode()) -// .map(w -> w % 500) -// .min(); -// -// assertTrue(currentPrioCode.getAsInt() >= previousPrioCode.getAsInt()); -// previous = current; -// -// } - }); } diff --git a/core/src/main/kotlin/org/evomaster/core/output/TestSuiteOrganizer.kt b/core/src/main/kotlin/org/evomaster/core/output/TestSuiteOrganizer.kt index c4565b85e0..10744045b4 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/TestSuiteOrganizer.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/TestSuiteOrganizer.kt @@ -26,11 +26,17 @@ class TestSuiteOrganizer( */ fun createSortedTestCases(solution: Solution<*>): List { + /* + Tests MUST be sorted before they are named, as their position might influence + their name (eg, "test_0_...") + */ + sortingHelper.sort(solution.individuals, config.testCaseSortingStrategy) + val namingStrategy = TestCaseNamingStrategyFactory(config).create(solution) val tests = namingStrategy.getTestCases() - return sortingHelper.sort(tests, config.testCaseSortingStrategy) + return tests } diff --git a/core/src/main/kotlin/org/evomaster/core/output/sorting/SortingHelper.kt b/core/src/main/kotlin/org/evomaster/core/output/sorting/SortingHelper.kt index ec049847e3..9430579b2a 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/sorting/SortingHelper.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/sorting/SortingHelper.kt @@ -39,8 +39,6 @@ class SortingHelper { * - second: 2xx * - third: 4xx */ - - private val statusCode: Comparator> = compareBy { ind -> val min = ind.seeResults().filterIsInstance().minByOrNull { it.getStatusCode()?.rem(500) ?: 0 @@ -84,7 +82,6 @@ class SortingHelper { * [comparatorList] holds those comparators that are currently selected for sorting * Note that the order of the comparators is the order their importance/priority. */ - private val comparatorList = listOf(statusCode, coveredTargets) private val restComparator: Comparator> = compareBy> { ind -> @@ -121,36 +118,33 @@ class SortingHelper { } - fun sort(tests: List, testCaseSortingStrategy: SortingStrategy): List { - val newSort = when (testCaseSortingStrategy) { + fun sort(tests: MutableList>, testCaseSortingStrategy: SortingStrategy) { + when (testCaseSortingStrategy) { SortingStrategy.COVERED_TARGETS -> sortByComparatorList(comparatorList, tests) SortingStrategy.TARGET_INCREMENTAL -> sortByTargetIncremental(tests) else -> throw IllegalStateException("Unrecognized sorting strategy $testCaseSortingStrategy") } - - return newSort } @Deprecated("Use other version") - fun sort(solution: Solution<*>, namingStrategy: TestCaseNamingStrategy, testCaseSortingStrategy: SortingStrategy) : List { - return sort(namingStrategy.getTestCases(), testCaseSortingStrategy) + fun sort( + solution: Solution<*>, + namingStrategy: TestCaseNamingStrategy, + testCaseSortingStrategy: SortingStrategy + ) : List { + sort(solution.individuals, testCaseSortingStrategy) + return namingStrategy.getTestCases() } - private fun getSortedTestCases(tests: List, comparator: Comparator>): List { - return getSortedTestCases(tests, singletonList(comparator)) + private fun getSortedTestCases(tests: MutableList>, comparator: Comparator>) { + getSortedTestCases(tests, singletonList(comparator)) } - private fun getSortedTestCases(tests: List, comparators: List>>): List { - - val copy = tests.toMutableList() - + private fun getSortedTestCases(tests: MutableList>, comparators: List>>) { comparators.asReversed().forEach { comp -> - val w = Comparator{a, b -> comp.compare(a.test,b.test)} - copy.sortWith(w) + tests.sortWith(comp) } - - return copy } @@ -159,9 +153,9 @@ class SortingHelper { *Sorting is done according to the comparator list. If no list is provided, individuals are sorted by max status. */ private fun sortByComparatorList (comparators: List>> = listOf(statusCode), - tests: List + tests: MutableList> - ): List { + ){ /** * Comparisons, as far as I understand them, are done as follows: * First, the list is sorted based on the first criterion. @@ -182,24 +176,24 @@ class SortingHelper { return getSortedTestCases(tests, comparators) } - private fun sortByTargetIncremental(tests: List): List { + private fun sortByTargetIncremental(tests: MutableList>) { if(tests.isEmpty()){ - return emptyList() + return } val comparator = when { - tests.any { it.test.individual is RestIndividual } -> restComparator - tests.any { it.test.individual is GraphQLIndividual } -> graphQLComparator - tests.any { it.test.individual is RPCIndividual } -> rpcComparator - tests.any { it.test.individual is WebIndividual } -> { + tests.any { it.individual is RestIndividual } -> restComparator + tests.any { it.individual is GraphQLIndividual } -> graphQLComparator + tests.any { it.individual is RPCIndividual } -> rpcComparator + tests.any { it.individual is WebIndividual } -> { log.warn("Web individuals do not have action based test case naming yet. Defaulting to Numbered strategy.") statusCode } else -> throw IllegalStateException("Unrecognized test individuals with no target incremental based sorting strategy set.") } - return getSortedTestCases(tests, comparator) + getSortedTestCases(tests, comparator) } From 400d83484788dc3d7ac9681864338f8a4eedb299 Mon Sep 17 00:00:00 2001 From: arcuri82 Date: Mon, 1 Jun 2026 10:06:52 +0200 Subject: [PATCH 11/11] cleaning --- .../core/output/TestSuiteOrganizer.kt | 3 + .../core/output/sorting/SortingHelper.kt | 57 +++++++------------ 2 files changed, 25 insertions(+), 35 deletions(-) diff --git a/core/src/main/kotlin/org/evomaster/core/output/TestSuiteOrganizer.kt b/core/src/main/kotlin/org/evomaster/core/output/TestSuiteOrganizer.kt index 10744045b4..3a25261807 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/TestSuiteOrganizer.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/TestSuiteOrganizer.kt @@ -23,6 +23,9 @@ class TestSuiteOrganizer( * Furthermore, this class is also responsible for deciding which * name each test will have. * + *
+ * WARNING: side-effect of sorting tests inside input [solution] object + * */ fun createSortedTestCases(solution: Solution<*>): List { diff --git a/core/src/main/kotlin/org/evomaster/core/output/sorting/SortingHelper.kt b/core/src/main/kotlin/org/evomaster/core/output/sorting/SortingHelper.kt index 9430579b2a..7245928412 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/sorting/SortingHelper.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/sorting/SortingHelper.kt @@ -1,6 +1,5 @@ package org.evomaster.core.output.sorting -import org.evomaster.core.Lazy import org.evomaster.core.output.TestCase import org.evomaster.core.output.naming.TestCaseNamingStrategy import org.evomaster.core.problem.graphql.GraphQLAction @@ -120,7 +119,7 @@ class SortingHelper { fun sort(tests: MutableList>, testCaseSortingStrategy: SortingStrategy) { when (testCaseSortingStrategy) { - SortingStrategy.COVERED_TARGETS -> sortByComparatorList(comparatorList, tests) + SortingStrategy.COVERED_TARGETS -> sortByComparatorList(tests, comparatorList) SortingStrategy.TARGET_INCREMENTAL -> sortByTargetIncremental(tests) else -> throw IllegalStateException("Unrecognized sorting strategy $testCaseSortingStrategy") } @@ -137,45 +136,33 @@ class SortingHelper { } - private fun getSortedTestCases(tests: MutableList>, comparator: Comparator>) { - getSortedTestCases(tests, singletonList(comparator)) + private fun sortByComparator(tests: MutableList>, comparator: Comparator>) { + sortByComparatorList(tests, singletonList(comparator)) } - private fun getSortedTestCases(tests: MutableList>, comparators: List>>) { + /** + * Sorting is done according to the comparator list. + * Comparisons, are done as follows: + * First, the list is sorted based on the first criterion. + * Then, the (now sorted) list, is sorted based on the second criterion. + * Where two values have equal priority with respect to the most recent sort, they maintain the order (and, thus, + * are still sorted according to the first criterion). + * + * So, a criterion with more priority overrides most other criteria, unless elements have the same value. + * If too many criteria are used, the ones that are lower on the priority list will not really have a chance to manifest. + * + * An example of how this approach is used: + * = first priority (thus, last to be executed and most likely to be observed) is the [statusCode]. Thus, every + * test case that contains a 500 code is at the top. + * = second priority (thus, second to last to be executed), is the [coveredTargets]. Thus, among those test cases + * that have the same code, the ones with the most covered targets will be at the top (among their sub-group). + */ + private fun sortByComparatorList(tests: MutableList>, comparators: List>>) { comparators.asReversed().forEach { comp -> tests.sortWith(comp) } } - - - /** - *Sorting is done according to the comparator list. If no list is provided, individuals are sorted by max status. - */ - private fun sortByComparatorList (comparators: List>> = listOf(statusCode), - tests: MutableList> - - ){ - /** - * Comparisons, as far as I understand them, are done as follows: - * First, the list is sorted based on the first criterion. - * Then, the (now sorted) list, is sorted based on the second criterion. - * Where two values have equal priority with respect to the most recent sort, they maintain the order (and, thus, - * are still sorted according to the first criterion). - * - * So, a criterion with more priority overrides most other criteria, unless elements have the same value. - * If too many criteria are used, the ones that are lower on the priority list will not really have a chance to manifest. - * - * An example of how this approach is used: - * = first priority (thus, last to be executed and most likely to be observed) is the [statusCode]. Thus, every - * test case that contains a 500 code is at the top. - * = second priority (thus, second to last to be executed), is the [coveredTargets]. Thus, among those test cases - * that have the same code, the ones with the most covered targets will be at the top (among their sub-group). - */ - - return getSortedTestCases(tests, comparators) - } - private fun sortByTargetIncremental(tests: MutableList>) { if(tests.isEmpty()){ @@ -193,7 +180,7 @@ class SortingHelper { else -> throw IllegalStateException("Unrecognized test individuals with no target incremental based sorting strategy set.") } - getSortedTestCases(tests, comparator) + sortByComparator(tests, comparator) }