diff --git a/core-tests/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java b/core-tests/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java index 3ea63d0b34..6bfe262a04 100644 --- a/core-tests/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java +++ b/core-tests/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java @@ -45,8 +45,10 @@ import java.util.List; import java.util.Objects; import java.util.function.Consumer; -import java.util.function.Function; import java.util.function.Predicate; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.*; @@ -639,7 +641,31 @@ protected void assertTextInTests(String outputFolder, String className, Predicat }catch (IOException e){ throw new IllegalStateException("Fail to get the test "+className+" in "+outputFolder+" with error "+ e.getMessage()); } + } + + protected List getFunctionNames(String outputFolder, String className){ + + //this specific on how we create tests + String regex = "\\s*fun\\s.*\\(\\)\\s*\\{\\s*"; + Pattern pattern = Pattern.compile(regex); + String path = outputFolderPath(outputFolder)+ "/"+String.join("/", className.split("\\."))+".kt"; + Path test = Paths.get(path); + try { + return Files.lines(test) + .filter(it -> { + Matcher matcher = pattern.matcher(it); + return matcher.find(); + } ) + .map(it -> { + int start = it.indexOf("fun") + 3; + int end = it.indexOf("()"); + return it.substring(start, end).trim(); + }) + .collect(Collectors.toList()); + }catch (IOException e){ + throw new IllegalStateException("Fail to get the test "+className+" in "+outputFolder+" with error "+ e.getMessage()); + } } /** diff --git a/core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/sort/SortEMTest.java b/core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/sort/SortEMTest.java index 4c1d1f5402..bfe54a9471 100644 --- a/core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/sort/SortEMTest.java +++ b/core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/sort/SortEMTest.java @@ -1,10 +1,9 @@ package org.evomaster.e2etests.spring.examples.sort; -import org.evomaster.core.output.TestCase; -import org.evomaster.core.output.TestSuiteOrganizer; + +import org.evomaster.core.output.naming.NamingStrategy; import org.evomaster.core.output.naming.NumberedTestCaseNamingStrategy; import org.evomaster.core.output.naming.TestCaseNamingStrategy; -import org.evomaster.core.output.sorting.SortingStrategy; import org.evomaster.core.problem.rest.data.HttpVerb; import org.evomaster.core.problem.rest.data.RestCallResult; import org.evomaster.core.problem.rest.data.RestIndividual; @@ -16,7 +15,9 @@ import java.util.Iterator; import java.util.List; import java.util.OptionalInt; +import java.util.stream.Collectors; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -29,11 +30,17 @@ public class SortEMTest extends NRTestBase { @Test public void testRunEM() throws Throwable { + String outputFolderName = "SortEM"; + String className = "org.bar.SortEM"; + runTestHandlingFlakyAndCompilation( - "SortEM", - "org.bar.SortEM", + outputFolderName, + className, 3_000, (args) -> { + + setOption(args, "namingStrategy", NamingStrategy.DETERMINISTIC.name()); + Solution solution = initAndRun(args); assertTrue(solution.getIndividuals().size() >= 1); @@ -43,64 +50,26 @@ public void testRunEM() throws Throwable { assertHasAtLeastOne(solution, HttpVerb.PUT, 500); assertHasAtLeastOne(solution, HttpVerb.POST, 500); - TestSuiteOrganizer organizer = new TestSuiteOrganizer(); - - TestCaseNamingStrategy namingStrategy = new NumberedTestCaseNamingStrategy(solution); - - List tclist = organizer.sortTests(solution, namingStrategy, SortingStrategy.COVERED_TARGETS); - - //Iterator iterator = tclist.iterator(); - //TestCase current, previous = iterator.next(); - /* - while(iterator.hasNext()){ - current = iterator.next(); - // Check that a TC with 500 in the name does not follow a TC without a 500 in the name (500 should be first). - - if(current.getName().contains("500")){ - assertTrue(previous.getName().contains("500")); - } - previous = current; - } - */ - - Iterator> iterator = solution.getIndividuals().iterator(); - EvaluatedIndividual current, previous = iterator.next(); - - - - while(iterator.hasNext()){ - current = iterator.next(); - - if (current.seeResults(null).stream() - .filter(w -> w instanceof RestCallResult) - .anyMatch(r -> ((RestCallResult) r).getStatusCode() == 500)) { - - assertTrue(previous.seeResults(null).stream() - .filter(w -> w instanceof RestCallResult) - .anyMatch(r -> ((RestCallResult) r).getStatusCode() == 500)); - } - - - - // Check that the current "priority code" is less than the previous priority code - - OptionalInt currentPrioCode = current.seeResults(null).stream() - .filter(w -> w instanceof RestCallResult) - .mapToInt(w -> ((RestCallResult) w).getStatusCode()) - .map(w -> w % 500) - .min(); - - OptionalInt previousPrioCode = previous.seeResults(null).stream() - .filter(w -> w instanceof RestCallResult) - .mapToInt(w -> ((RestCallResult) w).getStatusCode()) - .map(w -> w % 500) - .min(); - - assertTrue(currentPrioCode.getAsInt() >= previousPrioCode.getAsInt()); - previous = current; - - } - + List functionNames = getFunctionNames(outputFolderName, className); + // 3 initializations (eg @BeforEach), plus 1 per individuals + int expected = 3 + solution.getIndividuals().size(); + assertEquals(expected, functionNames.size()); + + String prefix = NumberedTestCaseNamingStrategy.TEST_NAME_PREFIX; + List testNames = functionNames.stream() + .filter(it -> it.startsWith(prefix)) + .collect(Collectors.toList()); + assertEquals(solution.getIndividuals().size(), testNames.size()); + + for(int i=0; i < testNames.size(); i++){ + String name = testNames.get(i); + int index = Integer.parseInt( + name.substring(prefix.length(), name.indexOf("_", prefix.length() + 1)) + ); + assertEquals(i, index, + "Wrong numbering." + + " Expected: " + i + ", but found " + index + " for test name " + name); + } }); } diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index 6bda393e9d..cc041e3535 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -79,7 +79,7 @@ class EMConfig { private val defaultOutputFormatForBlackBox = OutputFormat.PYTHON_UNITTEST - private val defaultTestCaseNamingStrategy = NamingStrategy.ACTION + private val defaultTestCaseNamingStrategy = NamingStrategy.DETERMINISTIC private val defaultTestCaseSortingStrategy = SortingStrategy.TARGET_INCREMENTAL @@ -870,6 +870,10 @@ class EMConfig { if (sutJarEnvVarName.isEmpty()) throw ConfigProblemException("'sutJarEnvVarName' must be specified if 'useEnvVarsForPathInTests' is enabled.") } + + if(namingStrategy == NamingStrategy.LLM && !llm){ + throw ConfigProblemException("Naming strategy LLM require the setup and use of an LLM") + } } private fun checkPropertyConstraints(m: KMutableProperty<*>) { diff --git a/core/src/main/kotlin/org/evomaster/core/llm/service/LlmService.kt b/core/src/main/kotlin/org/evomaster/core/llm/service/LlmService.kt index a570443bcb..e767eaa46c 100644 --- a/core/src/main/kotlin/org/evomaster/core/llm/service/LlmService.kt +++ b/core/src/main/kotlin/org/evomaster/core/llm/service/LlmService.kt @@ -6,6 +6,7 @@ import org.evomaster.core.EMConfig import org.evomaster.core.config.ConfigProblemException import org.evomaster.core.llm.LlmSupport import java.util.concurrent.CompletableFuture +import java.util.concurrent.Future import javax.annotation.PostConstruct import javax.inject.Inject @@ -44,7 +45,7 @@ class LlmService { } } - fun chatAsync(userMessage: String): CompletableFuture{ + fun chatAsync(userMessage: String): Future { checkUsingLLM() return LlmSupport.chatAsync(model, userMessage) diff --git a/core/src/main/kotlin/org/evomaster/core/output/TestSuiteOrganizer.kt b/core/src/main/kotlin/org/evomaster/core/output/TestSuiteOrganizer.kt index 0fd305be57..3a25261807 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/TestSuiteOrganizer.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/TestSuiteOrganizer.kt @@ -1,318 +1,50 @@ package org.evomaster.core.output -import org.evomaster.core.Lazy -import org.evomaster.core.output.naming.TestCaseNamingStrategy -import org.evomaster.core.output.sorting.SortingStrategy -import org.evomaster.core.problem.graphql.GraphQLAction -import org.evomaster.core.problem.graphql.GraphQLIndividual -import org.evomaster.core.problem.httpws.HttpWsCallResult -import org.evomaster.core.problem.rest.data.HttpVerb -import org.evomaster.core.problem.rest.data.RestCallAction -import org.evomaster.core.problem.rest.data.RestIndividual -import org.evomaster.core.problem.rpc.RPCCallAction -import org.evomaster.core.problem.rpc.RPCIndividual -import org.evomaster.core.problem.webfrontend.WebIndividual -import org.evomaster.core.search.EvaluatedIndividual +import org.evomaster.core.EMConfig +import org.evomaster.core.output.naming.TestCaseNamingStrategyFactory +import org.evomaster.core.output.sorting.SortingHelper import org.evomaster.core.search.Solution -import org.slf4j.Logger -import org.slf4j.LoggerFactory -import kotlin.reflect.KFunction1 -/** - * This class is responsible to decide the order in which - * the test cases should be written in the final test suite. - * Ideally, the most "interesting" tests should be written first. - * - *
- * - * Furthermore, this class is also responsible for deciding which - * name each test will have - */ -class TestSuiteOrganizer { - private val sortingHelper = SortingHelper() - - private val defaultSorting = listOf(0, 1) - - fun sortTests(solution: Solution<*>, namingStrategy: TestCaseNamingStrategy, testCaseSortingStrategy: SortingStrategy): List { - //sortingHelper.selectCriteriaByIndex(defaultSorting) - //TODO here in the future we will have something a bit smarter - return sortingHelper.sort(solution, namingStrategy, testCaseSortingStrategy) - } -} - -class NamingHelper { - /** - * The presence of a call with a 500 status code will be added to the test name. - */ - private fun criterion1_500 (individual: EvaluatedIndividual<*>): String{ - if (individual.seeResults().filterIsInstance().any{ it.getStatusCode() == 500 }){ - return "_with500" - } - return "" - } - - private fun criterion2_hasPost (individual: EvaluatedIndividual<*>): String{ - if(individual.individual.seeAllActions().filterIsInstance().any{it.verb == HttpVerb.POST} ){ - return "_hasPost" - } +class TestSuiteOrganizer( + private val config: EMConfig +) { - return "" - } + private val sortingHelper = SortingHelper() - /** - * The type of sample is added to the name. This is tied to the the [RestIndividual] and will change with a new problem. - */ - private fun criterion3_sampling(individual: EvaluatedIndividual<*>): String{ - if(individual.individual is RestIndividual) - return "_" + (individual.individual as RestIndividual).sampleType - else return "" - } /** - * The presence of separate steps for DB initialization will be added to the test name. This is currently tied to - * the [RestIndividual] and will change with a new problem - */ - private fun criterion4_dbInit(individual: EvaluatedIndividual<*>): String{ - if ((individual.individual is RestIndividual) && (individual.individual as RestIndividual).seeInitializingActions().isNotEmpty()){ - return "_" + "hasDbInit" - } - else return "" - } - -// private fun criterion5_partialOracle(individual: EvaluatedIndividual<*>): String{ -// var name = "" -// partialOracles.adjustName().forEach { -// if(!it.adjustName().isNullOrBlank() -// && it.generatesExpectation(individual)){ -// name = name + it.adjustName() -// } -// } -// return name -// } - -// fun setPartialOracles(partialOracles: PartialOracles){ -// this.partialOracles = partialOracles -// } - -// private var partialOracles = PartialOracles() - private var namingCriteria = listOf(::criterion1_500 ) //, ::criterion5_partialOracle) - private val availableCriteria = listOf(::criterion1_500, ::criterion2_hasPost, ::criterion3_sampling, ::criterion4_dbInit) //, ::criterion5_partialOracle) - - - fun suggestName(individual: EvaluatedIndividual<*>): String{ - return namingCriteria.map { it(individual) }.joinToString("") - } - - fun getAvailableCriteria(): List, String>> { - return availableCriteria - } - - fun selectCriteria(selected: List, String>>){ - if (availableCriteria.containsAll(selected)){ - namingCriteria = selected - } - else { - throw UnsupportedOperationException("The naming criteria chosen appear to not be supported at the moment.") - } - } - - fun selectCriteriaByIndex(selected: List){ - if (availableCriteria.indices.toList().containsAll(selected)){ - for (i in selected) - namingCriteria = availableCriteria.filterIndexed{ index, _ -> - selected.contains(index) - } as List, String>> - } - else { - throw UnsupportedOperationException("The naming criteria chosen appear to not be supported at the moment.") - } - } - - -} - - -class SortingHelper { - - companion object { - private val log: Logger = LoggerFactory.getLogger(SortingHelper::class.java) - } - - /** [maxStatusCode] sorts Evaluated individuals based on the highest status code (e.g., 500s are first). + * This method is responsible to decide the order in which + * the test cases should be written in the final test suite. + * Ideally, the most "interesting" tests should be written first. + * + *
+ * + * Furthermore, this class is also responsible for deciding which + * name each test will have. + * + *
+ * WARNING: side-effect of sorting tests inside input [solution] object * - * **/ - private val maxStatusCode: Comparator> = compareBy>{ ind -> - val max = ind.seeResults().filterIsInstance().maxByOrNull { it.getStatusCode()?:0 } - (max as HttpWsCallResult).getStatusCode() ?: 0 - }.reversed() - - /** - * [statusCode] sorts Evaluated individuals based on the status code, as follows: - * - first: 5xx - * - second: 2xx - * - third: 4xx - */ - - - private val statusCode: Comparator> = compareBy { ind -> - val min = ind.seeResults().filterIsInstance().minByOrNull { - it.getStatusCode()?.rem(500) ?: 0 - } - (min?.getStatusCode())?.rem(500) ?: 0 - } - - /** [maxActions] sorts Evaluated individuals based on the number of actions (most actions first). - */ - private val maxActions: Comparator> = compareBy>{ ind -> - ind.individual.seeAllActions().size - }.reversed() - - /** [minActions] sorts Evaluated individuals based on the number of actions (most actions first). - */ - private val minActions: Comparator> = compareBy { ind -> - ind.individual.seeAllActions().size - } - - /** - * dbInitSize sorts [EvaluatedIndividual] objects on the basis of the presence (and number) of db initialization actions. - * Currently, this is only supported for [RestIndividual]. - * Note, writing the comparator as [EvaluatedIndividual>] seems to break the .sortWith() later on. - */ - private val dbInitSize: Comparator> = compareBy>{ ind -> - if(ind.individual is RestIndividual) { - ind.individual.seeInitializingActions().size - } - else 0 - }.reversed() - - /** - * [coveredTargets] sorts [EvaluatedIndividual] objects on based on the higher number of covered targets. - * The purpose is to give an example of sorting based on fitness information. - */ - private val coveredTargets: Comparator> = compareBy { - it.fitness.numberOfCoveredTargets() - } - - /** - * [comparatorList] holds those comparators that are currently selected for sorting - * Note that the order of the comparators is the order their importance/priority. */ + fun createSortedTestCases(solution: Solution<*>): List { - var comparatorList = listOf(statusCode, coveredTargets) - - val restComparator: Comparator> = compareBy> { ind -> - (ind.evaluatedMainActions().last().action as RestCallAction).path.levels() - } - .thenBy { ind -> - val min = ind.seeResults().filterIsInstance().minByOrNull { - it.getStatusCode()?.rem(500) ?: 0 - } - (min?.getStatusCode())?.rem(500) ?: 0 - } - .thenBy { ind -> - (ind.evaluatedMainActions().last().action as RestCallAction).verb - } - - val graphQLComparator: Comparator> = compareBy> { ind -> - (ind.evaluatedMainActions().last().action as GraphQLAction).methodName - } - .thenBy { ind -> - (ind.evaluatedMainActions().last().action as GraphQLAction).methodType - } - .thenBy { ind -> - (ind.evaluatedMainActions().last().action as GraphQLAction).parameters.size - } - - val rpcComparator: Comparator> = compareBy> { ind -> - (ind.evaluatedMainActions().last().action as RPCCallAction).getSimpleClassName() - } - .thenBy { ind -> - (ind.evaluatedMainActions().last().action as RPCCallAction).getExecutedFunctionName() - } - .thenBy { ind -> - (ind.evaluatedMainActions().last().action as RPCCallAction).parameters.size - } - - private val availableSortCriteria = listOf(statusCode, minActions, coveredTargets, maxStatusCode, maxActions, dbInitSize) + /* + Tests MUST be sorted before they are named, as their position might influence + their name (eg, "test_0_...") + */ + sortingHelper.sort(solution.individuals, config.testCaseSortingStrategy) + val namingStrategy = TestCaseNamingStrategyFactory(config).create(solution) + val tests = namingStrategy.getTestCases() - fun getAvailableCriteria(): List>> { - return availableSortCriteria + return tests } - fun selectCriteria(selected: List>>){ - if (availableSortCriteria.containsAll(selected)){ - comparatorList = selected - } - else { - throw UnsupportedOperationException("The sorting criteria chosen appear to not be supported at the moment.") - } - } - fun selectCriteriaByIndex(selected: List){ - if (availableSortCriteria.indices.toList().containsAll(selected)){ - comparatorList = availableSortCriteria.filterIndexed{ index, _ -> - selected.contains(index) - } - } - else { - throw UnsupportedOperationException("The sorting criteria chosen appear to not be supported at the moment.") - } - } - - /** - *Sorting is done according to the comparator list. If no list is provided, individuals are sorted by max status. - */ - private fun sortByComparatorList (comparators: List>> = listOf(statusCode), - namingStrategy: TestCaseNamingStrategy - - ): List { - /** - * Comparisons, as far as I understand them, are done as follows: - * First, the list is sorted based on the first criterion. - * Then, the (now sorted) list, is sorted based on the second criterion. - * Where two values have equal priority with respect to the most recent sort, they maintain the order (and, thus, - * are still sorted according to the first criterion). - * - * So, a criterion with more priority overrides most other criteria, unless elements have the same value. - * If too many criteria are used, the ones that are lower on the priority list will not really have a chance to manifest. - * - * An example of how this approach is used: - * = first priority (thus, last to be executed and most likely to be observed) is the [statusCode]. Thus, every - * test case that contains a 500 code is at the top. - * = second priority (thus, second to last to be executed), is the [coveredTargets]. Thus, among those test cases - * that have the same code, the ones with the most covered targets will be at the top (among their sub-group). - */ - - return namingStrategy.getSortedTestCases(comparators) - } +} - private fun sortByTargetIncremental(solution: Solution<*>, namingStrategy: TestCaseNamingStrategy): List { - val individuals = solution.individuals - val comparator = when { - individuals.any { it.individual is RestIndividual } -> restComparator - individuals.any { it.individual is GraphQLIndividual } -> graphQLComparator - individuals.any { it.individual is RPCIndividual } -> rpcComparator - individuals.any { it.individual is WebIndividual } -> { - log.warn("Web individuals do not have action based test case naming yet. Defaulting to Numbered strategy.") - statusCode - } - else -> throw IllegalStateException("Unrecognized test individuals with no target incremental based sorting strategy set.") - } - return namingStrategy.getSortedTestCases(comparator) - } - fun sort(solution: Solution<*>, namingStrategy: TestCaseNamingStrategy, testCaseSortingStrategy: SortingStrategy): List { - val newSort = when (testCaseSortingStrategy) { - SortingStrategy.COVERED_TARGETS -> sortByComparatorList(comparatorList, namingStrategy) - SortingStrategy.TARGET_INCREMENTAL -> sortByTargetIncremental(solution, namingStrategy) - else -> throw IllegalStateException("Unrecognized sorting strategy $testCaseSortingStrategy") - } - Lazy.assert { solution.individuals.toSet() == newSort.map { it.test }.toSet()} - return newSort - } -} diff --git a/core/src/main/kotlin/org/evomaster/core/output/naming/ActionTestCaseNamingStrategy.kt b/core/src/main/kotlin/org/evomaster/core/output/naming/ActionTestCaseNamingStrategy.kt index c02b497219..9e5f42a681 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/naming/ActionTestCaseNamingStrategy.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/naming/ActionTestCaseNamingStrategy.kt @@ -95,7 +95,7 @@ abstract class ActionTestCaseNamingStrategy( protected fun namePrefixChars(): Int { val digitsUsedForTestNumbering = testCasesSize.toString().length - return "test_".length + digitsUsedForTestNumbering + 1 + return TEST_NAME_PREFIX.length + digitsUsedForTestNumbering + 1 } protected fun addNameTokensIfAllowed(nameTokens: MutableList, targetStrings: List, remainingNameChars: Int): Int { diff --git a/core/src/main/kotlin/org/evomaster/core/output/naming/NamingHelper.kt b/core/src/main/kotlin/org/evomaster/core/output/naming/NamingHelper.kt new file mode 100644 index 0000000000..6cf3d497d4 --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/output/naming/NamingHelper.kt @@ -0,0 +1,99 @@ +package org.evomaster.core.output.naming + +import org.evomaster.core.problem.httpws.HttpWsCallResult +import org.evomaster.core.problem.rest.data.HttpVerb +import org.evomaster.core.problem.rest.data.RestCallAction +import org.evomaster.core.problem.rest.data.RestIndividual +import org.evomaster.core.search.EvaluatedIndividual +import kotlin.reflect.KFunction1 + +class NamingHelper { + /** + * The presence of a call with a 500 status code will be added to the test name. + */ + private fun criterion1_500 (individual: EvaluatedIndividual<*>): String{ + if (individual.seeResults().filterIsInstance().any{ it.getStatusCode() == 500 }){ + return "_with500" + } + return "" + } + + private fun criterion2_hasPost (individual: EvaluatedIndividual<*>): String{ + if(individual.individual.seeAllActions().filterIsInstance().any{it.verb == HttpVerb.POST} ){ + return "_hasPost" + } + + return "" + } + + /** + * The type of sample is added to the name. This is tied to the the [RestIndividual] and will change with a new problem. + */ + private fun criterion3_sampling(individual: EvaluatedIndividual<*>): String{ + if(individual.individual is RestIndividual) + return "_" + (individual.individual as RestIndividual).sampleType + else return "" + } + + /** + * The presence of separate steps for DB initialization will be added to the test name. This is currently tied to + * the [RestIndividual] and will change with a new problem + */ + private fun criterion4_dbInit(individual: EvaluatedIndividual<*>): String{ + if ((individual.individual is RestIndividual) && (individual.individual as RestIndividual).seeInitializingActions().isNotEmpty()){ + return "_" + "hasDbInit" + } + else return "" + } + +// private fun criterion5_partialOracle(individual: EvaluatedIndividual<*>): String{ +// var name = "" +// partialOracles.adjustName().forEach { +// if(!it.adjustName().isNullOrBlank() +// && it.generatesExpectation(individual)){ +// name = name + it.adjustName() +// } +// } +// return name +// } + +// fun setPartialOracles(partialOracles: PartialOracles){ +// this.partialOracles = partialOracles +// } + + // private var partialOracles = PartialOracles() + private var namingCriteria = listOf(::criterion1_500 ) //, ::criterion5_partialOracle) + private val availableCriteria = listOf(::criterion1_500, ::criterion2_hasPost, ::criterion3_sampling, ::criterion4_dbInit) //, ::criterion5_partialOracle) + + + fun suggestName(individual: EvaluatedIndividual<*>): String{ + return namingCriteria.map { it(individual) }.joinToString("") + } + + fun getAvailableCriteria(): List, String>> { + return availableCriteria + } + + fun selectCriteria(selected: List, String>>){ + if (availableCriteria.containsAll(selected)){ + namingCriteria = selected + } + else { + throw UnsupportedOperationException("The naming criteria chosen appear to not be supported at the moment.") + } + } + + fun selectCriteriaByIndex(selected: List){ + if (availableCriteria.indices.toList().containsAll(selected)){ + for (i in selected) + namingCriteria = availableCriteria.filterIndexed{ index, _ -> + selected.contains(index) + } as List, String>> + } + else { + throw UnsupportedOperationException("The naming criteria chosen appear to not be supported at the moment.") + } + } + + +} diff --git a/core/src/main/kotlin/org/evomaster/core/output/naming/NamingHelperNumberedTestCaseNamingStrategy.kt b/core/src/main/kotlin/org/evomaster/core/output/naming/NamingHelperNumberedTestCaseNamingStrategy.kt index 42bfa2d9a9..962c8b2575 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/naming/NamingHelperNumberedTestCaseNamingStrategy.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/naming/NamingHelperNumberedTestCaseNamingStrategy.kt @@ -1,6 +1,5 @@ package org.evomaster.core.output.naming -import org.evomaster.core.output.NamingHelper import org.evomaster.core.search.EvaluatedIndividual import org.evomaster.core.search.Solution diff --git a/core/src/main/kotlin/org/evomaster/core/output/naming/NamingStrategy.kt b/core/src/main/kotlin/org/evomaster/core/output/naming/NamingStrategy.kt index fbb023d6a0..8ab14b038b 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/naming/NamingStrategy.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/naming/NamingStrategy.kt @@ -1,12 +1,24 @@ package org.evomaster.core.output.naming +/** + * + */ enum class NamingStrategy { + /** + * Standard, naive approach. + * Each test gets a unique, incremental number + */ NUMBERED, - ACTION - ; - fun isNumbered() = this.name.startsWith("numbered", true) + /** + * Apply a deterministic set of rules based on the actions' content. + */ + DETERMINISTIC, - fun isAction() = this.name.startsWith("action", true) + /** + * Call an LLM to create the names based on the tests' content, if available + */ + LLM + ; } diff --git a/core/src/main/kotlin/org/evomaster/core/output/naming/NumberedTestCaseNamingStrategy.kt b/core/src/main/kotlin/org/evomaster/core/output/naming/NumberedTestCaseNamingStrategy.kt index da35a4e105..3320bdbf91 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/naming/NumberedTestCaseNamingStrategy.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/naming/NumberedTestCaseNamingStrategy.kt @@ -9,23 +9,15 @@ open class NumberedTestCaseNamingStrategy( solution: Solution<*> ) : TestCaseNamingStrategy(solution) { - override fun getTestCases(): List { - return generateNames(solution.individuals) + companion object{ + const val TEST_NAME_PREFIX = "test_" } - override fun getSortedTestCases(comparator: Comparator>): List { - return getSortedTestCases(singletonList(comparator)) + override fun getTestCases(): List { + return generateNames(solution.individuals) } - override fun getSortedTestCases(comparators: List>>): List { - val inds = solution.individuals - comparators.asReversed().forEach { - inds.sortWith(it) - } - - return generateNames(inds) - } // numbered strategy will not expand the name unless it is using the namingHelper override fun expandName(individual: EvaluatedIndividual<*>, nameTokens: MutableList, ambiguitySolvers: List): String { @@ -38,7 +30,7 @@ open class NumberedTestCaseNamingStrategy( } private fun concatName(counter: Int, expandedName: String): String { - return "test_${counter}${expandedName}" + return "$TEST_NAME_PREFIX${counter}${expandedName}" } private fun generateNames(individuals: List>) : List { diff --git a/core/src/main/kotlin/org/evomaster/core/output/naming/TestCaseNamingStrategy.kt b/core/src/main/kotlin/org/evomaster/core/output/naming/TestCaseNamingStrategy.kt index fccb6b2b0a..24d997526c 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/naming/TestCaseNamingStrategy.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/naming/TestCaseNamingStrategy.kt @@ -17,19 +17,6 @@ abstract class TestCaseNamingStrategy( */ abstract fun getTestCases(): List - /** - * @param comparator used to sort the test cases - * - * @return the list of sorted TestCase with the generated name given the naming strategy - */ - abstract fun getSortedTestCases(comparator: Comparator>): List - - /** - * @param comparators used to sort the test cases - * - * @return the list of sorted TestCase with the generated name given the naming strategy - */ - abstract fun getSortedTestCases(comparators: List>>): List /** * @param individual containing information for the test about to be named diff --git a/core/src/main/kotlin/org/evomaster/core/output/naming/TestCaseNamingStrategyFactory.kt b/core/src/main/kotlin/org/evomaster/core/output/naming/TestCaseNamingStrategyFactory.kt index de5766b513..50d46b7b86 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/naming/TestCaseNamingStrategyFactory.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/naming/TestCaseNamingStrategyFactory.kt @@ -24,26 +24,27 @@ class TestCaseNamingStrategyFactory( } fun create(solution: Solution<*>): TestCaseNamingStrategy { - return when { - namingStrategy.isNumbered() -> NamingHelperNumberedTestCaseNamingStrategy(solution) - namingStrategy.isAction() -> actionBasedNamingStrategy(solution) + return when(namingStrategy) { + NamingStrategy.NUMBERED -> NamingHelperNumberedTestCaseNamingStrategy(solution) + NamingStrategy.DETERMINISTIC -> deterministicActionBasedNamingStrategy(solution) + //TODO LLM else -> throw IllegalStateException("Unrecognized naming strategy $namingStrategy") } } - private fun actionBasedNamingStrategy(solution: Solution<*>): NumberedTestCaseNamingStrategy { + private fun deterministicActionBasedNamingStrategy(solution: Solution<*>): NumberedTestCaseNamingStrategy { val individuals = solution.individuals return when { - individuals.any { it.individual is RestIndividual } -> return RestActionTestCaseNamingStrategy(solution, languageConventionFormatter, nameWithQueryParameters, maxTestCaseNameLength) - individuals.any { it.individual is GraphQLIndividual } -> return GraphQLActionTestCaseNamingStrategy(solution, languageConventionFormatter, maxTestCaseNameLength) - individuals.any { it.individual is RPCIndividual } -> return RPCActionTestCaseNamingStrategy(solution, languageConventionFormatter, maxTestCaseNameLength) + individuals.any { it.individual is RestIndividual } -> RestActionTestCaseNamingStrategy(solution, languageConventionFormatter, nameWithQueryParameters, maxTestCaseNameLength) + individuals.any { it.individual is GraphQLIndividual } -> GraphQLActionTestCaseNamingStrategy(solution, languageConventionFormatter, maxTestCaseNameLength) + individuals.any { it.individual is RPCIndividual } -> RPCActionTestCaseNamingStrategy(solution, languageConventionFormatter, maxTestCaseNameLength) individuals.any { it.individual is WebIndividual } -> { log.warn("Web individuals do not have action based test case naming yet. Defaulting to Numbered strategy.") - return NamingHelperNumberedTestCaseNamingStrategy(solution) + NamingHelperNumberedTestCaseNamingStrategy(solution) } individuals.isEmpty() -> { log.warn("No individuals present in the solution. Defaulting to Numbered strategy.") - return NumberedTestCaseNamingStrategy(solution) + NumberedTestCaseNamingStrategy(solution) } else -> throw IllegalStateException("Unrecognized test individuals with no action based naming strategy set.") } diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt index fe08c169ec..071f6ef3b3 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt @@ -137,8 +137,7 @@ class TestSuiteWriter { ): TestSuiteCode { val lines = Lines(config.outputFormat) - val testSuiteOrganizer = TestSuiteOrganizer() - val namingStrategy = TestCaseNamingStrategyFactory(config).create(solution) + val testSuiteOrganizer = TestSuiteOrganizer(config) header(solution, testSuiteFileName, lines, timestamp, controllerName) @@ -153,10 +152,11 @@ class TestSuiteWriter { beforeAfterMethods(solution, controllerName, controllerInput, lines, config.outputFormat, testSuiteFileName) + //FIXME should solve all problems that happen in the EM tests //catch any sorting problems (see NPE is SortingHelper on Trello) val tests = try { // TODO skip to sort RPC for the moment - testSuiteOrganizer.sortTests(solution, namingStrategy, config.testCaseSortingStrategy) + testSuiteOrganizer.createSortedTestCases(solution) } catch (ex: Exception) { log.warn( "A failure has occurred with the test sorting. Reverting to default settings. \n" @@ -167,6 +167,7 @@ class TestSuiteWriter { NumberedTestCaseNamingStrategy(solution).getTestCases() } + val testSuitePath = getTestSuitePath(testSuiteFileName, config) val tc = tests.mapNotNull { test -> diff --git a/core/src/main/kotlin/org/evomaster/core/output/sorting/SortingHelper.kt b/core/src/main/kotlin/org/evomaster/core/output/sorting/SortingHelper.kt new file mode 100644 index 0000000000..7245928412 --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/output/sorting/SortingHelper.kt @@ -0,0 +1,187 @@ +package org.evomaster.core.output.sorting + +import org.evomaster.core.output.TestCase +import org.evomaster.core.output.naming.TestCaseNamingStrategy +import org.evomaster.core.problem.graphql.GraphQLAction +import org.evomaster.core.problem.graphql.GraphQLIndividual +import org.evomaster.core.problem.httpws.HttpWsCallResult +import org.evomaster.core.problem.rest.data.RestCallAction +import org.evomaster.core.problem.rest.data.RestIndividual +import org.evomaster.core.problem.rpc.RPCCallAction +import org.evomaster.core.problem.rpc.RPCIndividual +import org.evomaster.core.problem.webfrontend.WebIndividual +import org.evomaster.core.search.EvaluatedIndividual +import org.evomaster.core.search.Solution +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import java.util.Collections.singletonList + +class SortingHelper { + + companion object { + private val log: Logger = LoggerFactory.getLogger(SortingHelper::class.java) + } + + + + /** [maxStatusCode] sorts Evaluated individuals based on the highest status code (e.g., 500s are first). + * + * **/ + private val maxStatusCode: Comparator> = compareBy>{ ind -> + val max = ind.seeResults().filterIsInstance().maxByOrNull { it.getStatusCode()?:0 } + (max as HttpWsCallResult).getStatusCode() ?: 0 + }.reversed() + + /** + * [statusCode] sorts Evaluated individuals based on the status code, as follows: + * - first: 5xx + * - second: 2xx + * - third: 4xx + */ + private val statusCode: Comparator> = compareBy { ind -> + val min = ind.seeResults().filterIsInstance().minByOrNull { + it.getStatusCode()?.rem(500) ?: 0 + } + (min?.getStatusCode())?.rem(500) ?: 0 + } + + /** [maxActions] sorts Evaluated individuals based on the number of actions (most actions first). + */ + private val maxActions: Comparator> = compareBy>{ ind -> + ind.individual.seeAllActions().size + }.reversed() + + /** [minActions] sorts Evaluated individuals based on the number of actions (most actions first). + */ + private val minActions: Comparator> = compareBy { ind -> + ind.individual.seeAllActions().size + } + + /** + * dbInitSize sorts [EvaluatedIndividual] objects on the basis of the presence (and number) of db initialization actions. + * Currently, this is only supported for [RestIndividual]. + * Note, writing the comparator as [EvaluatedIndividual>] seems to break the .sortWith() later on. + */ + private val dbInitSize: Comparator> = compareBy>{ ind -> + if(ind.individual is RestIndividual) { + ind.individual.seeInitializingActions().size + } + else 0 + }.reversed() + + /** + * [coveredTargets] sorts [EvaluatedIndividual] objects on based on the higher number of covered targets. + * The purpose is to give an example of sorting based on fitness information. + */ + private val coveredTargets: Comparator> = compareBy { + it.fitness.numberOfCoveredTargets() + } + + /** + * [comparatorList] holds those comparators that are currently selected for sorting + * Note that the order of the comparators is the order their importance/priority. + */ + private val comparatorList = listOf(statusCode, coveredTargets) + + private val restComparator: Comparator> = compareBy> { ind -> + (ind.evaluatedMainActions().last().action as RestCallAction).path.levels() + } + .thenBy { ind -> + val min = ind.seeResults().filterIsInstance().minByOrNull { + it.getStatusCode()?.rem(500) ?: 0 + } + (min?.getStatusCode())?.rem(500) ?: 0 + } + .thenBy { ind -> + (ind.evaluatedMainActions().last().action as RestCallAction).verb + } + + private val graphQLComparator: Comparator> = compareBy> { ind -> + (ind.evaluatedMainActions().last().action as GraphQLAction).methodName + } + .thenBy { ind -> + (ind.evaluatedMainActions().last().action as GraphQLAction).methodType + } + .thenBy { ind -> + (ind.evaluatedMainActions().last().action as GraphQLAction).parameters.size + } + + private val rpcComparator: Comparator> = compareBy> { ind -> + (ind.evaluatedMainActions().last().action as RPCCallAction).getSimpleClassName() + } + .thenBy { ind -> + (ind.evaluatedMainActions().last().action as RPCCallAction).getExecutedFunctionName() + } + .thenBy { ind -> + (ind.evaluatedMainActions().last().action as RPCCallAction).parameters.size + } + + + fun sort(tests: MutableList>, testCaseSortingStrategy: SortingStrategy) { + when (testCaseSortingStrategy) { + SortingStrategy.COVERED_TARGETS -> sortByComparatorList(tests, comparatorList) + SortingStrategy.TARGET_INCREMENTAL -> sortByTargetIncremental(tests) + else -> throw IllegalStateException("Unrecognized sorting strategy $testCaseSortingStrategy") + } + } + + @Deprecated("Use other version") + fun sort( + solution: Solution<*>, + namingStrategy: TestCaseNamingStrategy, + testCaseSortingStrategy: SortingStrategy + ) : List { + sort(solution.individuals, testCaseSortingStrategy) + return namingStrategy.getTestCases() + } + + + private fun sortByComparator(tests: MutableList>, comparator: Comparator>) { + sortByComparatorList(tests, singletonList(comparator)) + } + + /** + * Sorting is done according to the comparator list. + * Comparisons, are done as follows: + * First, the list is sorted based on the first criterion. + * Then, the (now sorted) list, is sorted based on the second criterion. + * Where two values have equal priority with respect to the most recent sort, they maintain the order (and, thus, + * are still sorted according to the first criterion). + * + * So, a criterion with more priority overrides most other criteria, unless elements have the same value. + * If too many criteria are used, the ones that are lower on the priority list will not really have a chance to manifest. + * + * An example of how this approach is used: + * = first priority (thus, last to be executed and most likely to be observed) is the [statusCode]. Thus, every + * test case that contains a 500 code is at the top. + * = second priority (thus, second to last to be executed), is the [coveredTargets]. Thus, among those test cases + * that have the same code, the ones with the most covered targets will be at the top (among their sub-group). + */ + private fun sortByComparatorList(tests: MutableList>, comparators: List>>) { + comparators.asReversed().forEach { comp -> + tests.sortWith(comp) + } + } + + private fun sortByTargetIncremental(tests: MutableList>) { + + if(tests.isEmpty()){ + return + } + + val comparator = when { + tests.any { it.individual is RestIndividual } -> restComparator + tests.any { it.individual is GraphQLIndividual } -> graphQLComparator + tests.any { it.individual is RPCIndividual } -> rpcComparator + tests.any { it.individual is WebIndividual } -> { + log.warn("Web individuals do not have action based test case naming yet. Defaulting to Numbered strategy.") + statusCode + } + else -> throw IllegalStateException("Unrecognized test individuals with no target incremental based sorting strategy set.") + } + + sortByComparator(tests, comparator) + } + + +} \ No newline at end of file diff --git a/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt b/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt index 0e021a8fd6..a31343e4a9 100644 --- a/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt @@ -685,7 +685,7 @@ internal class EMConfigTest{ config.updateProperties(parser.parse()) - assertEquals(NamingStrategy.ACTION, config.namingStrategy) + assertEquals(NamingStrategy.DETERMINISTIC, config.namingStrategy) } @Test @@ -693,10 +693,10 @@ internal class EMConfigTest{ val parser = EMConfig.getOptionParser() val config = EMConfig() - val options = parser.parse("--namingStrategy", "ACTION", "--nameWithQueryParameters", "true") + val options = parser.parse("--namingStrategy", "DETERMINISTIC", "--nameWithQueryParameters", "true") config.updateProperties(options) - assertEquals(NamingStrategy.ACTION, config.namingStrategy) + assertEquals(NamingStrategy.DETERMINISTIC, config.namingStrategy) assertTrue(config.nameWithQueryParameters) } @@ -705,10 +705,10 @@ internal class EMConfigTest{ val parser = EMConfig.getOptionParser() val config = EMConfig() - val options = parser.parse("--namingStrategy", "ACTION", "--nameWithQueryParameters", "false") + val options = parser.parse("--namingStrategy", "DETERMINISTIC", "--nameWithQueryParameters", "false") config.updateProperties(options) - assertEquals(NamingStrategy.ACTION, config.namingStrategy) + assertEquals(NamingStrategy.DETERMINISTIC, config.namingStrategy) assertFalse(config.nameWithQueryParameters) } diff --git a/core/src/test/kotlin/org/evomaster/core/output/TestSuiteOrganizerTest.kt b/core/src/test/kotlin/org/evomaster/core/output/TestSuiteOrganizerTest.kt index 55075ba0e6..8adef7e4c5 100644 --- a/core/src/test/kotlin/org/evomaster/core/output/TestSuiteOrganizerTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/output/TestSuiteOrganizerTest.kt @@ -4,6 +4,7 @@ import org.evomaster.core.TestUtils import org.evomaster.core.output.naming.NumberedTestCaseNamingStrategy import org.evomaster.core.output.naming.RestActionTestCaseUtils.getEvaluatedIndividualWith import org.evomaster.core.output.naming.RestActionTestCaseUtils.getRestCallAction +import org.evomaster.core.output.sorting.SortingHelper import org.evomaster.core.output.sorting.SortingStrategy import org.evomaster.core.problem.api.param.Param import org.evomaster.core.problem.enterprise.EnterpriseActionGroup diff --git a/docs/options.md b/docs/options.md index 75c6de0945..aa6b5227d1 100644 --- a/docs/options.md +++ b/docs/options.md @@ -178,7 +178,7 @@ There are 3 types of options: |`muPlusLambdaOffspringSize`| __Int__. Define the number of offspring (λ) generated per generation in (μ+λ) Evolutionary Algorithm. *Constraints*: `min=1.0`. *Default value*: `30`.| |`mutatedGeneFile`| __String__. Specify a path to save mutation details which is useful for debugging mutation. *DEBUG option*. *Default value*: `mutatedGeneInfo.csv`.| |`nameWithQueryParameters`| __Boolean__. Specify if true boolean query parameters are included in the test case name. Used for test case naming disambiguation. Only valid for Action based naming strategy. *Default value*: `true`.| -|`namingStrategy`| __Enum__. Specify the naming strategy for test cases. *Valid values*: `NUMBERED, ACTION`. *Default value*: `ACTION`.| +|`namingStrategy`| __Enum__. Specify the naming strategy for test cases. *Valid values*: `NUMBERED, DETERMINISTIC, LLM`. *Default value*: `DETERMINISTIC`.| |`outputExecutedSQL`| __Enum__. Whether to output executed sql info. *DEBUG option*. *Valid values*: `NONE, ALL_AT_END, ONCE_EXECUTED`. *Default value*: `NONE`.| |`overrideAuthExternalEndpointURL`| __String__. Override the value of externalEndpointURL in auth configurations. This is useful when the auth server is running locally on an ephemeral port, or when several instances are run in parallel, to avoid creating/modifying auth configuration files. If what provided is a URL starting with 'http', then full replacement will occur. Otherwise, the input will be treated as a 'hostname:port', and only that info will be updated (e.g., path element of the URL will not change). *Default value*: `null`.| |`populationSize`| __Int__. Define the population size in the search algorithms that use populations (e.g., Genetic Algorithms, but not MIO). *Constraints*: `min=1.0`. *Default value*: `30`.| diff --git a/scripts/loopback-aliases-macos.sh b/scripts/loopback-aliases-macos.sh index 7bc6316324..3295b6186d 100755 --- a/scripts/loopback-aliases-macos.sh +++ b/scripts/loopback-aliases-macos.sh @@ -77,3 +77,7 @@ then else echo "No operation selected to execute" fi + + +### On shell, can also just run: +### for i in {2..255}; do sudo ifconfig lo0 alias 127.0.0.$i up; done \ No newline at end of file