Naming phase

core-tests/e2e-tests/e2e-tests-utils/src/test/java/org/evomaster/e2etests/utils/EnterpriseTestBase.java

@@ -45,8 +45,10 @@
 import java.util.List;
 import java.util.Objects;
 import java.util.function.Consumer;
-import java.util.function.Function;
 import java.util.function.Predicate;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 import static org.junit.jupiter.api.Assertions.*;
 
@@ -639,7 +641,31 @@ protected void assertTextInTests(String outputFolder, String className, Predicat
         }catch (IOException e){
             throw new IllegalStateException("Fail to get the test "+className+" in "+outputFolder+" with error "+ e.getMessage());
         }
+    }
+
+    protected List<String> getFunctionNames(String outputFolder, String className){
+
+        //this specific on how we create tests
+        String regex = "\\s*fun\\s.*\\(\\)\\s*\\{\\s*";
+        Pattern pattern = Pattern.compile(regex);
 
+        String path = outputFolderPath(outputFolder)+ "/"+String.join("/", className.split("\\."))+".kt";
+        Path test = Paths.get(path);
+        try {
+            return Files.lines(test)
+                    .filter(it -> {
+                        Matcher matcher = pattern.matcher(it);
+                        return matcher.find();
+                    } )
+                    .map(it -> {
+                        int start = it.indexOf("fun") + 3;
+                        int end = it.indexOf("()");
+                        return it.substring(start, end).trim();
+                    })
+                    .collect(Collectors.toList());
+        }catch (IOException e){
+            throw new IllegalStateException("Fail to get the test "+className+" in "+outputFolder+" with error "+ e.getMessage());
+        }
     }
 
     /**

core-tests/jdk-8/spring-rest-openapi-v2-tests/src/test/java/org/evomaster/e2etests/spring/examples/sort/SortEMTest.java

@@ -1,10 +1,9 @@
 package org.evomaster.e2etests.spring.examples.sort;
 
-import org.evomaster.core.output.TestCase;
-import org.evomaster.core.output.TestSuiteOrganizer;
+
+import org.evomaster.core.output.naming.NamingStrategy;
 import org.evomaster.core.output.naming.NumberedTestCaseNamingStrategy;
 import org.evomaster.core.output.naming.TestCaseNamingStrategy;
-import org.evomaster.core.output.sorting.SortingStrategy;
 import org.evomaster.core.problem.rest.data.HttpVerb;
 import org.evomaster.core.problem.rest.data.RestCallResult;
 import org.evomaster.core.problem.rest.data.RestIndividual;
@@ -16,7 +15,9 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.OptionalInt;
+import java.util.stream.Collectors;
 
+import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 /**
@@ -29,11 +30,17 @@ public class SortEMTest extends NRTestBase {
     @Test
     public void testRunEM() throws Throwable {
 
+        String outputFolderName = "SortEM";
+        String className = "org.bar.SortEM";
+
         runTestHandlingFlakyAndCompilation(
-                "SortEM",
-                "org.bar.SortEM",
+                outputFolderName,
+                className,
                 3_000,
                 (args) -> {
+
+                    setOption(args, "namingStrategy", NamingStrategy.DETERMINISTIC.name());
+
                     Solution<RestIndividual> solution = initAndRun(args);
 
                     assertTrue(solution.getIndividuals().size() >= 1);
@@ -43,64 +50,26 @@ public void testRunEM() throws Throwable {
                     assertHasAtLeastOne(solution, HttpVerb.PUT, 500);
                     assertHasAtLeastOne(solution, HttpVerb.POST, 500);
 
-                    TestSuiteOrganizer organizer = new TestSuiteOrganizer();
-
-                    TestCaseNamingStrategy namingStrategy = new NumberedTestCaseNamingStrategy(solution);
-
-                    List<TestCase> tclist = organizer.sortTests(solution, namingStrategy, SortingStrategy.COVERED_TARGETS);
-
-                    //Iterator<TestCase> iterator = tclist.iterator();
-                    //TestCase current, previous = iterator.next();
-                    /*
-                    while(iterator.hasNext()){
-                        current = iterator.next();
-                        // Check that a TC with 500 in the name does not follow a TC without a 500 in the name (500 should be first).
-
-                        if(current.getName().contains("500")){
-                            assertTrue(previous.getName().contains("500"));
-                        }
-                        previous = current;
-                     }
-                     */
-
-                    Iterator<EvaluatedIndividual<RestIndividual>> iterator = solution.getIndividuals().iterator();
-                    EvaluatedIndividual<RestIndividual> current, previous = iterator.next();
-
-
-
-                    while(iterator.hasNext()){
-                        current = iterator.next();
-
-                        if (current.seeResults(null).stream()
-                                .filter(w -> w instanceof RestCallResult)
-                                .anyMatch(r -> ((RestCallResult) r).getStatusCode() == 500)) {
-
-                            assertTrue(previous.seeResults(null).stream()
-                                    .filter(w -> w instanceof RestCallResult)
-                                    .anyMatch(r -> ((RestCallResult) r).getStatusCode() == 500));
-                        }
-
-
-
-                        // Check that the current "priority code" is less than the previous priority code
-
-                        OptionalInt currentPrioCode = current.seeResults(null).stream()
-                                .filter(w -> w instanceof RestCallResult)
-                                .mapToInt(w -> ((RestCallResult) w).getStatusCode())
-                                .map(w -> w % 500)
-                                .min();
-
-                        OptionalInt previousPrioCode = previous.seeResults(null).stream()
-                                .filter(w -> w instanceof RestCallResult)
-                                .mapToInt(w -> ((RestCallResult) w).getStatusCode())
-                                .map(w -> w % 500)
-                                .min();
-
-                        assertTrue(currentPrioCode.getAsInt() >= previousPrioCode.getAsInt());
-                        previous = current;
-
-                }
-
+                    List<String> functionNames = getFunctionNames(outputFolderName, className);
+                    // 3 initializations (eg @BeforEach), plus 1 per individuals
+                    int expected = 3 + solution.getIndividuals().size();
+                    assertEquals(expected, functionNames.size());
+
+                    String prefix = NumberedTestCaseNamingStrategy.TEST_NAME_PREFIX;
+                    List<String> testNames = functionNames.stream()
+                            .filter(it -> it.startsWith(prefix))
+                            .collect(Collectors.toList());
+                    assertEquals(solution.getIndividuals().size(), testNames.size());
+
+                    for(int i=0; i < testNames.size(); i++){
+                        String name = testNames.get(i);
+                        int index = Integer.parseInt(
+                                name.substring(prefix.length(), name.indexOf("_", prefix.length() + 1))
+                        );
+                        assertEquals(i, index,
+                                "Wrong numbering." +
+                                        " Expected: " + i + ", but found " + index + " for test name " + name);
+                    }
                 });
     }
 

core/src/main/kotlin/org/evomaster/core/EMConfig.kt

@@ -79,7 +79,7 @@ class EMConfig {
 
         private val defaultOutputFormatForBlackBox = OutputFormat.PYTHON_UNITTEST
 
-        private val defaultTestCaseNamingStrategy = NamingStrategy.ACTION
+        private val defaultTestCaseNamingStrategy = NamingStrategy.DETERMINISTIC
 
         private val defaultTestCaseSortingStrategy = SortingStrategy.TARGET_INCREMENTAL
 
@@ -870,6 +870,10 @@ class EMConfig {
             if (sutJarEnvVarName.isEmpty())
                 throw ConfigProblemException("'sutJarEnvVarName' must be specified if 'useEnvVarsForPathInTests' is enabled.")
         }
+
+        if(namingStrategy == NamingStrategy.LLM && !llm){
+            throw ConfigProblemException("Naming strategy LLM require the setup and use of an LLM")
+        }
     }
 
     private fun checkPropertyConstraints(m: KMutableProperty<*>) {

core/src/main/kotlin/org/evomaster/core/llm/service/LlmService.kt

@@ -6,6 +6,7 @@ import org.evomaster.core.EMConfig
 import org.evomaster.core.config.ConfigProblemException
 import org.evomaster.core.llm.LlmSupport
 import java.util.concurrent.CompletableFuture
+import java.util.concurrent.Future
 import javax.annotation.PostConstruct
 import javax.inject.Inject
 
@@ -44,7 +45,7 @@ class LlmService {
         }
     }
 
-    fun chatAsync(userMessage: String): CompletableFuture<String>{
+    fun chatAsync(userMessage: String): Future<String> {
         checkUsingLLM()
 
         return LlmSupport.chatAsync(model, userMessage)