Merge pull request #1610 from WebFuzzing/phg/llm-naming-json

Set LLM naming strategy prompts to return a json answer

Author: arcuri82

core/src/main/kotlin/org/evomaster/core/llm/Prompts.kt

@@ -56,7 +56,7 @@ object Prompts {
     const val NEW_TEST_CASE_NAME = """
         You are an expert software engineer specializing in test naming.
 
-        Given the following test case written in [targetLanguage], produce a descriptive suffix to append to its existing name.
+        Given the following test case written in [targetLanguage], produce a descriptive suffix to append to its existing name. Return a JSON string.
 
         ## Rules
 
@@ -87,7 +87,7 @@ object Prompts {
         [testLines]
     """
 
-    const val RE_ITERATE_TEST_CASE_NAME = "Your previous response contained more than just the suffix. Output only the suffix, nothing else. No explanation, no punctuation, no extra text, do not exceed max chars."
+    const val RE_ITERATE_TEST_CASE_NAME = "Your previous response contained more than just the suffix. Output only the suffix, nothing else. No explanation, no punctuation, no extra text, do not exceed max chars. Return a JSON string."
 
     fun getPromptForNameDescription(name: String, description: String?): Pair<String,String> {
         var user = "Your input is\n [name]:$name"

core/src/main/kotlin/org/evomaster/core/output/naming/LlmServiceTestCaseNamingStrategy.kt

@@ -23,6 +23,7 @@ class LlmServiceTestCaseNamingStrategy(
 ) : NumberedTestCaseNamingStrategy(solution) {
 
     private val log: Logger = LoggerFactory.getLogger(TestSuiteWriter::class.java)
+    private val fallbackLlmTestCaseName = "llmInteractionFailed_reviewName"
     private val generatedNames = mutableSetOf<String>()
 
     private val remainingNameChars = maxTestCaseNameLength - namePrefixChars()
@@ -38,8 +39,14 @@ class LlmServiceTestCaseNamingStrategy(
 
     private fun generateLlmName(test: TestCase): String {
         var newName = sanitizeName(getNewName(test))
-        while (!isValidSuffix(newName)) {
+        if (!isValidSuffix(newName)) {
             newName = sanitizeName(promptReIterateName())
+            if (!isValidSuffix(newName)) {
+                // If prompting the LLM to re-iterate the naming returned an invalid name again,
+                // then we fall back to a default name. Since this is a special case that should not happen,
+                // this name is not added to the list of names the LLM is provided to avoid repetition
+                return fallbackLlmTestCaseName
+            }
         }
         generatedNames.add(newName)
         return newName

core/src/test/kotlin/org/evomaster/core/output/naming/LlmServiceTestCaseNamingStrategyTest.kt

@@ -32,7 +32,9 @@ import kotlin.jvm.java
 class LlmServiceTestCaseNamingStrategyTest {
 
     companion object {
-        const val MAX_NAME_LENGTH = 80
+        // Setting a shorter name length just to force a condition in which the LLM response does not comply
+        // with the ask since the mock will return a longer name, we force the re-prompt and later the fallback name
+        const val MAX_NAME_LENGTH = 40
     }
 
     @Test
@@ -69,6 +71,23 @@ class LlmServiceTestCaseNamingStrategyTest {
         assertEquals("test_0_thisTestNameWasLlmGenerated", testCases[0].name)
     }
 
+    @Test
+    fun testFallbackNameIsAssignedAfterTwoFailedPrompts() {
+        val outputFormat = OutputFormat.JAVA_JUNIT_4
+        val baseModule = BaseModule(arrayOf("--llm", "true", "--llmProvider", "MOCK", "--outputFormat", outputFormat.name))
+        val llmService = getLlmService(baseModule)
+        val graphTestCaseWriter = getTestCaseWriter(baseModule.getEMConfig())
+
+        MockChatModel.reset()
+        MockChatModel.mockResponse("[\"aNameThatExceeds The Amount of Characters and has ! f\"]") { it.contains("[targetLanguage]:Java") }
+        MockChatModel.mockResponse("\"This name+ will !!! also have over 40chars and shouldBe_rejected") { it.contains("Your previous response") }
+
+        val testCases = generateTestCases(outputFormat, llmService, graphTestCaseWriter)
+
+        assertEquals(1, testCases.size)
+        assertEquals("test_0_llmInteractionFailed_reviewName", testCases[0].name)
+    }
+
     private fun getLlmService(baseModule: BaseModule): LlmService {
         val injector: Injector = LifecycleInjector.builder()
             .withModules(listOf<com.google.inject.Module>(baseModule))