|
| 1 | +package org.evomaster.core.output.naming |
| 2 | + |
| 3 | +import org.evomaster.core.llm.Prompts.RE_ITERATE_TEST_CASE_NAME |
| 4 | +import org.evomaster.core.llm.Prompts.getPromptForTestCaseName |
| 5 | +import org.evomaster.core.llm.service.LlmService |
| 6 | +import org.evomaster.core.output.Lines |
| 7 | +import org.evomaster.core.output.OutputFormat |
| 8 | +import org.evomaster.core.output.TestCase |
| 9 | +import org.evomaster.core.output.TestWriterUtils |
| 10 | +import org.evomaster.core.output.service.TestCaseWriter |
| 11 | +import org.evomaster.core.output.service.TestSuiteWriter |
| 12 | +import org.evomaster.core.search.EvaluatedIndividual |
| 13 | +import org.evomaster.core.search.Solution |
| 14 | +import org.slf4j.Logger |
| 15 | +import org.slf4j.LoggerFactory |
| 16 | + |
| 17 | +class LlmServiceTestCaseNamingStrategy( |
| 18 | + solution: Solution<*>, |
| 19 | + private val outputFormat: OutputFormat, |
| 20 | + private val llmService: LlmService, |
| 21 | + maxTestCaseNameLength: Int, |
| 22 | + private val testCaseWriter: TestCaseWriter |
| 23 | +) : NumberedTestCaseNamingStrategy(solution) { |
| 24 | + |
| 25 | + private val log: Logger = LoggerFactory.getLogger(TestSuiteWriter::class.java) |
| 26 | + private val generatedNames = mutableSetOf<String>() |
| 27 | + |
| 28 | + private val remainingNameChars = maxTestCaseNameLength - namePrefixChars() |
| 29 | + |
| 30 | + override fun expandName( |
| 31 | + individual: EvaluatedIndividual<*>, |
| 32 | + nameTokens: MutableList<String>, |
| 33 | + ambiguitySolvers: List<AmbiguitySolver> |
| 34 | + ): String { |
| 35 | + val newName = generateLlmName(TestCase(individual, "test")) |
| 36 | + return if (newName.isNotEmpty()) "_$newName" else "" |
| 37 | + } |
| 38 | + |
| 39 | + private fun generateLlmName(test: TestCase): String { |
| 40 | + var newName = sanitizeName(getNewName(test)) |
| 41 | + while (!isValidSuffix(newName)) { |
| 42 | + newName = sanitizeName(promptReIterateName()) |
| 43 | + } |
| 44 | + generatedNames.add(newName) |
| 45 | + return newName |
| 46 | + } |
| 47 | + |
| 48 | + // LLM is sometimes returning names as "\n\ntheNewName_" so we need to fix that and return "theNewName". |
| 49 | + private fun sanitizeName(testName: String): String { |
| 50 | + return TestWriterUtils.safeVariableName(testName.trim().replace("\n", ""), "") |
| 51 | + } |
| 52 | + |
| 53 | + private fun getNewName(test: TestCase): String { |
| 54 | + val testLines = getTestSourceCode(test) |
| 55 | + val targetLanguage = getTargetLanguage() |
| 56 | + val prompt = getPromptForTestCaseName(targetLanguage, remainingNameChars, generatedNames, testLines.toString()) |
| 57 | + return llmService.chat(prompt.first, prompt.second) |
| 58 | + } |
| 59 | + |
| 60 | + // Just in case the LLM did not follow the directive of just giving the new name as output. |
| 61 | + private fun promptReIterateName(): String { |
| 62 | + return llmService.chat(RE_ITERATE_TEST_CASE_NAME) |
| 63 | + } |
| 64 | + |
| 65 | + // With this regex, we check that the output by the LLM is only the test case name. We validate: |
| 66 | + // 1. Whitespace check — if the output contains a newline, it's not a bare name. |
| 67 | + // 2. Word count — split on spaces |
| 68 | + // 3. Illegal character check — a valid name contains only alphanumeric characters and underscores |
| 69 | + // 4. Length check — if the output exceeds it, it's invalid regardless of format. |
| 70 | + private fun isValidSuffix(output: String): Boolean { |
| 71 | + val stripped = output.trim() |
| 72 | + return stripped.matches(Regex("[A-Za-z0-9_]+")) && stripped.length <= remainingNameChars |
| 73 | + } |
| 74 | + |
| 75 | + private fun getTargetLanguage(): String { |
| 76 | + return when { |
| 77 | + outputFormat.isJava() -> "Java" |
| 78 | + outputFormat.isKotlin() -> "Kotlin" |
| 79 | + outputFormat.isJavaScript() -> "JavaScript" |
| 80 | + outputFormat.isPython() -> "Python" |
| 81 | + else -> throw IllegalStateException("Unrecognized language: $outputFormat") |
| 82 | + } |
| 83 | + } |
| 84 | + |
| 85 | + private fun getTestSourceCode(test: TestCase): Lines { |
| 86 | + return try { |
| 87 | + testCaseWriter.convertToCompilableTestCode(test, TestSuiteWriter.baseUrlOfSut, null) |
| 88 | + } catch (ex: Exception) { |
| 89 | + log.warn( |
| 90 | + "A failure has occurred in generating test code ${test.name} for LLM naming strategy. \n " |
| 91 | + + "Exception: ${ex.localizedMessage} \n" |
| 92 | + + "At ${ex.stackTrace.joinToString(separator = " \n -> ")}. " |
| 93 | + ) |
| 94 | + assert(false) // in our tests, this should not happen... but should not crash in production |
| 95 | + Lines(outputFormat) |
| 96 | + } |
| 97 | + } |
| 98 | +} |
0 commit comments