Merge pull request #1571 from WebFuzzing/llm-dictionary-creation

building dictionary

Author: arcuri82

core/src/main/kotlin/org/evomaster/core/llm/DictionaryCreator.kt

@@ -12,6 +12,7 @@ import org.evomaster.core.utils.TimeUtils
 import java.io.File
 import java.util.Collections
 import java.util.concurrent.Executors
+import java.util.concurrent.TimeUnit
 import kotlin.io.path.Path
 
 
@@ -20,9 +21,21 @@ object DictionaryCreator {
     @JvmStatic
     fun main(args: Array<String>) {
 
+        /*
+            June 2026
+            111M tokens
+            deepseek-v4-flash
+            cost: ~$19
+            size: 25.3Mb
+         */
+
+        // TODO need to add manually
+        // WARNING: make sure you do NOT commit it
         val API_KEY = ""
 
         val file = File("src/main/resources/llm_dictionary.jsonl")
+        val errors = 0
+        val alreadyHandled = errors + file.bufferedReader().use { it.lineSequence().count() }
 
 //        val modelName = "deepseek-v4-pro"
         val modelName = "deepseek-v4-flash"
@@ -33,17 +46,24 @@ object DictionaryCreator {
         TimeUtils.measureTimeMillis(
             {ms, res -> println("Took ${ms/1000}s")},
             {
-                val executor = Executors.newFixedThreadPool(10)
+                val parallelism = 20
+                val executor = Executors.newFixedThreadPool(parallelism)
                 val list = Collections.synchronizedList(mutableListOf<String>())
 
                 val futures = data.entries
                     .sortedBy { it.key }
-                    //.drop(1) //TODO for debugging
-                    .take(10) //TODO for debugging
+                    .drop(alreadyHandled)
+                    .take(2000) //TODO use for incremental job
                     .map { entry ->
                         executor.submit { handleEntry(entry, model, list) }
                     }
-                futures.forEach { it.get() }
+                futures.forEach {
+                    try {
+                        it.get(120, TimeUnit.SECONDS)
+                    }catch (e: Exception) {
+                        println("ERROR. Failed to wait for job: ${e.message}")
+                    }
+                }
                 executor.shutdown()
 
                 list.sorted().forEach {
@@ -62,18 +82,22 @@ object DictionaryCreator {
         val description = entry.value.maxByOrNull { it.length }
         val prompt = Prompts.getPromptForNameDescription(name, description)
         var result = LlmSupport.chat(model, prompt.first, prompt.second)
-        val list = try {
+        try {
             mapper.readValue(result, object : TypeReference<List<String>>() {})
         } catch (e: Exception) {
-            val failed = Prompts.getPromptForFailedName(e.toString())
-            result = LlmSupport.chat(model, failed.first, failed.second)
-            mapper.readValue(result, object : TypeReference<List<String>>() {})
-        } catch (e: Exception) {
-            print("Failed handling response: $result")
-            throw e
+            print("ERROR. Failed handling response: $result")
+            try {
+                val failed = Prompts.getPromptForFailedName(e.toString())
+                result = LlmSupport.chat(model, failed.first, failed.second)
+                mapper.readValue(result, object : TypeReference<List<String>>() {})
+            } catch (e: Exception) {
+                print("ERROR. Failed again handling response: $result")
+                throw e
+            }
         }
 
-        val row = "{ \"$name\": $result }"
+        //must be a single row, and we have it in the prompt, but sometimes it is ignored
+        val row = "{ \"$name\": $result }".replace('\n',' ')
         println(row)
         buffer.add(row)
     }
@@ -83,8 +107,7 @@ object DictionaryCreator {
         val data = mutableMapOf<String, MutableSet<String>>()
 
         val locations = scanForSchemas("./src/test/resources/swagger")
-            //TODO put back once rest finished
-            //.plus(scanForSchemas("../core-tests/integration-tests/core-it/src/test/resources/APIs_guru"))
+            .plus(scanForSchemas("../core-tests/integration-tests/core-it/src/test/resources/APIs_guru"))
 
         for(l in locations){
             println("Analyzing: $l")

core/src/main/kotlin/org/evomaster/core/llm/service/DictionaryService.kt

@@ -56,6 +56,7 @@ class DictionaryService {
 
             val toFind = names.toList().sorted()
             var index = 0
+            var previous = ""
 
             reader.lineSequence()
                 .takeWhile { index < toFind.size }
@@ -66,6 +67,11 @@ class DictionaryService {
                     val endQuote = line.indexOf('"', startQuote+1)
                     val x = line.substring(startQuote+1, endQuote)
 
+                    if(x != "" && x <= previous) {
+                        throw IllegalStateException("Not sorted: $x should be before $previous")
+                    }
+                    previous = x
+
                     var handled = false
 
                     while(!handled && index < toFind.size) {
@@ -104,12 +110,15 @@ class DictionaryService {
         ) {
             val node = mapper.readTree(line)
             if (!node.isObject) {
-                log.warn("Not an object: $line")
+                throw IllegalStateException("Not an object: $line")
             } else {
                 node.fields().forEach { field ->
                     if (!field.value.isArray) {
-                        log.warn("Not containing an array: $line")
+                        throw IllegalStateException("Not containing an array: $line")
                     } else {
+                        if(found.containsKey(field.key)) {
+                            throw IllegalStateException("Already handled: ${field.key}")
+                        }
                         found[field.key] = mapper.convertValue(field.value, object : TypeReference<Set<String>>() {})
                     }
                 }