diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index d1c9d4559e..417b601905 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -19,6 +19,7 @@ import org.evomaster.core.problem.enterprise.ExperimentalFaultCategory import org.evomaster.core.search.impact.impactinfocollection.GeneMutationSelectionMethod import org.evomaster.core.search.service.IdMapper import org.slf4j.LoggerFactory +import java.lang.reflect.ParameterizedType import java.net.MalformedURLException import java.net.URL import java.nio.file.Files @@ -235,17 +236,27 @@ class EMConfig { } } - var experimentalValues = "" - var validValues = "" - val returnType = m.returnType.javaType as Class<*> - - if (returnType.isEnum) { - val elements = returnType.getDeclaredMethod("values") - .invoke(null) as Array<*> - val experimentElements = elements.filter { it is WithExperimentalOptions && it.isExperimental() } - val validElements = elements.filter { it !is WithExperimentalOptions || !it.isExperimental() } - experimentalValues = experimentElements.joinToString(", ") - validValues = validElements.joinToString(", ") + val returnType = m.returnType.javaType + + val (validValues, experimentalValues) = when (returnType) { + is Class<*> if returnType.isEnum -> { + getValidAndExperimentalEnumValues(returnType) + } + + is ParameterizedType -> { + if (!Collection::class.java.isAssignableFrom(returnType.rawType as Class<*>)) { + throw IllegalStateException("Configuration '${m.name}' is parameterized, but not a collection") + } + if (returnType.actualTypeArguments.size != 1) { + throw IllegalStateException("Configuration '${m.name}' has more than 1 generic type in a collection") + } + val generic = returnType.actualTypeArguments[0] as Class<*> + getValidAndExperimentalEnumValues(generic) + } + + else -> { + Pair("", "") + } } val experimental = (m.annotations.find { it is Experimental } as? Experimental) @@ -266,6 +277,17 @@ class EMConfig { ) } + private fun getValidAndExperimentalEnumValues(enumClass: Class<*>) : Pair{ + assert(enumClass.isEnum) + val elements = enumClass.getDeclaredMethod("values") + .invoke(null) as Array<*> + val experimentElements = elements.filter { it is WithExperimentalOptions && it.isExperimental() } + val validElements = elements.filter { it !is WithExperimentalOptions || !it.isExperimental() } + + val experimentalValues = experimentElements.joinToString(", ") + val validValues = validElements.joinToString(", ") + return Pair(validValues,experimentalValues) + } fun getConfigurationProperties(): List> { return EMConfig::class.members @@ -452,39 +474,71 @@ class EMConfig { private fun updateValue(optionValue: String, m: KMutableProperty<*>) { - val returnType = m.returnType.javaType as Class<*> + val returnType = m.returnType.javaType - /* + if(returnType is Class<*>) { + /* TODO: ugly checks. But not sure yet if can be made better in Kotlin. Could be improved with isSubtypeOf from 1.1? http://stackoverflow.com/questions/41553647/kotlin-isassignablefrom-and-reflection-type-checks */ - try { - if (Integer.TYPE.isAssignableFrom(returnType)) { - m.setter.call(this, Integer.parseInt(optionValue)) + try { + if (Integer.TYPE.isAssignableFrom(returnType)) { + m.setter.call(this, Integer.parseInt(optionValue)) - } else if (java.lang.Long.TYPE.isAssignableFrom(returnType)) { - m.setter.call(this, java.lang.Long.parseLong(optionValue)) + } else if (java.lang.Long.TYPE.isAssignableFrom(returnType)) { + m.setter.call(this, java.lang.Long.parseLong(optionValue)) - } else if (java.lang.Double.TYPE.isAssignableFrom(returnType)) { - m.setter.call(this, java.lang.Double.parseDouble(optionValue)) + } else if (java.lang.Double.TYPE.isAssignableFrom(returnType)) { + m.setter.call(this, java.lang.Double.parseDouble(optionValue)) - } else if (java.lang.Boolean.TYPE.isAssignableFrom(returnType)) { - m.setter.call(this, parseBooleanStrict(optionValue)) + } else if (java.lang.Boolean.TYPE.isAssignableFrom(returnType)) { + m.setter.call(this, parseBooleanStrict(optionValue)) - } else if (java.lang.String::class.java.isAssignableFrom(returnType)) { - m.setter.call(this, optionValue) + } else if (java.lang.String::class.java.isAssignableFrom(returnType)) { + m.setter.call(this, optionValue) - } else if (returnType.isEnum) { - val valueOfMethod = returnType.getDeclaredMethod("valueOf", - java.lang.String::class.java) - m.setter.call(this, valueOfMethod.invoke(null, optionValue)) + } else if (returnType.isEnum) { + val valueOfMethod = returnType.getDeclaredMethod("valueOf", java.lang.String::class.java) + m.setter.call(this, valueOfMethod.invoke(null, optionValue)) - } else { - throw IllegalStateException("BUG: cannot handle type $returnType") + } else { + throw IllegalStateException("BUG: cannot handle type $returnType") + } + } catch (e: Exception) { + throw ConfigProblemException("Failed to handle property '${m.name}': ${e.message}") } - } catch (e: Exception) { - throw ConfigProblemException("Failed to handle property '${m.name}': ${e.message}") + } else if(returnType is ParameterizedType) { + + if (!Collection::class.java.isAssignableFrom(returnType.rawType as Class<*>)) { + throw IllegalStateException("Configuration '${m.name}' is parameterized, but not a collection") + } + if (returnType.actualTypeArguments.size != 1) { + throw IllegalStateException("Configuration '${m.name}' has more than 1 generic type in a collection") + } + val generic = returnType.actualTypeArguments[0] as Class<*> + if(!generic.isEnum){ + throw IllegalStateException("Content for configuration '${m.name}' is not an enumeration: ${generic.name}") + } + + val valueOfMethod = generic.getDeclaredMethod("valueOf", java.lang.String::class.java) + + val collection = optionValue.split(",") + .map { + try { + valueOfMethod.invoke(null, it) + }catch (e: Exception){ + throw ConfigProblemException("Failed to handle property '${m.name}': ${e.message}") + } + } + .run { + if(Set::class.java.isAssignableFrom(returnType.rawType as Class<*>)){ + toSet() + } else { + this + } + } + m.setter.call(this, collection) } } @@ -964,6 +1018,7 @@ class EMConfig { val properties = getConfigurationProperties() .filter { it.annotations.find { it is Experimental } != null } + .filter{ it.returnType.javaType is Class<*>} // TODO handle Lists of Enum .filter { val returnType = it.returnType.javaType as Class<*> when { @@ -975,6 +1030,7 @@ class EMConfig { .map { it.name } val enums = getConfigurationProperties() + .filter{ it.returnType.javaType is Class<*>} // TODO handle Lists of Enum .filter { val returnType = it.returnType.javaType as Class<*> if (returnType.isEnum) { @@ -1556,7 +1612,7 @@ class EMConfig { @Cfg("Models used to learn input constraints and predict the response status before issuing a request. " + "Supports both single-model and ensemble configurations. " + "Ensemble model is a combination of a comma-separated list, e.g., GLM,NN,KDE.") - var aiModelForResponseClassification: String = "NONE" + var aiModelForResponseClassification: Set = setOf(AIResponseClassifierModel.NONE) @Experimental @Cfg("Learning rate controlling the step size during parameter updates in classifiers. " + @@ -3387,8 +3443,7 @@ class EMConfig { // Sets the AI response classification models programmatically. fun setAIModels(vararg models: AIResponseClassifierModel) { - aiModelForResponseClassification = - models.joinToString(",") { it.name } + aiModelForResponseClassification = models.toSet() } /** @@ -3399,17 +3454,7 @@ class EMConfig { */ fun getAIModelForResponseClassification(): List { val models = aiModelForResponseClassification - .split(",") - .map { it.trim() } - .filter { it.isNotEmpty() } - .map { - try { - AIResponseClassifierModel.valueOf(it) - } catch (e: Exception) { - throw ConfigProblemException("Invalid AI model: $it") - } - } - .distinct() + .toList() .sorted() // EvoMaster accept NONE or a combination of the AI models and not both diff --git a/core/src/main/kotlin/org/evomaster/core/docs/ConfigToMarkdown.kt b/core/src/main/kotlin/org/evomaster/core/docs/ConfigToMarkdown.kt index 802f94ddd1..e196b26a10 100644 --- a/core/src/main/kotlin/org/evomaster/core/docs/ConfigToMarkdown.kt +++ b/core/src/main/kotlin/org/evomaster/core/docs/ConfigToMarkdown.kt @@ -3,6 +3,7 @@ package org.evomaster.core.docs import org.evomaster.core.EMConfig import org.evomaster.core.utils.StringUtils import java.io.File +import java.lang.reflect.ParameterizedType import java.nio.charset.Charset import kotlin.reflect.KMutableProperty import kotlin.reflect.jvm.javaType @@ -151,11 +152,15 @@ object ConfigToMarkdown { if(default.isBlank()){ default = "\"\"" } - val type = (opt.returnType.javaType as Class<*>) - val typeName = if(type.isEnum){ - "Enum" + val type = opt.returnType.javaType + val typeName = if(type is Class<*>) { + if (type.isEnum) { + "Enum" + } else { + StringUtils.capitalization(type.simpleName) + } } else { - StringUtils.capitalization(type.simpleName) + StringUtils.capitalization(((type as ParameterizedType).rawType as Class<*>).simpleName) } val description = EMConfig.getDescription(opt) diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt b/core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt index 3b39d8cfbb..94da2c063a 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt @@ -494,7 +494,7 @@ class Statistics : SearchListener { return aiMetricsAsPairs( enabled = true, - type = config.aiModelForResponseClassification, + type = config.aiModelForResponseClassification.joinToString(","), accuracy = metrics.accuracy, precision = metrics.precision400, sensitivity = metrics.sensitivity400, diff --git a/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt b/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt index 4fd119e817..0e021a8fd6 100644 --- a/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/EMConfigTest.kt @@ -1,6 +1,7 @@ package org.evomaster.core import org.evomaster.client.java.controller.api.ControllerConstants +import org.evomaster.core.EMConfig.AIResponseClassifierModel import org.evomaster.core.config.ConfigProblemException import org.evomaster.core.output.OutputFormat import org.evomaster.core.output.naming.NamingStrategy @@ -21,6 +22,27 @@ internal class EMConfigTest{ private val blackBox = "blackBox" private val generateMongoData = "generateMongoData" + + @Test + fun testChangeSetEnum(){ + + val parser = EMConfig.getOptionParser() + + val opt = parser.recognizedOptions()["aiModelForResponseClassification"] ?: + throw Exception("Cannot find option") + + val x = "${AIResponseClassifierModel.DETERMINISTIC},${AIResponseClassifierModel.GAUSSIAN}" + val options = parser.parse("--aiModelForResponseClassification", x) + + assertEquals(x, opt.value(options)) + + val config = EMConfig() + assertTrue("" + config.aiModelForResponseClassification.map { it.name }.sorted().joinToString(",") != x) + + config.updateProperties(options) + assertEquals(x, "" + config.aiModelForResponseClassification.map { it.name }.sorted().joinToString(",")) + } + @Test fun testDependsOnTrue(){ diff --git a/docs/options.md b/docs/options.md index 1eea634712..f09b63f1e4 100644 --- a/docs/options.md +++ b/docs/options.md @@ -260,7 +260,7 @@ There are 3 types of options: |`abstractInitializationGeneToMutate`| __Boolean__. During mutation, whether to abstract genes for repeated SQL actions. *Default value*: `false`.| |`aiClassifierRepairActivation`| __Enum__. Specify how the classification of actions's response will be used to execute a possible repair on the action. *Valid values*: `PROBABILITY, THRESHOLD`. *Default value*: `THRESHOLD`.| |`aiEncoderType`| __Enum__. The encoding strategy applied to transform raw data to the encoded version. *Valid values*: `RAW, NORMAL, UNIT_NORMAL`. *Default value*: `NORMAL`.| -|`aiModelForResponseClassification`| __String__. Models used to learn input constraints and predict the response status before issuing a request. Supports both single-model and ensemble configurations. Ensemble model is a combination of a comma-separated list, e.g., GLM,NN,KDE. *Default value*: `NONE`.| +|`aiModelForResponseClassification`| __Set__. Models used to learn input constraints and predict the response status before issuing a request. Supports both single-model and ensemble configurations. Ensemble model is a combination of a comma-separated list, e.g., GLM,NN,KDE. *Valid values*: `NONE, GAUSSIAN, KDE, KNN, NN, GLM, DETERMINISTIC`. *Default value*: `[NONE]`.| |`aiResponseClassifierLearningRate`| __Double__. Learning rate controlling the step size during parameter updates in classifiers. Relevant for gradient-based models such as GLM and neural networks. A smaller value ensures stable but slower convergence, while a larger value speeds up training but may cause instability. *Default value*: `0.01`.| |`aiResponseClassifierMaxStoredSamples`| __Int__. Maximum number of stored samples for classifiers such as KNN and KDE models that rely on retaining encoded inputs. This value specifies the maximum number of samples stored for each endpoint. A higher value can improve classification accuracy by leveraging more historical data, but also increases memory usage. A lower value reduces memory consumption but may limit the classifier’s knowledge base. Typically, it is safe to keep this value between 10,000 and 50,000 when the encoded input vector is usually a list of doubles with a length under 20. Reservoir sampling is applied independently for each endpoint: if this maximum number is exceeded, new samples randomly replace existing ones, ensuring an unbiased selection of preserved data. As an example, for an API with 100 endpoints and an input vector of size 20, a maximum of 10,000 samples per endpoint would require roughly 200 MB of memory. *Default value*: `10000`.| |`aiResponseClassifierWarmup`| __Int__. Number of training iterations required to update classifier parameters. For example, in the Gaussian model this affects mean and variance updates. For neural network (NN) models, the warm-up should typically be larger than 1000. *Default value*: `100`.|