Skip to content

Ensemble model #1535

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
arcuri82 merged 8 commits into from
May 19, 2026
Merged

Ensemble model #1535

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
6a52c65
Initial updates
MohsenTaheriShalmani May 4, 2026
866db06
Update
MohsenTaheriShalmani May 5, 2026
eca6bf7
debugging
MohsenTaheriShalmani May 6, 2026
816a8f7
Update AIResponseClassifier.kt
MohsenTaheriShalmani May 6, 2026
1e8eb9d
Update AIResponseClassifier.kt
MohsenTaheriShalmani May 7, 2026
34480af
Update
MohsenTaheriShalmani May 11, 2026
7e4b4b9
Address the issues
MohsenTaheriShalmani May 12, 2026
8b13918
Update
MohsenTaheriShalmani May 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 30 additions & 19 deletions ...n/org/evomaster/e2etests/spring/openapi/v3/aiclassification/AIClassificationEMTestBase.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ abstract class AIClassificationEMTestBase : SpringTestBase(){
): EvaluatedIndividual<RestIndividual> {

val sampler = injector.getInstance(AbstractRestSampler::class.java)

val ind = sampler.createIndividual(sampleT, actions.toMutableList())

val ff = injector.getInstance(AbstractRestFitness::class.java)
Expand All @@ -43,30 +42,42 @@ abstract class AIClassificationEMTestBase : SpringTestBase(){
injector: Injector,
ok2xx: List<RestCallAction>,
fail400: List<RestCallAction>,
threshold: Double = injector.getInstance(EMConfig::class.java).classificationRepairThreshold
repairThreshold: Double = injector.getInstance(EMConfig::class.java).classificationRepairThreshold,
randomPerformanceThreshold: Double = 0.50
) {

val model = injector.getInstance(AIResponseClassifier::class.java)
model.disableLearning() // no side-effects

for(ok in ok2xx){
val resOK = evaluateAction(injector, ok)
assertTrue(resOK.getStatusCode() in 200..299)
val mOK= model.classify(ok)
assertTrue(
mOK.probabilityOf400() < threshold,
"Too high probability of 400 for OK ${ok.getName()}: ${mOK.probabilityOf400()}")
var correctPrediction = 0
// 400
for (fail in fail400) {
val result = evaluateAction(injector, fail)
assertEquals(400, result.getStatusCode())

val probability = model.classify(fail).probabilityOf400()
if (probability >= repairThreshold) {
correctPrediction++
}
}
// 2xx
for (ok in ok2xx) {
val result = evaluateAction(injector, ok)
assertTrue(result.getStatusCode() in 200..299)

for(fail in fail400) {
val resFail = evaluateAction(injector, fail)
assertEquals(400, resFail.getStatusCode())
val mFail = model.classify(fail)
assertTrue(
mFail.probabilityOf400() >= threshold,
"Too low probability of 400 for Fail ${fail.getName()}: ${mFail.probabilityOf400()}"
)
val probability = model.classify(ok).probabilityOf400()
if (probability < repairThreshold) {
correctPrediction++
}
}
}

}
val totalSize = ok2xx.size + fail400.size
val accuracy =
if (totalSize > 0) correctPrediction.toDouble() / totalSize else 0.0

assertTrue(
accuracy > randomPerformanceThreshold,
"Too low total accuracy: $accuracy"
)
}
}
17 changes: 15 additions & 2 deletions .../org/evomaster/e2etests/spring/openapi/v3/aiclassification/allornone/ACAllOrNoneEMTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,15 +57,28 @@ class ACAllOrNoneEMTest : AIClassificationEMTestBase() {
testRunEM(AIResponseClassifierModel.NN)
}

private fun testRunEM(model: AIResponseClassifierModel) {
@Test
fun testRunEnsemble(){
testRunEM(
AIResponseClassifierModel.GAUSSIAN,
AIResponseClassifierModel.GLM,
AIResponseClassifierModel.KDE,
AIResponseClassifierModel.KNN,
AIResponseClassifierModel.NN
)
}

private fun testRunEM(vararg models: AIResponseClassifierModel) {

val modelString = models.joinToString(",") { it.name }

runTestHandlingFlakyAndCompilation(
"ACAllOrNoneEM",
500
) { args: MutableList<String> ->

args.add("--aiModelForResponseClassification")
args.add("$model")
args.add(modelString)

val (injector, solution) = initAndDebug(args)

Expand Down
17 changes: 15 additions & 2 deletions ...rg/evomaster/e2etests/spring/openapi/v3/aiclassification/arithmetic/ACArithmeticEMTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,15 +57,28 @@ class ACArithmeticEMTest : AIClassificationEMTestBase() {
testRunEM(AIResponseClassifierModel.NN)
}

private fun testRunEM(model: AIResponseClassifierModel) {
@Test
fun testRunEnsemble(){
testRunEM(
AIResponseClassifierModel.GAUSSIAN,
AIResponseClassifierModel.GLM,
AIResponseClassifierModel.KDE,
AIResponseClassifierModel.KNN,
AIResponseClassifierModel.NN
)
}

private fun testRunEM(vararg models: AIResponseClassifierModel) {

val modelString = models.joinToString(",") { it.name }

runTestHandlingFlakyAndCompilation(
"ACArithmeticEM",
500
) { args: MutableList<String> ->

args.add("--aiModelForResponseClassification")
args.add("$model")
args.add(modelString)

val (injector, solution) = initAndDebug(args)

Expand Down
18 changes: 16 additions & 2 deletions ...t/kotlin/org/evomaster/e2etests/spring/openapi/v3/aiclassification/basic/ACBasicEMTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ class ACBasicEMTest : AIClassificationEMTestBase() {
}
}

@Disabled
@Test
fun testRunDeterministic(){
testRunEM(AIResponseClassifierModel.DETERMINISTIC)
Expand Down Expand Up @@ -56,15 +57,28 @@ class ACBasicEMTest : AIClassificationEMTestBase() {
testRunEM(AIResponseClassifierModel.NN)
}

private fun testRunEM(model: AIResponseClassifierModel) {
@Test
fun testRunEnsemble(){
testRunEM(
AIResponseClassifierModel.GAUSSIAN,
AIResponseClassifierModel.GLM,
AIResponseClassifierModel.KDE,
AIResponseClassifierModel.KNN,
AIResponseClassifierModel.NN
)
}

private fun testRunEM(vararg models: AIResponseClassifierModel) {

val modelString = models.joinToString(",") { it.name }

runTestHandlingFlakyAndCompilation(
"ACBasicEM",
500
) { args: MutableList<String> ->

args.add("--aiModelForResponseClassification")
args.add("$model")
args.add(modelString)

val (injector, solution) = initAndDebug(args)

Expand Down
17 changes: 15 additions & 2 deletions ...t/kotlin/org/evomaster/e2etests/spring/openapi/v3/aiclassification/imply/ACImplyEMTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,15 +65,28 @@ class ACImplyEMTest : AIClassificationEMTestBase() {
testRunEM(AIResponseClassifierModel.NN)
}

private fun testRunEM(model: AIResponseClassifierModel) {
@Test
fun testRunEnsemble(){
testRunEM(
AIResponseClassifierModel.GAUSSIAN,
AIResponseClassifierModel.GLM,
AIResponseClassifierModel.KDE,
AIResponseClassifierModel.KNN,
AIResponseClassifierModel.NN
)
}

private fun testRunEM(vararg models: AIResponseClassifierModel) {

val modelString = models.joinToString(",") { it.name }

runTestHandlingFlakyAndCompilation(
"ACImplyEM",
500
) { args: MutableList<String> ->

args.add("--aiModelForResponseClassification")
args.add("$model")
args.add(modelString)

val (injector, solution) = initAndDebug(args)

Expand Down
17 changes: 15 additions & 2 deletions ...t/kotlin/org/evomaster/e2etests/spring/openapi/v3/aiclassification/mixed/ACMixedEMTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,15 +66,28 @@ class ACMixedEMTest : AIClassificationEMTestBase() {
testRunEM(AIResponseClassifierModel.NN)
}

private fun testRunEM(model: AIResponseClassifierModel) {
@Test
fun testRunEnsemble(){
testRunEM(
AIResponseClassifierModel.GAUSSIAN,
AIResponseClassifierModel.GLM,
AIResponseClassifierModel.KDE,
AIResponseClassifierModel.KNN,
AIResponseClassifierModel.NN
)
}

private fun testRunEM(vararg models: AIResponseClassifierModel) {

val modelString = models.joinToString(",") { it.name }

runTestHandlingFlakyAndCompilation(
"ACMixedEM",
500
) { args: MutableList<String> ->

args.add("--aiModelForResponseClassification")
args.add("$model")
args.add(modelString)

val (injector, solution) = initAndDebug(args)

Expand Down
17 changes: 15 additions & 2 deletions ...tlin/org/evomaster/e2etests/spring/openapi/v3/aiclassification/onlyone/ACOnlyOneEMTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,15 +63,28 @@ class ACOnlyOneEMTest : AIClassificationEMTestBase() {
testRunEM(AIResponseClassifierModel.NN)
}

private fun testRunEM(model: AIResponseClassifierModel) {
@Test
fun testRunEnsemble(){
testRunEM(
AIResponseClassifierModel.GAUSSIAN,
AIResponseClassifierModel.GLM,
AIResponseClassifierModel.KDE,
AIResponseClassifierModel.KNN,
AIResponseClassifierModel.NN
)
}

private fun testRunEM(vararg models: AIResponseClassifierModel) {

val modelString = models.joinToString(",") { it.name }

runTestHandlingFlakyAndCompilation(
"ACOnlyOneEM",
500
) { args: MutableList<String> ->

args.add("--aiModelForResponseClassification")
args.add("$model")
args.add(modelString)

val (injector, solution) = initAndDebug(args)

Expand Down
17 changes: 15 additions & 2 deletions ...rc/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/aiclassification/or/ACOrEMTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,15 +62,28 @@ class ACOrEMTest : AIClassificationEMTestBase() {
testRunEM(AIResponseClassifierModel.NN)
}

private fun testRunEM(model: AIResponseClassifierModel) {
@Test
fun testRunEnsemble(){
testRunEM(
AIResponseClassifierModel.GAUSSIAN,
AIResponseClassifierModel.GLM,
AIResponseClassifierModel.KDE,
AIResponseClassifierModel.KNN,
AIResponseClassifierModel.NN
)
}

private fun testRunEM(vararg models: AIResponseClassifierModel) {

val modelString = models.joinToString(",") { it.name }

runTestHandlingFlakyAndCompilation(
"ACOrEM",
500
) { args: MutableList<String> ->

args.add("--aiModelForResponseClassification")
args.add("$model")
args.add(modelString)

val (injector, solution) = initAndDebug(args)

Expand Down
17 changes: 15 additions & 2 deletions ...in/org/evomaster/e2etests/spring/openapi/v3/aiclassification/required/ACRequiredEMTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,15 +61,28 @@ class ACRequiredEMTest : AIClassificationEMTestBase() {
testRunEM(AIResponseClassifierModel.NN)
}

private fun testRunEM(model: AIResponseClassifierModel) {
@Test
fun testRunEnsemble(){
testRunEM(
AIResponseClassifierModel.GAUSSIAN,
AIResponseClassifierModel.GLM,
AIResponseClassifierModel.KDE,
AIResponseClassifierModel.KNN,
AIResponseClassifierModel.NN
)
}

private fun testRunEM(vararg models: AIResponseClassifierModel) {

val modelString = models.joinToString(",") { it.name }

runTestHandlingFlakyAndCompilation(
"ACRequiredEM",
500
) { args: MutableList<String> ->

args.add("--aiModelForResponseClassification")
args.add("$model")
args.add(modelString)

val (injector, solution) = initAndDebug(args)

Expand Down
17 changes: 15 additions & 2 deletions .../org/evomaster/e2etests/spring/openapi/v3/aiclassification/zeroorone/ACZeroOrOneEMTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,15 +65,28 @@ class ACZeroOrOneEMTest : AIClassificationEMTestBase() {
testRunEM(AIResponseClassifierModel.NN)
}

private fun testRunEM(model: AIResponseClassifierModel) {
@Test
fun testRunEnsemble(){
testRunEM(
AIResponseClassifierModel.GAUSSIAN,
AIResponseClassifierModel.GLM,
AIResponseClassifierModel.KDE,
AIResponseClassifierModel.KNN,
AIResponseClassifierModel.NN
)
}

private fun testRunEM(vararg models: AIResponseClassifierModel) {

val modelString = models.joinToString(",") { it.name }

runTestHandlingFlakyAndCompilation(
"ACZeroOrOneEM",
500
) { args: MutableList<String> ->

args.add("--aiModelForResponseClassification")
args.add("$model")
args.add(modelString)

val (injector, solution) = initAndDebug(args)

Expand Down
Loading
Loading