Skip to content

Commit 625ce66

Browse files
arcuri82arcuri82
authored
Merge pull request #1535 from WebFuzzing/ai-ensemble
Ensemble model
2 parents fc3b0aa + 8b13918 commit 625ce66

18 files changed

Lines changed: 546 additions & 224 deletions

File tree

‎core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/aiclassification/AIClassificationEMTestBase.kt‎

Lines changed: 30 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@ abstract class AIClassificationEMTestBase : SpringTestBase(){
3030
): EvaluatedIndividual<RestIndividual> {
3131

3232
val sampler = injector.getInstance(AbstractRestSampler::class.java)
33-
3433
val ind = sampler.createIndividual(sampleT, actions.toMutableList())
3534

3635
val ff = injector.getInstance(AbstractRestFitness::class.java)
@@ -43,30 +42,42 @@ abstract class AIClassificationEMTestBase : SpringTestBase(){
4342
injector: Injector,
4443
ok2xx: List<RestCallAction>,
4544
fail400: List<RestCallAction>,
46-
threshold: Double = injector.getInstance(EMConfig::class.java).classificationRepairThreshold
45+
repairThreshold: Double = injector.getInstance(EMConfig::class.java).classificationRepairThreshold,
46+
randomPerformanceThreshold: Double = 0.50
4747
) {
4848

4949
val model = injector.getInstance(AIResponseClassifier::class.java)
5050
model.disableLearning() // no side-effects
5151

52-
for(ok in ok2xx){
53-
val resOK = evaluateAction(injector, ok)
54-
assertTrue(resOK.getStatusCode() in 200..299)
55-
val mOK= model.classify(ok)
56-
assertTrue(
57-
mOK.probabilityOf400() < threshold,
58-
"Too high probability of 400 for OK ${ok.getName()}: ${mOK.probabilityOf400()}")
52+
var correctPrediction = 0
53+
// 400
54+
for (fail in fail400) {
55+
val result = evaluateAction(injector, fail)
56+
assertEquals(400, result.getStatusCode())
57+
58+
val probability = model.classify(fail).probabilityOf400()
59+
if (probability >= repairThreshold) {
60+
correctPrediction++
61+
}
5962
}
63+
// 2xx
64+
for (ok in ok2xx) {
65+
val result = evaluateAction(injector, ok)
66+
assertTrue(result.getStatusCode() in 200..299)
6067

61-
for(fail in fail400) {
62-
val resFail = evaluateAction(injector, fail)
63-
assertEquals(400, resFail.getStatusCode())
64-
val mFail = model.classify(fail)
65-
assertTrue(
66-
mFail.probabilityOf400() >= threshold,
67-
"Too low probability of 400 for Fail ${fail.getName()}: ${mFail.probabilityOf400()}"
68-
)
68+
val probability = model.classify(ok).probabilityOf400()
69+
if (probability < repairThreshold) {
70+
correctPrediction++
71+
}
6972
}
70-
}
7173

72-
}
74+
val totalSize = ok2xx.size + fail400.size
75+
val accuracy =
76+
if (totalSize > 0) correctPrediction.toDouble() / totalSize else 0.0
77+
78+
assertTrue(
79+
accuracy > randomPerformanceThreshold,
80+
"Too low total accuracy: $accuracy"
81+
)
82+
}
83+
}

‎core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/aiclassification/allornone/ACAllOrNoneEMTest.kt‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,15 +57,28 @@ class ACAllOrNoneEMTest : AIClassificationEMTestBase() {
5757
testRunEM(AIResponseClassifierModel.NN)
5858
}
5959

60-
private fun testRunEM(model: AIResponseClassifierModel) {
60+
@Test
61+
fun testRunEnsemble(){
62+
testRunEM(
63+
AIResponseClassifierModel.GAUSSIAN,
64+
AIResponseClassifierModel.GLM,
65+
AIResponseClassifierModel.KDE,
66+
AIResponseClassifierModel.KNN,
67+
AIResponseClassifierModel.NN
68+
)
69+
}
70+
71+
private fun testRunEM(vararg models: AIResponseClassifierModel) {
72+
73+
val modelString = models.joinToString(",") { it.name }
6174

6275
runTestHandlingFlakyAndCompilation(
6376
"ACAllOrNoneEM",
6477
500
6578
) { args: MutableList<String> ->
6679

6780
args.add("--aiModelForResponseClassification")
68-
args.add("$model")
81+
args.add(modelString)
6982

7083
val (injector, solution) = initAndDebug(args)
7184

‎core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/aiclassification/arithmetic/ACArithmeticEMTest.kt‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,15 +57,28 @@ class ACArithmeticEMTest : AIClassificationEMTestBase() {
5757
testRunEM(AIResponseClassifierModel.NN)
5858
}
5959

60-
private fun testRunEM(model: AIResponseClassifierModel) {
60+
@Test
61+
fun testRunEnsemble(){
62+
testRunEM(
63+
AIResponseClassifierModel.GAUSSIAN,
64+
AIResponseClassifierModel.GLM,
65+
AIResponseClassifierModel.KDE,
66+
AIResponseClassifierModel.KNN,
67+
AIResponseClassifierModel.NN
68+
)
69+
}
70+
71+
private fun testRunEM(vararg models: AIResponseClassifierModel) {
72+
73+
val modelString = models.joinToString(",") { it.name }
6174

6275
runTestHandlingFlakyAndCompilation(
6376
"ACArithmeticEM",
6477
500
6578
) { args: MutableList<String> ->
6679

6780
args.add("--aiModelForResponseClassification")
68-
args.add("$model")
81+
args.add(modelString)
6982

7083
val (injector, solution) = initAndDebug(args)
7184

‎core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/aiclassification/basic/ACBasicEMTest.kt‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ class ACBasicEMTest : AIClassificationEMTestBase() {
2121
}
2222
}
2323

24+
@Disabled
2425
@Test
2526
fun testRunDeterministic(){
2627
testRunEM(AIResponseClassifierModel.DETERMINISTIC)
@@ -56,15 +57,28 @@ class ACBasicEMTest : AIClassificationEMTestBase() {
5657
testRunEM(AIResponseClassifierModel.NN)
5758
}
5859

59-
private fun testRunEM(model: AIResponseClassifierModel) {
60+
@Test
61+
fun testRunEnsemble(){
62+
testRunEM(
63+
AIResponseClassifierModel.GAUSSIAN,
64+
AIResponseClassifierModel.GLM,
65+
AIResponseClassifierModel.KDE,
66+
AIResponseClassifierModel.KNN,
67+
AIResponseClassifierModel.NN
68+
)
69+
}
70+
71+
private fun testRunEM(vararg models: AIResponseClassifierModel) {
72+
73+
val modelString = models.joinToString(",") { it.name }
6074

6175
runTestHandlingFlakyAndCompilation(
6276
"ACBasicEM",
6377
500
6478
) { args: MutableList<String> ->
6579

6680
args.add("--aiModelForResponseClassification")
67-
args.add("$model")
81+
args.add(modelString)
6882

6983
val (injector, solution) = initAndDebug(args)
7084

‎core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/aiclassification/imply/ACImplyEMTest.kt‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,15 +65,28 @@ class ACImplyEMTest : AIClassificationEMTestBase() {
6565
testRunEM(AIResponseClassifierModel.NN)
6666
}
6767

68-
private fun testRunEM(model: AIResponseClassifierModel) {
68+
@Test
69+
fun testRunEnsemble(){
70+
testRunEM(
71+
AIResponseClassifierModel.GAUSSIAN,
72+
AIResponseClassifierModel.GLM,
73+
AIResponseClassifierModel.KDE,
74+
AIResponseClassifierModel.KNN,
75+
AIResponseClassifierModel.NN
76+
)
77+
}
78+
79+
private fun testRunEM(vararg models: AIResponseClassifierModel) {
80+
81+
val modelString = models.joinToString(",") { it.name }
6982

7083
runTestHandlingFlakyAndCompilation(
7184
"ACImplyEM",
7285
500
7386
) { args: MutableList<String> ->
7487

7588
args.add("--aiModelForResponseClassification")
76-
args.add("$model")
89+
args.add(modelString)
7790

7891
val (injector, solution) = initAndDebug(args)
7992

‎core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/aiclassification/mixed/ACMixedEMTest.kt‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,15 +66,28 @@ class ACMixedEMTest : AIClassificationEMTestBase() {
6666
testRunEM(AIResponseClassifierModel.NN)
6767
}
6868

69-
private fun testRunEM(model: AIResponseClassifierModel) {
69+
@Test
70+
fun testRunEnsemble(){
71+
testRunEM(
72+
AIResponseClassifierModel.GAUSSIAN,
73+
AIResponseClassifierModel.GLM,
74+
AIResponseClassifierModel.KDE,
75+
AIResponseClassifierModel.KNN,
76+
AIResponseClassifierModel.NN
77+
)
78+
}
79+
80+
private fun testRunEM(vararg models: AIResponseClassifierModel) {
81+
82+
val modelString = models.joinToString(",") { it.name }
7083

7184
runTestHandlingFlakyAndCompilation(
7285
"ACMixedEM",
7386
500
7487
) { args: MutableList<String> ->
7588

7689
args.add("--aiModelForResponseClassification")
77-
args.add("$model")
90+
args.add(modelString)
7891

7992
val (injector, solution) = initAndDebug(args)
8093

‎core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/aiclassification/onlyone/ACOnlyOneEMTest.kt‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,15 +63,28 @@ class ACOnlyOneEMTest : AIClassificationEMTestBase() {
6363
testRunEM(AIResponseClassifierModel.NN)
6464
}
6565

66-
private fun testRunEM(model: AIResponseClassifierModel) {
66+
@Test
67+
fun testRunEnsemble(){
68+
testRunEM(
69+
AIResponseClassifierModel.GAUSSIAN,
70+
AIResponseClassifierModel.GLM,
71+
AIResponseClassifierModel.KDE,
72+
AIResponseClassifierModel.KNN,
73+
AIResponseClassifierModel.NN
74+
)
75+
}
76+
77+
private fun testRunEM(vararg models: AIResponseClassifierModel) {
78+
79+
val modelString = models.joinToString(",") { it.name }
6780

6881
runTestHandlingFlakyAndCompilation(
6982
"ACOnlyOneEM",
7083
500
7184
) { args: MutableList<String> ->
7285

7386
args.add("--aiModelForResponseClassification")
74-
args.add("$model")
87+
args.add(modelString)
7588

7689
val (injector, solution) = initAndDebug(args)
7790

‎core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/aiclassification/or/ACOrEMTest.kt‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,15 +62,28 @@ class ACOrEMTest : AIClassificationEMTestBase() {
6262
testRunEM(AIResponseClassifierModel.NN)
6363
}
6464

65-
private fun testRunEM(model: AIResponseClassifierModel) {
65+
@Test
66+
fun testRunEnsemble(){
67+
testRunEM(
68+
AIResponseClassifierModel.GAUSSIAN,
69+
AIResponseClassifierModel.GLM,
70+
AIResponseClassifierModel.KDE,
71+
AIResponseClassifierModel.KNN,
72+
AIResponseClassifierModel.NN
73+
)
74+
}
75+
76+
private fun testRunEM(vararg models: AIResponseClassifierModel) {
77+
78+
val modelString = models.joinToString(",") { it.name }
6679

6780
runTestHandlingFlakyAndCompilation(
6881
"ACOrEM",
6982
500
7083
) { args: MutableList<String> ->
7184

7285
args.add("--aiModelForResponseClassification")
73-
args.add("$model")
86+
args.add(modelString)
7487

7588
val (injector, solution) = initAndDebug(args)
7689

‎core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/aiclassification/required/ACRequiredEMTest.kt‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,15 +61,28 @@ class ACRequiredEMTest : AIClassificationEMTestBase() {
6161
testRunEM(AIResponseClassifierModel.NN)
6262
}
6363

64-
private fun testRunEM(model: AIResponseClassifierModel) {
64+
@Test
65+
fun testRunEnsemble(){
66+
testRunEM(
67+
AIResponseClassifierModel.GAUSSIAN,
68+
AIResponseClassifierModel.GLM,
69+
AIResponseClassifierModel.KDE,
70+
AIResponseClassifierModel.KNN,
71+
AIResponseClassifierModel.NN
72+
)
73+
}
74+
75+
private fun testRunEM(vararg models: AIResponseClassifierModel) {
76+
77+
val modelString = models.joinToString(",") { it.name }
6578

6679
runTestHandlingFlakyAndCompilation(
6780
"ACRequiredEM",
6881
500
6982
) { args: MutableList<String> ->
7083

7184
args.add("--aiModelForResponseClassification")
72-
args.add("$model")
85+
args.add(modelString)
7386

7487
val (injector, solution) = initAndDebug(args)
7588

‎core-tests/e2e-tests/spring/spring-rest-openapi-v3/src/test/kotlin/org/evomaster/e2etests/spring/openapi/v3/aiclassification/zeroorone/ACZeroOrOneEMTest.kt‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -65,15 +65,28 @@ class ACZeroOrOneEMTest : AIClassificationEMTestBase() {
6565
testRunEM(AIResponseClassifierModel.NN)
6666
}
6767

68-
private fun testRunEM(model: AIResponseClassifierModel) {
68+
@Test
69+
fun testRunEnsemble(){
70+
testRunEM(
71+
AIResponseClassifierModel.GAUSSIAN,
72+
AIResponseClassifierModel.GLM,
73+
AIResponseClassifierModel.KDE,
74+
AIResponseClassifierModel.KNN,
75+
AIResponseClassifierModel.NN
76+
)
77+
}
78+
79+
private fun testRunEM(vararg models: AIResponseClassifierModel) {
80+
81+
val modelString = models.joinToString(",") { it.name }
6982

7083
runTestHandlingFlakyAndCompilation(
7184
"ACZeroOrOneEM",
7285
500
7386
) { args: MutableList<String> ->
7487

7588
args.add("--aiModelForResponseClassification")
76-
args.add("$model")
89+
args.add(modelString)
7790

7891
val (injector, solution) = initAndDebug(args)
7992

0 commit comments

Comments
 (0)