Skip to content

Commit d310ec3

Browse files
author
Varun Iyengar
committed
Only run mutation analysis if needed (specify through -s flag), and trim mutants to better account for memory
1 parent 52ccd07 commit d310ec3

63 files changed

Lines changed: 455 additions & 177 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎scripts/mutation-evosuite.sh‎

Lines changed: 35 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -23,14 +23,15 @@
2323
#------------------------------------------------------------------------------
2424
# Options (command-line arguments):
2525
#------------------------------------------------------------------------------
26-
USAGE_STRING="usage: mutation-evosuite.sh [-o RESULTS_CSV] [-t total_time] [-c time_per_class] [-n num_iterations] [-r] [-v] [-h] TEST-CASE-NAME
26+
USAGE_STRING="usage: mutation-evosuite.sh [-o RESULTS_CSV] [-t total_time] [-c time_per_class] [-n num_iterations] [-s] [-r] [-v] [-h] TEST-CASE-NAME
2727
-o N Write experiment results to this CSV file (N should end in '.csv').
2828
If the file does not exist, a header row will be created automatically.
2929
Paths are not allowed; only a filename may be given.
3030
-t N Total time limit for test generation (in seconds).
3131
-c N Per-class time limit (in seconds, default: 2s/class).
3232
Mutually exclusive with -t.
3333
-n N Number of iterations to run the experiment (default: 1).
34+
-s Skip mutation analysis (only run test generation and coverage).
3435
-r Redirect logs and diagnostics to results/result/mutation_output.txt.
3536
-v Enables verbose mode.
3637
-h Displays this help message.
@@ -83,10 +84,11 @@ fi
8384
NUM_LOOP=1 # Number of experiment runs (10 in GRT paper)
8485
VERBOSE=0 # Verbose option
8586
REDIRECT=0 # Redirect output to mutation_output.txt
87+
SKIP_MUTATION=0 # Skip mutation analysis
8688
UUID=$(uuidgen) # Generate a unique identifier per instance
8789

8890
# Parse command-line arguments
89-
while getopts ":hvro:t:c:n:" opt; do
91+
while getopts ":hvrso:t:c:n:" opt; do
9092
case ${opt} in
9193
h)
9294
# Display help message
@@ -101,6 +103,10 @@ while getopts ":hvro:t:c:n:" opt; do
101103
# Redirect output to a log file
102104
REDIRECT=1
103105
;;
106+
s)
107+
# Skip mutation analysis
108+
SKIP_MUTATION=1
109+
;;
104110
o)
105111
RESULTS_CSV="$OPTARG"
106112
;;
@@ -490,6 +496,13 @@ for i in $(seq 1 "$NUM_LOOP"); do
490496
"$MAJOR_HOME"/bin/ant -f "$SCRIPT_DIR"/program-config/"$1"/${buildfile} -Dbasedir="$SCRIPT_DIR" -Dbindir="$SCRIPT_DIR/build/bin/$FILE_SUFFIX" -Dresultdir="$RESULT_DIR" -Dmutator="mml:$MAJOR_HOME/mml/all.mml.bin" -Dsrc="$JAVA_SRC_DIR" -Dtargetdir="$COVERAGE_DIRECTORY" -Dlibdir="$SCRIPT_DIR/build/lib/$UUID" compile.mutation
491497
"$MAJOR_HOME"/bin/ant -f "$SCRIPT_DIR"/program-config/"$1"/${buildfile} -Dbasedir="$SCRIPT_DIR" -Dbindir="$SCRIPT_DIR/build/bin/$FILE_SUFFIX" -Dresultdir="$RESULT_DIR" -Dmutator="mml:$MAJOR_HOME/mml/all.mml.bin" -Dsrc="$JAVA_SRC_DIR" -Dtargetdir="$COVERAGE_DIRECTORY" -Dlibdir="$SCRIPT_DIR/build/lib/$UUID" compile.jacoco
492498

499+
PYTHON_EXECUTABLE=$(command -v python3 2> /dev/null || command -v python 2> /dev/null)
500+
if [ -z "$PYTHON_EXECUTABLE" ]; then
501+
echo "Error: Python is not installed." >&2
502+
exit 2
503+
fi
504+
"$PYTHON_EXECUTABLE" "$SCRIPT_DIR"/trim_mutants.py "$RESULT_DIR/mutants.log"
505+
493506
echo
494507
echo "Compiling tests..."
495508
if [[ "$VERBOSE" -eq 1 ]]; then
@@ -530,28 +543,30 @@ for i in $(seq 1 "$NUM_LOOP"); do
530543
# resulting in a lot of methods being excluded from mutation analysis.
531544
# We use a Python script to convert the tests.
532545
if [ "$SUBJECT_PROGRAM" == "jdom-1.0" ]; then
533-
PYTHON_EXECUTABLE=$(command -v python3 2> /dev/null || command -v python 2> /dev/null)
534-
if [ -z "$PYTHON_EXECUTABLE" ]; then
535-
echo "Error: Python is not installed." >&2
536-
exit 2
537-
fi
538546
"$PYTHON_EXECUTABLE" "$SCRIPT_DIR"/convert_test_runners.py "$TEST_DIRECTORY" --mode evosuite-to-randoop
539547
fi
540548

541-
echo
542-
echo "Running tests with mutation analysis..."
543-
if [[ "$VERBOSE" -eq 1 ]]; then
544-
echo command:
545-
echo "$MAJOR_HOME"/bin/"$ANT" -f "$SCRIPT_DIR"/program-config/"$1"/${buildfile} -Dbasedir="$SCRIPT_DIR" -Dbindir="$SCRIPT_DIR/build/bin/$FILE_SUFFIX" -Dresultdir="$RESULT_DIR" -Dtest="$TEST_DIRECTORY" -Dlibdir="$SCRIPT_DIR/build/lib/$UUID" mutation.test
549+
# Run mutation analysis unless -s flag is set
550+
if [[ "$SKIP_MUTATION" -eq 0 ]]; then
551+
echo
552+
echo "Running tests with mutation analysis..."
553+
if [[ "$VERBOSE" -eq 1 ]]; then
554+
echo command:
555+
echo "$MAJOR_HOME"/bin/"$ANT" -f "$SCRIPT_DIR"/program-config/"$1"/${buildfile} -Dbasedir="$SCRIPT_DIR" -Dbindir="$SCRIPT_DIR/build/bin/$FILE_SUFFIX" -Dresultdir="$RESULT_DIR" -Dtest="$TEST_DIRECTORY" -Dlibdir="$SCRIPT_DIR/build/lib/$UUID" mutation.test
556+
fi
557+
echo
558+
"$MAJOR_HOME"/bin/"$ANT" -f "$SCRIPT_DIR"/program-config/"$1"/${buildfile} -Dbasedir="$SCRIPT_DIR" -Dbindir="$SCRIPT_DIR/build/bin/$FILE_SUFFIX" -Dresultdir="$RESULT_DIR" -Dtest="$TEST_DIRECTORY" -Dlibdir="$SCRIPT_DIR/build/lib/$UUID" mutation.test
559+
560+
# Calculate Mutation Score
561+
mutants_generated=$(awk -F, 'NR==2 {print $2}' "$RESULT_DIR"/summary.csv)
562+
mutants_killed=$(awk -F, 'NR==2 {print $4}' "$RESULT_DIR"/summary.csv)
563+
mutation_score=$(echo "scale=4; $mutants_killed / $mutants_generated * 100" | bc)
564+
mutation_score=$(printf "%.2f" "$mutation_score")
565+
else
566+
echo
567+
echo "Skipping mutation analysis (use -s flag)."
568+
mutation_score="N/A"
546569
fi
547-
echo
548-
"$MAJOR_HOME"/bin/"$ANT" -f "$SCRIPT_DIR"/program-config/"$1"/${buildfile} -Dbasedir="$SCRIPT_DIR" -Dbindir="$SCRIPT_DIR/build/bin/$FILE_SUFFIX" -Dresultdir="$RESULT_DIR" -Dtest="$TEST_DIRECTORY" -Dlibdir="$SCRIPT_DIR/build/lib/$UUID" mutation.test
549-
550-
# Calculate Mutation Score
551-
mutants_generated=$(awk -F, 'NR==2 {print $1}' "$RESULT_DIR"/summary.csv)
552-
mutants_killed=$(awk -F, 'NR==2 {print $4}' "$RESULT_DIR"/summary.csv)
553-
mutation_score=$(echo "scale=4; $mutants_killed / $mutants_generated * 100" | bc)
554-
mutation_score=$(printf "%.2f" "$mutation_score")
555570

556571
echo "Instruction Coverage: $instruction_coverage%"
557572
echo "Branch Coverage: $branch_coverage%"

‎scripts/mutation-randoop.sh‎

Lines changed: 41 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
#------------------------------------------------------------------------------
2424
# Options (command-line arguments):
2525
#------------------------------------------------------------------------------
26-
USAGE_STRING="usage: mutation-randoop.sh [-f features] [-o RESULTS_CSV] [-t total_time] [-c time_per_class] [-n num_iterations] [-r] [-v] [-h] TEST-CASE-NAME
26+
USAGE_STRING="usage: mutation-randoop.sh [-f features] [-o RESULTS_CSV] [-t total_time] [-c time_per_class] [-n num_iterations] [-s] [-r] [-v] [-h] TEST-CASE-NAME
2727
-f Specify the Randoop features to use.
2828
Available features: BASELINE, BLOODHOUND, ORIENTEERING, DETECTIVE, GRT_FUZZING, ELEPHANT_BRAIN, CONSTANT_MINING.
2929
example usage: -f BASELINE,BLOODHOUND
@@ -34,6 +34,7 @@ USAGE_STRING="usage: mutation-randoop.sh [-f features] [-o RESULTS_CSV] [-t tota
3434
-c N Per-class time limit (in seconds, default: 2s/class).
3535
Mutually exclusive with -t.
3636
-n N Number of iterations to run the experiment (default: 1).
37+
-s Skip mutation analysis (only run test generation and coverage).
3738
-r Redirect logs and diagnostics to results/result/mutation_output.txt.
3839
-v Enables verbose mode.
3940
-h Displays this help message.
@@ -92,10 +93,11 @@ fi
9293
NUM_LOOP=1 # Number of experiment runs (10 in GRT paper)
9394
VERBOSE=0 # Verbose option
9495
REDIRECT=0 # Redirect output to mutation_output.txt
96+
SKIP_MUTATION=0 # Skip mutation analysis
9597
UUID=$(uuidgen) # Generate a unique identifier per instance
9698

9799
# Parse command-line arguments
98-
while getopts ":hvrf:o:t:c:n:" opt; do
100+
while getopts ":hvrsf:o:t:c:n:" opt; do
99101
case ${opt} in
100102
h)
101103
# Display help message
@@ -110,6 +112,10 @@ while getopts ":hvrf:o:t:c:n:" opt; do
110112
# Redirect output to a log file
111113
REDIRECT=1
112114
;;
115+
s)
116+
# Skip mutation analysis
117+
SKIP_MUTATION=1
118+
;;
113119
f)
114120
FEATURES_OPT="$OPTARG"
115121
;;
@@ -161,10 +167,10 @@ declare -A FEATURE_FLAGS
161167
FEATURE_FLAGS=(
162168
["BLOODHOUND"]="--method-selection=BLOODHOUND"
163169
["ORIENTEERING"]="--input-selection=ORIENTEERING"
164-
["DETECTIVE"]="--demand-driven=true"
170+
["DETECTIVE"]="--call-non-sut-methods=true"
165171
["GRT_FUZZING"]="--grt-fuzzing=true"
166172
["ELEPHANT_BRAIN"]="--cast-to-run-time-type=true"
167-
["CONSTANT_MINING"]="--constant-mining=true"
173+
["CONSTANT_MINING"]="--literal-tfidf=true --literal-tfidf-probability=1.0 --include-superclass-literals=true"
168174
["BASELINE"]=""
169175
)
170176

@@ -174,7 +180,10 @@ for feat in "${RANDOOP_FEATURES[@]}"; do
174180
if [[ "${FEATURE_FLAGS[$feat]+exists}" ]]; then
175181
flag="${FEATURE_FLAGS[$feat]}"
176182
if [[ -n "$flag" ]]; then
177-
EXPANDED_FEATURE_FLAGS+=("$flag")
183+
# Split multi-argument feature strings into separate array entries
184+
# shellcheck disable=SC2206
185+
split_flags=($flag)
186+
EXPANDED_FEATURE_FLAGS+=("${split_flags[@]}")
178187
fi
179188
else
180189
echo "${SCRIPT_NAME}: error: unknown feature '$feat'"
@@ -610,6 +619,13 @@ for i in $(seq 1 "$NUM_LOOP"); do
610619
"$MAJOR_HOME"/bin/ant -f "$SCRIPT_DIR"/program-config/"$1"/${buildfile} -Dbasedir="$SCRIPT_DIR" -Dbindir="$SCRIPT_DIR/build/bin/$FILE_SUFFIX" -Dresultdir="$RESULT_DIR" -Dmutator="mml:$MAJOR_HOME/mml/all.mml.bin" -Dsrc="$JAVA_SRC_DIR" -Dtargetdir="$COVERAGE_DIRECTORY" -Dlibdir="$SCRIPT_DIR/build/lib/$UUID" compile.mutation
611620
"$MAJOR_HOME"/bin/ant -f "$SCRIPT_DIR"/program-config/"$1"/${buildfile} -Dbasedir="$SCRIPT_DIR" -Dbindir="$SCRIPT_DIR/build/bin/$FILE_SUFFIX" -Dresultdir="$RESULT_DIR" -Dmutator="mml:$MAJOR_HOME/mml/all.mml.bin" -Dsrc="$JAVA_SRC_DIR" -Dtargetdir="$COVERAGE_DIRECTORY" -Dlibdir="$SCRIPT_DIR/build/lib/$UUID" compile.jacoco
612621

622+
PYTHON_EXECUTABLE=$(command -v python3 2> /dev/null || command -v python 2> /dev/null)
623+
if [ -z "$PYTHON_EXECUTABLE" ]; then
624+
echo "Error: Python is not installed." >&2
625+
exit 2
626+
fi
627+
"$PYTHON_EXECUTABLE" "$SCRIPT_DIR"/trim_mutants.py "$RESULT_DIR/mutants.log"
628+
613629
echo
614630
echo "Compiling tests..."
615631
if [[ "$VERBOSE" -eq 1 ]]; then
@@ -653,28 +669,30 @@ for i in $(seq 1 "$NUM_LOOP"); do
653669
# proper isolation and compatibility for accurate mutant coverage.
654670

655671
if [ "$SUBJECT_PROGRAM" == "hamcrest-core-1.3" ]; then
656-
PYTHON_EXECUTABLE=$(command -v python3 2> /dev/null || command -v python 2> /dev/null)
657-
if [ -z "$PYTHON_EXECUTABLE" ]; then
658-
echo "Error: Python is not installed." >&2
659-
exit 1
660-
fi
661672
"$PYTHON_EXECUTABLE" "$SCRIPT_DIR"/convert_test_runners.py "$TEST_DIRECTORY" --mode randoop-to-evosuite
662673
fi
663674

664-
echo
665-
echo "Running tests with mutation analysis..."
666-
if [[ "$VERBOSE" -eq 1 ]]; then
667-
echo command:
668-
echo "$MAJOR_HOME"/bin/"$ANT" -f "$SCRIPT_DIR"/program-config/"$1"/${buildfile} -Dbasedir="$SCRIPT_DIR" -Dbindir="$SCRIPT_DIR/build/bin/$FILE_SUFFIX" -Dresultdir="$RESULT_DIR" -Dtest="$TEST_DIRECTORY" -Dlibdir="$SCRIPT_DIR/build/lib/$UUID" mutation.test
675+
# Run mutation analysis unless -s flag is set
676+
if [[ "$SKIP_MUTATION" -eq 0 ]]; then
677+
echo
678+
echo "Running tests with mutation analysis..."
679+
if [[ "$VERBOSE" -eq 1 ]]; then
680+
echo command:
681+
echo "$MAJOR_HOME"/bin/"$ANT" -f "$SCRIPT_DIR"/program-config/"$1"/${buildfile} -Dbasedir="$SCRIPT_DIR" -Dbindir="$SCRIPT_DIR/build/bin/$FILE_SUFFIX" -Dresultdir="$RESULT_DIR" -Dtest="$TEST_DIRECTORY" -Dlibdir="$SCRIPT_DIR/build/lib/$UUID" mutation.test
682+
fi
683+
echo
684+
"$MAJOR_HOME"/bin/"$ANT" -f "$SCRIPT_DIR"/program-config/"$1"/${buildfile} -Dbasedir="$SCRIPT_DIR" -Dbindir="$SCRIPT_DIR/build/bin/$FILE_SUFFIX" -Dresultdir="$RESULT_DIR" -Dtest="$TEST_DIRECTORY" -Dlibdir="$SCRIPT_DIR/build/lib/$UUID" mutation.test
685+
686+
# Calculate Mutation Score
687+
mutants_generated=$(awk -F, 'NR==2 {print $2}' "$RESULT_DIR"/summary.csv)
688+
mutants_killed=$(awk -F, 'NR==2 {print $4}' "$RESULT_DIR"/summary.csv)
689+
mutation_score=$(echo "scale=4; $mutants_killed / $mutants_generated * 100" | bc)
690+
mutation_score=$(printf "%.2f" "$mutation_score")
691+
else
692+
echo
693+
echo "Skipping mutation analysis (use -s flag)."
694+
mutation_score="N/A"
669695
fi
670-
echo
671-
"$MAJOR_HOME"/bin/"$ANT" -f "$SCRIPT_DIR"/program-config/"$1"/${buildfile} -Dbasedir="$SCRIPT_DIR" -Dbindir="$SCRIPT_DIR/build/bin/$FILE_SUFFIX" -Dresultdir="$RESULT_DIR" -Dtest="$TEST_DIRECTORY" -Dlibdir="$SCRIPT_DIR/build/lib/$UUID" mutation.test
672-
673-
# Calculate Mutation Score
674-
mutants_generated=$(awk -F, 'NR==2 {print $1}' "$RESULT_DIR"/summary.csv)
675-
mutants_killed=$(awk -F, 'NR==2 {print $4}' "$RESULT_DIR"/summary.csv)
676-
mutation_score=$(echo "scale=4; $mutants_killed / $mutants_generated * 100" | bc)
677-
mutation_score=$(printf "%.2f" "$mutation_score")
678696

679697
echo "Instruction Coverage: $instruction_coverage%"
680698
echo "Branch Coverage: $branch_coverage%"

‎scripts/program-config/ClassViewer-5.0.5b/build-evosuite.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@
8383
testOrder="sort_methods"
8484
serializedMapsFile="${resultdir}/preprocessing.ser"
8585
mutantsLogFile="${resultdir}/mutants.log"
86+
excludeMutantsFile="${resultdir}/exclude_mutants.txt"
8687
summaryFile="${resultdir}/summary.csv"
8788
mutantDetailsFile="${resultdir}/details.csv"
8889
covMapFile="${resultdir}/covMap.csv"

‎scripts/program-config/ClassViewer-5.0.5b/build-randoop.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@
8383
excludeFailingTests="false"
8484
serializedMapsFile="${resultdir}/preprocessing.ser"
8585
mutantsLogFile="${resultdir}/mutants.log"
86+
excludeMutantsFile="${resultdir}/exclude_mutants.txt"
8687
summaryFile="${resultdir}/summary.csv"
8788
mutantDetailsFile="${resultdir}/details.csv"
8889
covMapFile="${resultdir}/covMap.csv"

‎scripts/program-config/JSAP-2.1/build-evosuite.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@
8787
testOrder="sort_methods"
8888
serializedMapsFile="${resultdir}/preprocessing.ser"
8989
mutantsLogFile="${resultdir}/mutants.log"
90+
excludeMutantsFile="${resultdir}/exclude_mutants.txt"
9091
summaryFile="${resultdir}/summary.csv"
9192
mutantDetailsFile="${resultdir}/details.csv"
9293
covMapFile="${resultdir}/covMap.csv"

‎scripts/program-config/JSAP-2.1/build-randoop.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@
8686
excludeFailingTests="false"
8787
serializedMapsFile="${resultdir}/preprocessing.ser"
8888
mutantsLogFile="${resultdir}/mutants.log"
89+
excludeMutantsFile="${resultdir}/exclude_mutants.txt"
8990
summaryFile="${resultdir}/summary.csv"
9091
mutantDetailsFile="${resultdir}/details.csv"
9192
covMapFile="${resultdir}/covMap.csv"

‎scripts/program-config/a4j-1.0b/build-evosuite.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@
8383
testOrder="sort_methods"
8484
serializedMapsFile="${resultdir}/preprocessing.ser"
8585
mutantsLogFile="${resultdir}/mutants.log"
86+
excludeMutantsFile="${resultdir}/exclude_mutants.txt"
8687
summaryFile="${resultdir}/summary.csv"
8788
mutantDetailsFile="${resultdir}/details.csv"
8889
covMapFile="${resultdir}/covMap.csv"

‎scripts/program-config/a4j-1.0b/build-randoop.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@
8181
excludeFailingTests="false"
8282
serializedMapsFile="${resultdir}/preprocessing.ser"
8383
mutantsLogFile="${resultdir}/mutants.log"
84+
excludeMutantsFile="${resultdir}/exclude_mutants.txt"
8485
summaryFile="${resultdir}/summary.csv"
8586
mutantDetailsFile="${resultdir}/details.csv"
8687
covMapFile="${resultdir}/covMap.csv"

‎scripts/program-config/asm-5.0.1/build-evosuite.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@
8585
testOrder="sort_methods"
8686
serializedMapsFile="${resultdir}/preprocessing.ser"
8787
mutantsLogFile="${resultdir}/mutants.log"
88+
excludeMutantsFile="${resultdir}/exclude_mutants.txt"
8889
summaryFile="${resultdir}/summary.csv"
8990
mutantDetailsFile="${resultdir}/details.csv"
9091
covMapFile="${resultdir}/covMap.csv"

‎scripts/program-config/asm-5.0.1/build-randoop.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@
8484
excludeFailingTests="false"
8585
serializedMapsFile="${resultdir}/preprocessing.ser"
8686
mutantsLogFile="${resultdir}/mutants.log"
87+
excludeMutantsFile="${resultdir}/exclude_mutants.txt"
8788
summaryFile="${resultdir}/summary.csv"
8889
mutantDetailsFile="${resultdir}/details.csv"
8990
covMapFile="${resultdir}/covMap.csv"

0 commit comments

Comments
 (0)