Skip to content

Commit 6f4cf01

Browse files
arcuri82arcuri82
authored
Merge pull request #1607 from WebFuzzing/regex-java-external-flags
Regex java external flags
2 parents 6ddc929 + cab73aa commit 6f4cf01

9 files changed

Lines changed: 138 additions & 22 deletions

File tree

‎client-java/instrumentation-shared/src/main/java/org/evomaster/client/java/instrumentation/shared/StringSpecializationInfo.java‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,17 +16,29 @@ public class StringSpecializationInfo implements Serializable {
1616

1717
private final TaintType type;
1818

19+
/**
20+
* External regex flags bitmask, as accepted by java.util.regex.Pattern.compile(String, int).
21+
* Only meaningful when stringSpecialization is a regex type.
22+
* Defaults to 0 (no external flags).
23+
*/
24+
private final int externalRegexFlagsBitmask;
25+
1926
public StringSpecializationInfo(StringSpecialization stringSpecialization, String value) {
2027
this(stringSpecialization, value, TaintType.FULL_MATCH);
2128
}
2229

2330
public StringSpecializationInfo(StringSpecialization stringSpecialization, String value, TaintType taintType) {
31+
this(stringSpecialization, value, taintType, 0);
32+
}
33+
34+
public StringSpecializationInfo(StringSpecialization stringSpecialization, String value, TaintType taintType, int externalRegexFlagsBitmask) {
2435
this.stringSpecialization = Objects.requireNonNull(stringSpecialization);
2536
this.value = value;
2637
if(taintType == null || taintType == TaintType.NONE){
2738
throw new IllegalArgumentException("Invalid type: "+taintType);
2839
}
2940
this.type = taintType;
41+
this.externalRegexFlagsBitmask = externalRegexFlagsBitmask;
3042
}
3143

3244
public StringSpecialization getStringSpecialization() {
@@ -41,18 +53,25 @@ public TaintType getType() {
4153
return type;
4254
}
4355

56+
/**
57+
* Getter for regex flags bitmask, only meaningful when stringSpecialization is regex type. Defaults to 0 (no flags)
58+
* @return Integer bitmask for the regex flags associated to the string.
59+
*/
60+
public int getExternalRegexFlagsBitmask() { return externalRegexFlagsBitmask; }
61+
4462
@Override
4563
public boolean equals(Object o) {
4664
if (this == o) return true;
4765
if (o == null || getClass() != o.getClass()) return false;
4866
StringSpecializationInfo that = (StringSpecializationInfo) o;
4967
return stringSpecialization == that.stringSpecialization &&
5068
Objects.equals(value, that.value) &&
51-
type == that.type;
69+
type == that.type &&
70+
externalRegexFlagsBitmask == that.externalRegexFlagsBitmask;
5271
}
5372

5473
@Override
5574
public int hashCode() {
56-
return Objects.hash(stringSpecialization, value, type);
75+
return Objects.hash(stringSpecialization, value, type, externalRegexFlagsBitmask);
5776
}
5877
}

‎client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/coverage/methodreplacement/PatternMatchingHelper.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import org.evomaster.client.java.instrumentation.shared.ReplacementType;
66
import org.evomaster.client.java.instrumentation.shared.StringSpecialization;
77
import org.evomaster.client.java.instrumentation.shared.StringSpecializationInfo;
8+
import org.evomaster.client.java.instrumentation.shared.TaintType;
89
import org.evomaster.client.java.instrumentation.staticstate.ExecutionTracer;
910

1011
import java.util.Objects;
@@ -25,16 +26,16 @@ public static boolean matches(String regex, String input, String idTemplate) {
2526
/**
2627
* Invocation to Pattern.matches() is free of side-effects.
2728
*/
28-
public static boolean matches(String regex, int flags, String input, String idTemplate) {
29+
public static boolean matches(String regex, int externalRegexFlagsBitmask, String input, String idTemplate) {
2930
Objects.requireNonNull(regex);
3031
Objects.requireNonNull(input);
3132

3233
if (ExecutionTracer.isTaintInput(input)) {
3334
ExecutionTracer.addStringSpecialization(input,
34-
new StringSpecializationInfo(StringSpecialization.REGEX_WHOLE, regex));
35+
new StringSpecializationInfo(StringSpecialization.REGEX_WHOLE, regex, TaintType.FULL_MATCH, externalRegexFlagsBitmask));
3536
}
3637

37-
Pattern p = Pattern.compile(regex, flags);
38+
Pattern p = Pattern.compile(regex, externalRegexFlagsBitmask);
3839
Matcher m = p.matcher(input);
3940
boolean matches = m.matches();
4041

‎client-java/instrumentation/src/main/java/org/evomaster/client/java/instrumentation/coverage/methodreplacement/classes/MatcherClassReplacement.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -48,9 +48,9 @@ public static boolean matches(Matcher caller, String idTemplate) {
4848

4949
String text = getText(caller);
5050
String pattern = caller.pattern().toString();
51-
int flags = caller.pattern().flags();
51+
int externalRegexFlagsBitmask = caller.pattern().flags();
5252

53-
boolean patternMatchesResult = PatternMatchingHelper.matches(pattern, flags, text, idTemplate);
53+
boolean patternMatchesResult = PatternMatchingHelper.matches(pattern, externalRegexFlagsBitmask, text, idTemplate);
5454

5555
TaintType taintType = ExecutionTracer.getTaintType(text);
5656

@@ -60,7 +60,7 @@ public static boolean matches(Matcher caller, String idTemplate) {
6060
*/
6161
String regex = caller.pattern().toString();
6262
ExecutionTracer.addStringSpecialization(text,
63-
new StringSpecializationInfo(StringSpecialization.REGEX_WHOLE, regex, taintType));
63+
new StringSpecializationInfo(StringSpecialization.REGEX_WHOLE, regex, taintType, externalRegexFlagsBitmask));
6464
}
6565
boolean matcherMatchesResults = caller.matches();
6666
assert (patternMatchesResult == matcherMatchesResults);
@@ -73,6 +73,7 @@ public static boolean find(Matcher caller, String idTemplate) {
7373

7474
String input = getText(caller);
7575
String regex = caller.pattern().toString();
76+
int externalRegexFlagsBitmask = caller.pattern().flags();
7677
int end;
7778
try {
7879
end = caller.end();
@@ -105,7 +106,7 @@ match the regex, and find() only requires
105106
TaintType taintType = ExecutionTracer.getTaintType(substring);
106107
if (taintType.isTainted()) {
107108
ExecutionTracer.addStringSpecialization(substring,
108-
new StringSpecializationInfo(StringSpecialization.REGEX_PARTIAL, regex, taintType));
109+
new StringSpecializationInfo(StringSpecialization.REGEX_PARTIAL, regex, taintType, externalRegexFlagsBitmask));
109110
}
110111

111112
String anyPositionRegexMatch = RegexSharedUtils.handlePartialMatch(regex);

‎core/src/main/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitor.kt‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ private const val EOF_TOKEN = "<EOF>"
1111
/**
1212
* Created by arcuri82 on 11-Sep-19.
1313
*/
14-
class GeneRegexJavaVisitor : RegexJavaBaseVisitor<VisitResult>(){
14+
class GeneRegexJavaVisitor(externalRegexFlags: RegexFlags = RegexFlags()) : RegexJavaBaseVisitor<VisitResult>(){
1515

1616
private val hexEscapePrefixes = setOf('x', 'u')
1717

@@ -52,8 +52,9 @@ class GeneRegexJavaVisitor : RegexJavaBaseVisitor<VisitResult>(){
5252
/**
5353
* Tracks the flags active in the current lexical scope.
5454
* Updated when entering a flag group, restored on exit.
55+
* Initialized from [externalRegexFlags].
5556
*/
56-
private var currentFlags = RegexFlags()
57+
private var currentFlags = externalRegexFlags
5758

5859
/**
5960
* Parses a FLAG_GROUP_OPEN or FLAG_SCOPE_OPEN token text like "(?i:", "(?iu:", "(?-i:", "(?i-u:", "(?iu)", etc.

‎core/src/main/kotlin/org/evomaster/core/parser/RegexHandler.kt‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ package org.evomaster.core.parser
33
import org.antlr.v4.runtime.*
44
import org.antlr.v4.runtime.misc.ParseCancellationException
55
import org.evomaster.core.search.gene.regex.RegexGene
6+
import org.evomaster.core.utils.RegexFlags
7+
import org.evomaster.core.utils.RegexWithExternalFlags
68

79

810
/**
@@ -19,15 +21,16 @@ object RegexHandler {
1921
WARNING mutable static state, but those are just caches.
2022
Key -> regex
2123
*/
22-
private val cacheJVM : MutableMap<String, RegexGene> = mutableMapOf()
24+
private val cacheJVM : MutableMap<RegexWithExternalFlags, RegexGene> = mutableMapOf()
2325
private val cacheEcma262 : MutableMap<String, RegexGene> = mutableMapOf()
2426
private val cachePostgresLike : MutableMap<String, RegexGene> = mutableMapOf()
2527
private val cachePostgresSimilarTo : MutableMap<String, RegexGene> = mutableMapOf()
2628

27-
fun createGeneForJVM(regex: String) : RegexGene {
29+
fun createGeneForJVM(regex: String, externalRegexFlags: RegexFlags = RegexFlags()) : RegexGene {
2830

29-
if(cacheJVM.contains(regex)){
30-
return cacheJVM[regex]!!.copy() as RegexGene
31+
val key = RegexWithExternalFlags(regex, externalRegexFlags)
32+
if(cacheJVM.contains(key)){
33+
return cacheJVM[key]!!.copy() as RegexGene
3134
}
3235

3336
val stream = CharStreams.fromString(regex)
@@ -38,10 +41,10 @@ object RegexHandler {
3841

3942
val pattern = parser.pattern()
4043

41-
val res = GeneRegexJavaVisitor().visit(pattern)
44+
val res = GeneRegexJavaVisitor(externalRegexFlags).visit(pattern)
4245

4346
val gene= res.genes.first() as RegexGene
44-
cacheJVM[regex] = gene.copy() as RegexGene
47+
cacheJVM[key] = gene.copy() as RegexGene
4548
return gene
4649
}
4750

‎core/src/main/kotlin/org/evomaster/core/search/gene/string/StringGene.kt‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ import org.evomaster.core.search.service.Randomness
3636
import org.evomaster.core.search.service.mutator.MutationWeightControl
3737
import org.evomaster.core.search.service.mutator.genemutation.AdditionalGeneMutationInfo
3838
import org.evomaster.core.search.service.mutator.genemutation.SubsetGeneMutationSelectionStrategy
39+
import org.evomaster.core.utils.RegexWithExternalFlags
3940
import org.slf4j.Logger
4041
import org.slf4j.LoggerFactory
4142
import java.lang.IllegalStateException
@@ -811,16 +812,17 @@ class StringGene(
811812
.filter{RegexUtils.isMeaningfulRegex(it.value)}
812813
.filter{RegexUtils.isNotUselessRegex(it.value) }
813814
.map {
814-
if(it.stringSpecialization == StringSpecialization.REGEX_WHOLE) {
815+
val regex = if(it.stringSpecialization == StringSpecialization.REGEX_WHOLE) {
815816
RegexSharedUtils.forceFullMatch(it.value)
816817
} else {
817818
RegexSharedUtils.handlePartialMatch(it.value)
818819
}
820+
RegexWithExternalFlags(regex, it.externalRegexFlagsBitmask)
819821
}
820822
//.joinToString("|")
821-
.forEach {regex ->
823+
.forEach {(regex, externalFlags) ->
822824
try {
823-
toAddGenes.add(RegexHandler.createGeneForJVM(regex))
825+
toAddGenes.add(RegexHandler.createGeneForJVM(regex, externalFlags))
824826
log.trace("Regex, added specification for: {}", regex)
825827
} catch (e: Exception) {
826828
LoggingUtil.uniqueWarn(log, "Failed to handle regex: $regex")

‎core/src/main/kotlin/org/evomaster/core/utils/RegexFlags.kt‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package org.evomaster.core.utils
22

3+
import java.util.regex.Pattern
4+
35
/**
46
* Represents a parsed flag expression like "(?iu-s:".
57
* Encapsulates the flags being turned on and off, to be applied to an existing [RegexFlags] via [RegexFlags.merge].
@@ -57,6 +59,22 @@ data class RegexFlags(
5759
comments = 'x' in s,
5860
)
5961
}
62+
63+
/**
64+
* Constructs a [RegexFlags] from a Java regex flags bitmask, as accepted by
65+
* [java.util.regex.Pattern.compile]. This allows external flags passed to
66+
* [java.util.regex.Pattern.compile] to be preserved and applied when building
67+
* the gene tree, mirroring the behaviour of the Java regex engine.
68+
*/
69+
fun fromExternalJavaRegexFlagBitmask(externalRegexFlagsBitmask: Int): RegexFlags = RegexFlags(
70+
caseInsensitive = externalRegexFlagsBitmask and Pattern.CASE_INSENSITIVE != 0,
71+
unicodeCase = externalRegexFlagsBitmask and Pattern.UNICODE_CASE != 0,
72+
dotAll = externalRegexFlagsBitmask and Pattern.DOTALL != 0,
73+
multiline = externalRegexFlagsBitmask and Pattern.MULTILINE != 0,
74+
unixLines = externalRegexFlagsBitmask and Pattern.UNIX_LINES != 0,
75+
unicodeCharacterClass = externalRegexFlagsBitmask and Pattern.UNICODE_CHARACTER_CLASS != 0,
76+
comments = externalRegexFlagsBitmask and Pattern.COMMENTS != 0
77+
)
6078
}
6179

6280
/**

‎core/src/main/kotlin/org/evomaster/core/utils/RegexWithExternalFlags.kt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
package org.evomaster.core.utils
2+
3+
data class RegexWithExternalFlags(val regex: String, val externalRegexFlags: RegexFlags) {
4+
constructor (regex: String, externalRegexFlagBitmask: Int) : this(regex, RegexFlags.fromExternalJavaRegexFlagBitmask(externalRegexFlagBitmask))
5+
}

‎core/src/test/kotlin/org/evomaster/core/parser/RegexHandlerTest.kt‎

Lines changed: 68 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,13 @@ package org.evomaster.core.parser
33
import org.antlr.v4.runtime.misc.ParseCancellationException
44
import org.evomaster.client.java.instrumentation.heuristic.ValidatorHeuristics
55
import org.evomaster.client.java.instrumentation.shared.RegexSharedUtils
6-
import org.evomaster.core.search.gene.regex.RegexGene
76
import org.evomaster.core.search.service.AdaptiveParameterControl
87
import org.evomaster.core.search.service.Randomness
98
import org.evomaster.core.search.service.mutator.MutationWeightControl
9+
import org.evomaster.core.utils.RegexFlags
1010
import org.junit.jupiter.api.Assertions.*
1111
import org.junit.jupiter.api.Disabled
1212
import org.junit.jupiter.api.Test
13-
import org.junit.jupiter.api.assertThrows
1413
import java.util.regex.Pattern
1514

1615
internal class RegexHandlerTest{
@@ -170,4 +169,71 @@ internal class RegexHandlerTest{
170169
assertThrows(ParseCancellationException::class.java) { RegexHandler.createGeneForJVM("\\ugggg") }
171170
assertThrows(IllegalArgumentException::class.java) { RegexHandler.createGeneForJVM("[9-1]") }
172171
}
172+
173+
@Test
174+
fun testJVMExternalCaseInsensitiveFlagWithUnicodeCase() {
175+
// \u03A1 is greek capital Rho, \u03C1 is lowercase rho
176+
val regex = "\u03A1+"
177+
val flags = RegexFlags.fromExternalJavaRegexFlagBitmask(Pattern.CASE_INSENSITIVE or Pattern.UNICODE_CASE)
178+
val gene = RegexHandler.createGeneForJVM(regex, flags)
179+
val pattern = Pattern.compile(regex, Pattern.CASE_INSENSITIVE or Pattern.UNICODE_CASE)
180+
val rand = Randomness()
181+
182+
repeat(200) {
183+
gene.randomize(rand, false)
184+
val value = gene.getValueAsRawString()
185+
assertTrue(pattern.matcher(value).find())
186+
}
187+
}
188+
189+
@Test
190+
fun testJVMCacheDistinguishesByExternalFlags() {
191+
192+
val regex = "^abc$"
193+
val noFlags = RegexHandler.createGeneForJVM(regex)
194+
val withCI = RegexHandler.createGeneForJVM(regex, RegexFlags.fromExternalJavaRegexFlagBitmask(Pattern.CASE_INSENSITIVE))
195+
196+
val rand = Randomness()
197+
val noFlagsSamples = (1..200).map {
198+
noFlags.randomize(rand, false)
199+
noFlags.getValueAsRawString()
200+
}.toSet()
201+
202+
val withCISamples = (1..200).map {
203+
withCI.randomize(rand, false)
204+
withCI.getValueAsRawString()
205+
}.toSet()
206+
207+
assertTrue(noFlagsSamples.all { it == "abc" })
208+
assertTrue(withCISamples.size > 1)
209+
}
210+
211+
@Test
212+
fun testJVMInlineFlagNotDoubledByExternalFlag() {
213+
214+
val regex = "(?i:abc)"
215+
val withExternalCI = RegexHandler.createGeneForJVM(regex, RegexFlags.fromExternalJavaRegexFlagBitmask(Pattern.CASE_INSENSITIVE))
216+
val pattern = Pattern.compile(regex, Pattern.CASE_INSENSITIVE)
217+
val rand = Randomness()
218+
219+
repeat(200) {
220+
withExternalCI.randomize(rand, false)
221+
val value = withExternalCI.getValueAsRawString()
222+
assertTrue(pattern.matcher(value).find())
223+
}
224+
}
225+
226+
@Test
227+
fun testJVMInlineCanDisableExternalFlag() {
228+
229+
val regex = "^(?-i:abc)$"
230+
val withExternalCI = RegexHandler.createGeneForJVM(regex, RegexFlags.fromExternalJavaRegexFlagBitmask(Pattern.CASE_INSENSITIVE))
231+
val rand = Randomness()
232+
233+
repeat(200) {
234+
withExternalCI.randomize(rand, false)
235+
val value = withExternalCI.getValueAsRawString()
236+
assertEquals("abc", value)
237+
}
238+
}
173239
}

0 commit comments

Comments
 (0)