Skip to content

Commit 127bf4c

Browse files
agusaldasoroagusaldasoro
authored
Merge branch 'master' into dse/memoization
2 parents c36024f + 7578b60 commit 127bf4c

6 files changed

Lines changed: 211 additions & 50 deletions

File tree

‎core/src/main/antlr4/org/evomaster/core/parser/RegexJava.g4‎

Lines changed: 1 addition & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,6 @@ quoteChar
129129
| E
130130
;
131131

132-
//TODO
133132
CharacterEscape
134133
: SLASH ControlEscape
135134
| SLASH 'c' ControlLetter
@@ -138,8 +137,7 @@ CharacterEscape
138137
| SLASH OctalEscapeSequence
139138
| SLASH ('p' | 'P') BRACE_open PCharacterClassEscapeLabel BRACE_close // this is only implemented in Java at the moment
140139
// as on JS this is allowed only while certain flags are enabled
141-
142-
//| IdentityEscape
140+
| SLASH ~[a-zA-Z0-9] // identity escape
143141
;
144142

145143
// Instead of listing all unicode scripts, blocks, etc. the parser allows anything
@@ -164,13 +162,6 @@ fragment ControlLetter
164162
: [?-_a-z]
165163
;
166164

167-
168-
//TODO
169-
//fragment IdentityEscape ::
170-
//SourceCharacter but not IdentifierPart
171-
//<ZWJ>
172-
//<ZWNJ>
173-
174165
//TODO
175166
//DecimalEscape
176167
// //[lookahead ∉ DecimalDigit]
@@ -261,7 +252,6 @@ classEscape
261252
atomEscape
262253
: CharacterClassEscape
263254
| CharacterEscape
264-
| SyntaxEscapes
265255
| BackReference
266256
| NamedBackReference
267257
;
@@ -284,11 +274,6 @@ CharacterClassEscape
284274
: SLASH [dDsSwWvVhH]
285275
;
286276

287-
288-
SyntaxEscapes
289-
: SLASH [^$\\.*+?()[\]{}|/\-,:<>=!]
290-
;
291-
292277
CARET : '^';
293278
DOLLAR : '$';
294279
SLASH : '\\';

‎core/src/main/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitor.kt‎

Lines changed: 6 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,9 @@ class GeneRegexJavaVisitor(externalRegexFlags: RegexFlags = RegexFlags()) : Rege
2828
)
2929

3030
/**
31-
* These are the Java regex syntax characters, all of these can be escaped to be treated as literals.
31+
* None of these can be escaped to be treated as literals. Some may be part of legal escape sequences.
3232
*/
33-
private val allowedSyntaxEscapes = setOf(
34-
'^', '$', '\\', '.', '*', '+', '?',
35-
'(', ')', '[', ']', '{', '}', '|',
36-
'/', '-', ',' ,':', '<', '>', '=', '!'
37-
)
33+
private val notIdentityEscapes = ('a'..'z').toList() + ('A'..'Z').toList() + ('0'..'9').toList()
3834

3935
/**
4036
* Capture groups in order of appearance (1-based index -> list index 0).
@@ -64,24 +60,6 @@ class GeneRegexJavaVisitor(externalRegexFlags: RegexFlags = RegexFlags()) : Rege
6460
*/
6561
private var currentFlags = externalRegexFlags
6662

67-
/**
68-
* Parses a FLAG_GROUP_OPEN or FLAG_SCOPE_OPEN token text like "(?i:", "(?iu:", "(?-i:", "(?i-u:", "(?iu)", etc.
69-
* into a [ParsedFlagExpression] that can be applied to the current flags.
70-
*/
71-
private fun parseFlagToken(tokenText: String): ParsedFlagExpression {
72-
// strip "(?" from start and ":" (or ")") from end
73-
val inner = tokenText.drop(2).dropLast(1)
74-
75-
val (enableStr, disableStr) = if ('-' in inner)
76-
inner.split('-', limit = 2).let { it[0] to it[1] }
77-
else Pair(inner, "")
78-
79-
return ParsedFlagExpression(
80-
RegexFlags.fromString(enableStr),
81-
RegexFlags.fromString(disableStr)
82-
)
83-
}
84-
8563
/**
8664
* Builds DisjunctionListRxGenes from a disjunction context, returns null if disjunction is unsatisfiable.
8765
*/
@@ -181,7 +159,7 @@ class GeneRegexJavaVisitor(externalRegexFlags: RegexFlags = RegexFlags()) : Rege
181159
val previous = currentFlags
182160

183161
val merged = currentFlags.merge(
184-
parseFlagToken(term.FLAG_SCOPE_OPEN().text)
162+
ParsedFlagExpression.fromFlagToken(term.FLAG_SCOPE_OPEN().text)
185163
)
186164

187165
merged.validate()
@@ -367,7 +345,7 @@ class GeneRegexJavaVisitor(externalRegexFlags: RegexFlags = RegexFlags()) : Rege
367345
val previous = currentFlags
368346

369347
val merged = currentFlags.merge(
370-
parseFlagToken(ctx.FLAG_GROUP_OPEN().text)
348+
ParsedFlagExpression.fromFlagToken(ctx.FLAG_GROUP_OPEN().text)
371349
)
372350

373351
merged.validate()
@@ -546,7 +524,7 @@ class GeneRegexJavaVisitor(externalRegexFlags: RegexFlags = RegexFlags()) : Rege
546524
} else {
547525
// This case handles the escaped syntax characters, like "\." and "\+", etc. cases
548526
// where '.' and '+', etc. should be treated as regular chars
549-
assert(startText[0] == '\\' && startText[1] in allowedSyntaxEscapes)
527+
assert(startText[0] == '\\' && startText[1] !in notIdentityEscapes)
550528
start = startText[1]
551529
end = start
552530
}
@@ -722,7 +700,7 @@ class GeneRegexJavaVisitor(externalRegexFlags: RegexFlags = RegexFlags()) : Rege
722700
currentFlags
723701
)
724702
}
725-
in allowedSyntaxEscapes -> PatternCharacterBlockGene(txt, txt.substring(1), currentFlags)
703+
!in notIdentityEscapes -> PatternCharacterBlockGene(txt, txt.substring(1), currentFlags)
726704
else -> CharacterClassEscapeRxGene(txt.substring(1), currentFlags)
727705
})
728706
}

‎core/src/main/kotlin/org/evomaster/core/parser/RegexHandler.kt‎

Lines changed: 117 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package org.evomaster.core.parser
33
import org.antlr.v4.runtime.*
44
import org.antlr.v4.runtime.misc.ParseCancellationException
55
import org.evomaster.core.search.gene.regex.RegexGene
6+
import org.evomaster.core.utils.ParsedFlagExpression
67
import org.evomaster.core.utils.RegexFlags
78
import org.evomaster.core.utils.RegexWithExternalFlags
89

@@ -33,7 +34,9 @@ object RegexHandler {
3334
return cacheJVM[key]!!.copy() as RegexGene
3435
}
3536

36-
val stream = CharStreams.fromString(regex)
37+
val preprocessedRegex = preprocessCommentsForJavaRegex(regex, externalRegexFlags)
38+
39+
val stream = CharStreams.fromString(preprocessedRegex)
3740
val lexer = RegexJavaLexer(stream)
3841
val tokenStream = prepareLexer(lexer)
3942
val parser = RegexJavaParser(tokenStream)
@@ -48,6 +51,119 @@ object RegexHandler {
4851
return gene
4952
}
5053

54+
/**
55+
* This function handles comments and whitespace for Java regex, striping them when the "x" flag is on.
56+
*
57+
* This cannot be handled at the ANTLR level because the flag can be enabled and disabled
58+
* mid-pattern via inline flag groups like `(?x:...)` and `(?-x:...)`, which are only known
59+
* at parse time. Lexer modes cannot react to parser-level flag state.
60+
*
61+
* The visitor cannot handle this either, as some constructs (character classes, quantifier bounds, etc.)
62+
* are tokenised before the visitor sees them.
63+
*
64+
* This function therefore performs a linear scan of the raw string before ANTLR, tracking
65+
* flag state across inline scopes, producing a cleaned string that requires no special
66+
* handling in the lexer or visitor.
67+
*/
68+
private fun preprocessCommentsForJavaRegex(regex: String, externalRegexFlags: RegexFlags): String {
69+
val result = StringBuilder(regex.length)
70+
val scopeStack = ArrayDeque<RegexFlags>() // stack of flags per level
71+
var currentFlags = externalRegexFlags
72+
var i = 0
73+
74+
while (i < regex.length) {
75+
val c = regex[i]
76+
when {
77+
// backslash escape
78+
c == '\\' && i + 1 < regex.length -> {
79+
when {
80+
regex[i+1] == 'Q' -> {
81+
// \Q...\E quote block, copy everything
82+
result.append('\\'); result.append('Q')
83+
i += 2
84+
while (i < regex.length) {
85+
if (regex[i] == '\\' && i+1 < regex.length && regex[i+1] == 'E') {
86+
result.append('\\'); result.append('E')
87+
i += 2; break
88+
}
89+
result.append(regex[i++])
90+
}
91+
}
92+
else -> {
93+
// regular escape, copy both
94+
result.append(c); result.append(regex[i+1])
95+
i += 2
96+
}
97+
}
98+
}
99+
100+
// opening paren: check for flag group or scope
101+
c == '(' && i+1 < regex.length && regex[i+1] == '?' -> {
102+
// scan forward to find the flag content
103+
val flagStart = i + 2
104+
var j = flagStart
105+
// lookahead to end of group/scope/other, set j to that position
106+
while (j < regex.length && regex[j] != ':' && regex[j] != ')' && regex[j] != '(') j++
107+
108+
// check if regex[i..j] forms valid flag scope/group
109+
if (j < regex.length && (regex[j] == ':' || regex[j] == ')') && j > i+2
110+
&& regex.substring(i+2, j).all{ it in RegexFlags.validFlagCharacters || it == '-' }) {
111+
// valid flag group/scope
112+
if(regex[j] == ':') {
113+
// flag group (?flags:...): parse flags and push scope
114+
val flagToken = regex.substring(i, j+1) // e.g. "(?iu:"
115+
val newFlags = currentFlags.merge(ParsedFlagExpression.fromFlagToken(flagToken))
116+
scopeStack.addLast(currentFlags)
117+
currentFlags = newFlags
118+
result.append(regex.substring(i, j+1))
119+
i = j + 1
120+
} else {
121+
// flag scope (?flags), update currentFlags
122+
val flagToken = regex.substring(i, j+1) // e.g. "(?iu)"
123+
currentFlags = currentFlags.merge(ParsedFlagExpression.fromFlagToken(flagToken))
124+
result.append(regex.substring(i, j+1))
125+
i = j + 1
126+
}
127+
} else {
128+
// not a flag group/scope: push current flags unchanged
129+
scopeStack.addLast(currentFlags)
130+
result.append(c); i++
131+
}
132+
}
133+
134+
c == '(' -> {
135+
scopeStack.addLast(currentFlags)
136+
result.append(c); i++
137+
}
138+
139+
c == ')' -> {
140+
currentFlags = scopeStack.removeLastOrNull() ?: externalRegexFlags
141+
result.append(c); i++
142+
}
143+
144+
// comment (when COMMENTS flag is on)
145+
c == '#' && currentFlags.comments -> {
146+
i++
147+
// advance index until line terminator (eg: "#...\n") without copying
148+
while (i < regex.length && !currentFlags.isLineTerminator(regex[i])) i++
149+
// consume line terminator too:
150+
if (i < regex.length) {
151+
// \r\n is a 2-character line terminator
152+
if (regex[i] == '\r' && i+1 < regex.length && regex[i+1] == '\n') i += 2
153+
else i++
154+
}
155+
}
156+
157+
// whitespace, skip copying when comments flag is on
158+
c.isWhitespace() && currentFlags.comments -> i++
159+
160+
// else copy
161+
else -> { result.append(c); i++ }
162+
}
163+
}
164+
return result.toString()
165+
}
166+
51167
/**
52168
* Given a ECMA262 regex string, generate RegexGene for it.
53169
* Based on RegexEcma262.g4 file.

‎core/src/main/kotlin/org/evomaster/core/utils/RegexFlags.kt‎

Lines changed: 44 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,27 @@ data class ParsedFlagExpression(
2525
enable -> true
2626
else -> current
2727
}
28-
}
2928

30-
private val validFlagCharacters = setOf('i', 'u', 's', 'm', 'd', 'U', 'x')
29+
companion object {
30+
/**
31+
* Parses a FLAG_GROUP_OPEN or FLAG_SCOPE_OPEN token text like "(?i:", "(?iu:", "(?-i:", "(?i-u:", "(?iu)", etc.
32+
* into a [ParsedFlagExpression] that can be applied to the current flags.
33+
*/
34+
fun fromFlagToken(tokenText: String): ParsedFlagExpression {
35+
// strip "(?" from start and ":" (or ")") from end
36+
val inner = tokenText.drop(2).dropLast(1)
37+
38+
val (enableStr, disableStr) = if ('-' in inner)
39+
inner.split('-', limit = 2).let { it[0] to it[1] }
40+
else Pair(inner, "")
41+
42+
return ParsedFlagExpression(
43+
RegexFlags.fromString(enableStr),
44+
RegexFlags.fromString(disableStr)
45+
)
46+
}
47+
}
48+
}
3149

3250
data class RegexFlags(
3351
// currently implemented
@@ -43,6 +61,8 @@ data class RegexFlags(
4361
) {
4462

4563
companion object {
64+
val validFlagCharacters = setOf('i', 'u', 's', 'm', 'd', 'U', 'x')
65+
4666
/**
4767
* Parses a string of flag characters (e.g. "iu", "sm") into a [RegexFlags] instance.
4868
* Valid characters are: i, u, s, m, d, U, x.
@@ -90,8 +110,7 @@ data class RegexFlags(
90110
unicodeCharacterClass ||
91111
comments)) {
92112
return ""
93-
}
94-
else {
113+
} else {
95114
val sb = StringBuilder()
96115
sb.append("(?")
97116

@@ -122,6 +141,18 @@ data class RegexFlags(
122141
}
123142
}
124143

144+
fun toJavaFlagBitmask(): Int {
145+
var flags = 0
146+
if (caseInsensitive) flags = flags or Pattern.CASE_INSENSITIVE
147+
if (unicodeCase) flags = flags or Pattern.UNICODE_CASE
148+
if (dotAll) flags = flags or Pattern.DOTALL
149+
if (multiline) flags = flags or Pattern.MULTILINE
150+
if (unixLines) flags = flags or Pattern.UNIX_LINES
151+
if (unicodeCharacterClass) flags = flags or Pattern.UNICODE_CHARACTER_CLASS
152+
if (comments) flags = flags or Pattern.COMMENTS
153+
return flags
154+
}
155+
125156
/**
126157
* Merges this [RegexFlags] with a [ParsedFlagExpression], returning a new [RegexFlags] with the
127158
* enabled flags turned on and the disabled flags turned off.
@@ -139,9 +170,17 @@ data class RegexFlags(
139170
if (multiline) throw IllegalStateException("Regex flag 'm' (MULTILINE) is not yet supported")
140171
if (unixLines) throw IllegalStateException("Regex flag 'd' (UNIX_LINES) is not yet supported")
141172
if (unicodeCharacterClass) throw IllegalStateException("Regex flag 'U' (UNICODE_CHARACTER_CLASS) is not yet supported")
142-
if (comments) throw IllegalStateException("Regex flag 'x' (COMMENTS) is not yet supported")
143173
}
144174

175+
/**
176+
* Checks if the provided character is a line terminator according to the flag behavior.
177+
*/
178+
fun isLineTerminator(c: Char) = if (unixLines) {
179+
c == '\n'
180+
} else {
181+
c == '\n' || c == '\r' || c == '\u0085' || c == '\u2028' || c == '\u2029'
182+
}
183+
145184
/**
146185
* Checks if the provided character has a case variant according to the flag behavior, checking both caseInsensitive
147186
* and unicodeCase flag values.

‎core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package org.evomaster.core.parser
22

33
import org.evomaster.core.search.gene.regex.RegexGene
4+
import org.evomaster.core.utils.RegexFlags
45
import org.junit.jupiter.api.Test
56
import org.junit.jupiter.api.assertThrows
67

@@ -399,4 +400,26 @@ class GeneRegexJavaVisitorTest : GeneRegexEcma262VisitorTest() {
399400
assertThrows<IllegalStateException> { checkSameAsJava("a([b&&c])d") }
400401
assertThrows<IllegalStateException> { checkSameAsJava("abc|\\k<name>") }
401402
}
403+
404+
@Test
405+
fun testCommentsFlag(){
406+
val commentsOn = RegexFlags(comments=true)
407+
checkSameAsJava("a b c", commentsOn)
408+
checkSameAsJava("a b c #comment\n after comment", commentsOn)
409+
checkSameAsJava("[also within char classes#comments too\n]", commentsOn)
410+
checkSameAsJava("a#comment\nb#noNewLine", commentsOn)
411+
checkSameAsJava("a#c1\n#c2\nb", commentsOn)
412+
checkSameAsJava("(a|b|#comment\nc)", commentsOn)
413+
checkSameAsJava("(?-x)( #notAComment)")
414+
checkSameAsJava("(?-x)( #notAComment)", commentsOn)
415+
checkCanSample("(?x)(a|b|#comment\nc)", listOf("a", "b", "c"), 100)
416+
checkSameAsJava("a\\ b +", commentsOn)
417+
checkSameAsJava("\\#a{1,3 #comment\n} ", commentsOn)
418+
checkSameAsJava(" ", commentsOn)
419+
checkCanSample("(?x)a|#comment", listOf("a", ""), 100)
420+
checkSameAsJava("a(?x:b c(?-x: d )e f)g")
421+
checkSameAsJava("\\Q#not a comment\\E", commentsOn)
422+
checkSameAsJava("a b(?-x: c d(?x: e f)g h)i j", commentsOn)
423+
checkSameAsJava("a b(?-x: c d(?x: e f (?-x) #no (?x: a b))g h)i j", commentsOn)
424+
}
402425
}

0 commit comments

Comments
 (0)