Java regex simple lookaheads

core/src/main/antlr4/org/evomaster/core/parser/RegexJavaLexer.g4

@@ -76,6 +76,7 @@ CharacterClassEscape
   : SLASH [dDsSwWvVhH]
  ;
 
+EQUAL                      : '=';
 CARET                      : '^';
 DOLLAR                     : '$';
 SLASH                      : '\\';
@@ -96,7 +97,7 @@ COLON                      : ':';
 
 BaseChar
  // practically all chars but the ones used for control and digits
- : ~[0-9:,^$\\.*+?()[\]{}|-]
+ : ~[0-9:,^$\\.*+?()[\]{}|=-]
  ;
 
 fragment OctalEscapeSequence

core/src/main/antlr4/org/evomaster/core/parser/RegexJavaParser.g4

@@ -52,7 +52,7 @@ assertion
  //TODO
 //// | '\\' 'b'
 //// | '\\' 'B'
-//// | '(' '?' '=' disjunction ')'
+ | PAREN_open QUESTION EQUAL disjunction PAREN_close
 //// | '(' '?' '!' disjunction ')'
  ;
 
@@ -116,7 +116,7 @@ patternCharacter
  // These are also allowed as literals when no matching pair exists
  | BRACE_close
  | BRACKET_close
- | COLON
+ | COLON | EQUAL
  | DOUBLE_AMPERSAND // char class intersection not supported by default in JS, only supported if "v" flag is turned on.
  ;
 
@@ -168,7 +168,7 @@ classAtomNoDash
  | DecimalDigit
  | COMMA | CARET | DOLLAR | DOT | STAR | PLUS | QUESTION
  | PAREN_open | PAREN_close | BRACKET_open | BRACE_open | BRACE_close | OR
- | COLON
+ | COLON | EQUAL
  // should be interpreted literally:
  // As they are lexer tokens, these character sequences are captured as such. In particular these require some extra
  // steps to interpret them correctly given the context.

core/src/main/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitor.kt

@@ -11,7 +11,7 @@ private const val EOF_TOKEN = "<EOF>"
 /**
  * Created by arcuri82 on 11-Sep-19.
  */
-class GeneRegexJavaVisitor(externalRegexFlags: RegexFlags = RegexFlags()) : RegexJavaParserBaseVisitor<VisitResult>(){
+class GeneRegexJavaVisitor(val sourceRegex: String, val externalRegexFlags: RegexFlags = RegexFlags()) : RegexJavaParserBaseVisitor<VisitResult>(){
 
     private val hexEscapePrefixes = setOf('x', 'u')
 
@@ -60,6 +60,8 @@ class GeneRegexJavaVisitor(externalRegexFlags: RegexFlags = RegexFlags()) : Rege
      */
     private var currentFlags = externalRegexFlags
 
+    private var hasAssertions = false
+
     /**
      * Builds DisjunctionListRxGenes from a disjunction context, returns null if disjunction is unsatisfiable.
      */
@@ -87,6 +89,23 @@ class GeneRegexJavaVisitor(externalRegexFlags: RegexFlags = RegexFlags()) : Rege
         return disjList
     }
 
+    /**
+     * Walks up [ctx]'s ancestry towards the top-level pattern, searching for one of
+     * the currently-unsupported ways an assertion's ancestry can appear (nested).
+     * Returns true if it reaches the top-level pattern without hitting either, false otherwise.
+     */
+    private fun isAssertionNested(ctx: RegexJavaParser.AssertionContext): Boolean {
+        var current = ctx.parent
+        while (current != null && current !is RegexJavaParser.PatternContext) {
+            if (current is RegexJavaParser.AssertionContext
+                || (current is RegexJavaParser.AtomContext && current.disjunction() != null)) {
+                return true
+            }
+            current = current.parent
+        }
+        return false
+    }
+
     override fun visitPattern(ctx: RegexJavaParser.PatternContext): VisitResult {
 
         val res = ctx.disjunction().accept(this)
@@ -107,8 +126,10 @@ class GeneRegexJavaVisitor(externalRegexFlags: RegexFlags = RegexFlags()) : Rege
         val gene = RegexGene(
             "regex",
             disjList,
-            text.substring(0, text.length - EOF_TOKEN.length),
-            RegexType.JVM
+            sourceRegex,
+            RegexType.JVM,
+            externalRegexFlags = externalRegexFlags,
+            hasAssertions = hasAssertions
         )
 
         return VisitResult(gene)
@@ -231,7 +252,18 @@ class GeneRegexJavaVisitor(externalRegexFlags: RegexFlags = RegexFlags()) : Rege
         val res = VisitResult()
 
         if(ctx.assertion() != null){
-            res.data = ctx.assertion().text
+            val assertionCtx = ctx.assertion()
+            if (assertionCtx.CARET() != null || assertionCtx.DOLLAR() != null) {
+                res.data = ctx.assertion().text
+            } else {
+                require(!isAssertionNested(ctx.assertion())){
+                    "Nested assertions are not currently supported."
+                }
+                val innerDisjList = buildDisjunctionList(assertionCtx.disjunction())
+                val assertionGene = AssertionRxGene(innerDisjList)
+                hasAssertions = true
+                res.genes.add(assertionGene)
+            }
             return res
         }
 

core/src/main/kotlin/org/evomaster/core/parser/RegexHandler.kt

@@ -44,7 +44,7 @@ object RegexHandler {
 
         val pattern = parser.pattern()
 
-        val res = GeneRegexJavaVisitor(externalRegexFlags).visit(pattern)
+        val res = GeneRegexJavaVisitor(regex, externalRegexFlags).visit(pattern)
 
         val gene= res.genes.first() as RegexGene
         cacheJVM[key] = gene.copy() as RegexGene

core/src/main/kotlin/org/evomaster/core/search/gene/regex/AnyCharacterRxGene.kt

@@ -131,4 +131,38 @@ class AnyCharacterRxGene(
         return true
     }
 
+    /**
+     * 1 if [value]'s first character is within [validRanges] (i.e. one `.` itself could
+     * render, given the current flags), else 0.
+     * @see [RxAbsorbable.absorbableCount]
+     */
+    override fun absorbableCount(value: String): Int {
+        if (value.isEmpty()) {
+            return 0
+        }
+        return if (validRanges.contains(value[0])) {
+            1
+        } else {
+            0
+        }
+    }
+
+    /** Always false: `.` always renders exactly one character.
+     * @see [RxAbsorbable.canBeZeroWidth]
+     */
+    override val canBeZeroWidth: Boolean = false
+
+    /**
+     * Forces [value]'s first character onto this gene if `.` could render it; mirrors
+     * [absorbableCount], so it never mutates when [absorbableCount] would return 0.
+     * @see [RxAbsorbable.tryForce]
+     */
+    override fun tryForce(value: String): Int {
+        require(value.isNotEmpty())
+        val n = absorbableCount(value)
+        if (n == 1) {
+            this.value = value[0]
+        }
+        return n
+    }
 }

core/src/main/kotlin/org/evomaster/core/search/gene/regex/AssertionRxGene.kt

@@ -0,0 +1,126 @@
+package org.evomaster.core.search.gene.regex
+
+import org.evomaster.core.output.OutputFormat
+import org.evomaster.core.search.gene.Gene
+import org.evomaster.core.search.gene.root.SimpleGene
+import org.evomaster.core.search.gene.utils.GeneUtils
+import org.evomaster.core.search.service.AdaptiveParameterControl
+import org.evomaster.core.search.service.Randomness
+import org.evomaster.core.search.service.mutator.MutationWeightControl
+import org.evomaster.core.search.service.mutator.genemutation.AdditionalGeneMutationInfo
+import org.evomaster.core.search.service.mutator.genemutation.SubsetGeneMutationSelectionStrategy
+
+/**
+ * Represents a zero-width assertion in the regex gene tree.
+ *
+ * This gene is placed in the tree at the position where the assertion appeared in the
+ * source regex. It produces no characters ([getValueAsPrintableString] always returns
+ * "").
+ *
+ * Repair is triggered from [DisjunctionRxGene.attemptAssertionRepair], invoked by
+ * [RegexGene.randomize] after the disjunction's own sampled value is checked against
+ * the source pattern and found not to match.
+ */
+class AssertionRxGene(
+    /**
+     * The assertion's inner disjunction gene, can be null as the disjunction can be unsatisfiable,
+     * in that case [innerGene] is null.
+     */
+    val innerGene: DisjunctionListRxGene?
+) : RxTerm, SimpleGene("assertion") {
+
+    override fun checkForLocallyValidIgnoringChildren(): Boolean = true
+
+    override fun isUnsatisfiable(): Boolean = innerGene == null
+
+    override fun isMutable(): Boolean = innerGene?.isMutable() ?: false
+
+    override fun copyContent(): Gene {
+        val copy = AssertionRxGene(innerGene?.copy() as? DisjunctionListRxGene)
+        copy.name = this.name
+        return copy
+    }
+
+    override fun randomize(randomness: Randomness, tryToForceNewValue: Boolean) {
+        innerGene?.randomize(randomness, tryToForceNewValue)
+    }
+
+    override fun setValueWithRawString(value: String) {
+        innerGene?.setFromStringValue(value)
+    }
+
+    override fun shallowMutate(
+        randomness: Randomness,
+        apc: AdaptiveParameterControl,
+        mwc: MutationWeightControl,
+        selectionStrategy: SubsetGeneMutationSelectionStrategy,
+        enableAdaptiveGeneMutation: Boolean,
+        additionalGeneMutationInfo: AdditionalGeneMutationInfo?
+    ): Boolean {
+        return false
+    }
+
+    /**
+     * Zero-width: never contributes characters to the generated string.
+     */
+    override fun getValueAsPrintableString(
+        previousGenes: List<Gene>,
+        mode: GeneUtils.EscapeMode?,
+        targetFormat: OutputFormat?,
+        extraCheck: Boolean
+    ): String = ""
+
+    /**
+     * Returns the value currently sampled from [innerGene], or null if there is no
+     * inner gene to sample from.
+     */
+    fun sampledInnerValue(): String? {
+        if (innerGene == null) {
+            return null
+        }
+        return innerGene.getValueAsPrintableString(targetFormat = null)
+    }
+
+    override fun containsSameValueAs(other: Gene): Boolean {
+        if (other !is AssertionRxGene) {
+            return false
+        }
+        return sampledInnerValue() == other.sampledInnerValue()
+    }
+
+    override fun unsafeCopyValueFrom(other: Gene): Boolean {
+        if (other !is AssertionRxGene) {
+            return false
+        }
+        return if (innerGene != null && other.innerGene != null) {
+            innerGene.unsafeCopyValueFrom(other.innerGene)
+        } else {
+            innerGene == null && other.innerGene == null
+        }
+    }
+
+    /**
+     * Always true: an assertion never renders characters ([getValueAsPrintableString] is
+     * always ""), so it can always collapse to zero width.
+     * @see [RxAbsorbable.canBeZeroWidth]
+     */
+    override val canBeZeroWidth: Boolean = true
+
+    /**
+     * Always 0: an assertion never absorbs candidate text into itself, it only supplies a
+     * candidate for its siblings to absorb, via [sampledInnerValue].
+     * @see [RxAbsorbable.tryForce]
+     */
+    override fun tryForce(value: String): Int {
+        require(value.isNotEmpty())
+        return 0
+    }
+
+    /**
+     * No-op: already always zero-width, so there's nothing to force.
+     * @see [RxAbsorbable.forceZeroWidth]
+     */
+    override fun forceZeroWidth() {
+        // already always zero-width, nothing to do
+    }
+}

core/src/main/kotlin/org/evomaster/core/search/gene/regex/BackReferenceRxGene.kt

@@ -81,4 +81,20 @@ class BackReferenceRxGene(
         // nothing to copy, as the value comes from the capture group
         return containsSameValueAs(other)
     }
+
+    /**
+     * Returns false as we do not want backreferences to mutate a previous group.
+     * @see [RxAbsorbable.canBeZeroWidth]
+     */
+    override val canBeZeroWidth: Boolean = false
+
+    /**
+     * Always 0: a backreference's value is derived entirely from a previous [captureGroup],
+     * so unlike an ordinary leaf it can not be forced to absorb arbitrary candidate text.
+     * @see [RxAbsorbable.tryForce]
+     */
+    override fun tryForce(value: String): Int {
+        require(value.isNotEmpty())
+        return 0
+    }
 }

core/src/main/kotlin/org/evomaster/core/search/gene/regex/CharacterClassEscapeRxGene.kt

@@ -115,6 +115,13 @@ class CharacterClassEscapeRxGene(
     companion object{
         private val log = LoggerFactory.getLogger(CharacterRangeRxGene::class.java)
 
+        private fun Char.swapCase(): Char =
+            if (this.isUpperCase()) {
+                this.lowercaseChar()
+            } else {
+                this.uppercaseChar()
+            }
+
         private val digitSet = listOf(CharacterRange('0', '9'))
         private val asciiLetterSet = listOf(CharacterRange(FIRST_LOWER_CASE_CHAR, LAST_LOWER_CASE_CHAR),
             CharacterRange(FIRST_UPPER_CASE_CHAR, LAST_UPPER_CASE_CHAR))
@@ -352,4 +359,49 @@ class CharacterClassEscapeRxGene(
         return false
     }
 
+    /**
+     * 1 if [value]'s first character (or its case-swapped counterpart, when caseable) is
+     * within [multiCharRange], else 0.
+     * @see [RxAbsorbable.absorbableCount]
+     */
+    override fun absorbableCount(value: String): Int {
+        if (value.isEmpty()) {
+            return 0
+        }
+        val c = value[0]
+        if (multiCharRange.contains(c)) {
+            return 1
+        }
+        if (flags.isCaseable(c) && multiCharRange.contains(c.swapCase())) {
+            return 1
+        }
+        return 0
+    }
+
+    /**
+     * Always false: a character-class escape always renders exactly one character.
+     * @see [RxAbsorbable.canBeZeroWidth]
+     */
+    override val canBeZeroWidth: Boolean = false
+
+    /**
+     * Forces [value]'s first character onto this gene, swapping case if that's what
+     * matched; mirrors [absorbableCount].
+     * @see [RxAbsorbable.tryForce]
+     */
+    override fun tryForce(value: String): Int {
+        require(value.isNotEmpty())
+        val n = absorbableCount(value)
+        require(n == 0 || n == 1)
+        if (n == 1) {
+            val c = value[0]
+            this.value = (if (multiCharRange.contains(c)) {
+                    c
+                } else {
+                    c.swapCase()
+                }).toString()
+            this.useUpperCase = c.isUpperCase()
+        }
+        return n
+    }
 }