refactor: implement parse() in-place on the real combinators, drop grammar2

Replaces the standalone grammar2 PoC with the same list-of-successes idea grown
directly on the existing combinators, per review feedback: the parallel package
had a jarringly different surface and implied a rewrite of all 225 grammars.

Instead, add one new method — Combinator.parse(value, offset): Sequence<Parse> —
ALONGSIDE the existing SyntacticMatch/SemanticMatch, implemented on each of the 12
combinators next to its existing match logic. The caller (GrammarOptionValue) is
untouched and still uses the old methods; nothing in the 225 grammar definitions
changes. parse() can therefore be validated against the REAL production grammars
before any migration decision.

- Parse.kt: ParsedToken / Parse result types + validate() free function. One
  lenient pass folds the strict "semantic" check into a per-token `valid` flag, so
  it answers both syntactic (any full parse?) and semantic (a full parse with only
  valid tokens?) without two traversals.
- Each combinator returns every way it can match, lazily; Alt offers all options
  (ordering no longer matters), ZeroOrMore/OneOrMore/Repeat offer every count.
- ParseTest runs validate() against the actual ConfigParseAddressFamiliesOptionValue
  grammar and the real IPV6_ADDR combinator (15+ hand-ordered alternatives), an
  integer-range grammar, and the greedy Seq(ZeroOrMore("a"),"a") case — which it
  shows the old SemanticMatch still fails while parse() succeeds.

Known limitation pinned in a test: SyntaxError `furthest` is best-effort and
collapses to 0 when a trailing EOF() discards partial progress; precise
localization needs the frontier/expected-set layer (the same machinery as
completion, #343), deliberately out of scope here.

Refs #467 #345 #343 #342

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: Steve Ramage

src/main/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/semanticdata/optionvalues/grammar/AlternativeCombinator.kt

@@ -38,6 +38,10 @@ open class AlternativeCombinator(vararg val tokens: Combinator) : Combinator {
     return match(value, offset, Combinator::SemanticMatch)
   }
 
+  override fun parse(value: String, offset: Int): Sequence<Parse> =
+    // Offer every alternative's matches, so the order of options no longer affects correctness.
+    tokens.asSequence().flatMap { it.parse(value, offset) }
+
   override fun toString(): String = toStringIndented(0)
 
   override fun toStringIndented(indent: Int): String {

src/main/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/semanticdata/optionvalues/grammar/Combinator.kt

@@ -32,5 +32,16 @@ interface Combinator {
    */
   fun SemanticMatch(value : String, offset: Int): MatchResult
 
+  /**
+   * List-of-successes matcher (#467). Returns EVERY way this combinator can consume [value]
+   * starting at [offset], lazily; an empty sequence means no match.
+   *
+   * This lives alongside Syntactic/SemanticMatch and is a single lenient pass: each [ParsedToken]
+   * carries a `valid` flag for the strict (semantic) check. Because every alternative is offered
+   * rather than the first greedy one committed to, matching is complete — e.g.
+   * Seq(ZeroOrMore("a"), "a") on "aa" matches, because ZeroOrMore offers the shorter match too.
+   */
+  fun parse(value: String, offset: Int): Sequence<Parse>
+
   fun toStringIndented(indent: Int): String
 }

src/main/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/semanticdata/optionvalues/grammar/EOF.kt

@@ -17,6 +17,9 @@ class EOF : Combinator {
     }
   }
 
+  override fun parse(value: String, offset: Int): Sequence<Parse> =
+    if (offset == value.length) sequenceOf(Parse(offset, emptyList())) else emptySequence()
+
   override fun toStringIndented(indent: Int): String {
     return "EOF"
   }

src/main/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/semanticdata/optionvalues/grammar/FlexibleLiteralChoiceTerminal.kt

@@ -91,6 +91,15 @@ class FlexibleLiteralChoiceTerminal(vararg val choices: String) : TerminalCombin
     return NoMatch.copy(longestMatch = offset)
   }
 
+  override fun parse(value: String, offset: Int): Sequence<Parse> {
+    // Lenient shape match (so a wrong token like AF_BOGUS still matches and can be highlighted),
+    // valid only if the matched text is one of the exact choices.
+    val m = syntaticMatch.matchAt(value, offset) ?: return emptySequence()
+    val text = m.value
+    val valid = choices.any { it == text }
+    return sequenceOf(Parse(offset + text.length, listOf(ParsedToken(offset, offset + text.length, text, this, valid))))
+  }
+
   override fun toString(): String {
     return if (choices.size == 1) {
       "Literal(\"${choices[0]}\")"

src/main/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/semanticdata/optionvalues/grammar/IntegerTerminal.kt

@@ -29,6 +29,14 @@ class IntegerTerminal(private val minInclusive: Long,private val maxExclusive: L
     }
   }
 
+  override fun parse(value: String, offset: Int): Sequence<Parse> {
+    val m = intRegex.matchAt(value, offset) ?: return emptySequence()
+    val text = m.value
+    // Lenient: any integer matches (so we can locate it); valid only if it is within range.
+    val valid = text.toLongOrNull()?.let { it >= minInclusive && it < maxExclusive } ?: false
+    return sequenceOf(Parse(offset + text.length, listOf(ParsedToken(offset, offset + text.length, text, this, valid))))
+  }
+
   override fun toString(): String {
     return "Int($minInclusive,$maxExclusive)"
   }

src/main/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/semanticdata/optionvalues/grammar/LiteralChoiceTerminal.kt

@@ -24,6 +24,12 @@ class  LiteralChoiceTerminal(vararg var choices: String) : TerminalCombinator {
     return match(value, offset)
   }
 
+  override fun parse(value: String, offset: Int): Sequence<Parse> =
+    // Offer every choice that matches here (e.g. both ":" and "::"); each is always strictly valid.
+    choices.asSequence()
+      .filter { value.startsWith(it, offset) }
+      .map { Parse(offset + it.length, listOf(ParsedToken(offset, offset + it.length, it, this, valid = true))) }
+
   override fun toString(): String {
     return if (choices.size == 1) {
       "Literal(\"${choices[0]}\")"

src/main/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/semanticdata/optionvalues/grammar/OneOrMore.kt

@@ -40,6 +40,17 @@ class OneOrMore(val combinator : Combinator) : Combinator {
     return match(value, offset, combinator::SemanticMatch)
   }
 
+  override fun parse(value: String, offset: Int): Sequence<Parse> {
+    // Same as ZeroOrMore, but the first repetition is mandatory (and must make progress).
+    fun extend(from: Parse): Sequence<Parse> = sequence {
+      yield(from)
+      for (step in combinator.parse(value, from.end)) {
+        if (step.end > from.end) yieldAll(extend(Parse(step.end, from.tokens + step.tokens)))
+      }
+    }
+    return combinator.parse(value, offset).filter { it.end > offset }.flatMap { extend(it) }
+  }
+
   override fun toString(): String = toStringIndented(0)
 
   override fun toStringIndented(indent: Int): String {

src/main/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/semanticdata/optionvalues/grammar/Parse.kt

@@ -0,0 +1,60 @@
+package net.sjrx.intellij.plugins.systemdunitfiles.semanticdata.optionvalues.grammar
+
+/*
+ * List-of-successes matcher (GitHub #467, step 2).
+ *
+ * These types support a second matching method, `Combinator.parse()`, that lives ALONGSIDE the
+ * existing SyntacticMatch / SemanticMatch on every combinator. Nothing here is wired into
+ * GrammarOptionValue yet — the goal is to flesh the approach out on the real combinators and
+ * validate it against the real grammars in tests before deciding to migrate the caller.
+ *
+ * Where the existing engine returns ONE greedy result and runs two near-identical passes, parse()
+ * returns EVERY way a combinator can match (lazily), and folds the strict "semantic" check into a
+ * `valid` flag on each token. So one lenient pass answers both questions, and greedy traps like
+ * Seq(ZeroOrMore("a"), "a") on "aa" resolve themselves (see Combinator.parse docs).
+ */
+
+/** A single terminal token, with the strict-validity verdict (the old "semantic" check) folded in. */
+data class ParsedToken(
+  val start: Int,
+  val end: Int,
+  val text: String,
+  val terminal: TerminalCombinator,
+  val valid: Boolean,
+)
+
+/** One way a combinator consumed input from some offset: it ended at [end], producing [tokens]. */
+data class Parse(val end: Int, val tokens: List<ParsedToken>)
+
+/** The outcome of validating a whole value against a grammar via parse(). */
+sealed interface ParseOutcome {
+  /** Some path consumed the whole value with every token strictly valid. */
+  object Valid : ParseOutcome
+
+  /** A path consumed the whole value, but a token is not strictly valid (well-formed but wrong). */
+  data class SemanticError(val badToken: ParsedToken) : ParseOutcome
+
+  /** No path consumed the whole value. [furthest] is how far any path got (for error localization). */
+  data class SyntaxError(val furthest: Int) : ParseOutcome
+}
+
+/** Every way [this] grammar can consume the entire [value]. */
+fun Combinator.fullParses(value: String): Sequence<Parse> =
+  parse(value, 0).filter { it.end == value.length }
+
+/**
+ * One lenient parse answers both questions the old two passes did:
+ *  - syntactic ("could be this, color it"): did any path consume the whole value?
+ *  - semantic ("actually valid"):           did any such path use only valid tokens?
+ */
+fun Combinator.validate(value: String): ParseOutcome {
+  var firstBad: ParsedToken? = null
+  for (p in fullParses(value)) {
+    val bad = p.tokens.firstOrNull { !it.valid }
+    if (bad == null) return ParseOutcome.Valid // short-circuit on the first fully-valid full parse
+    if (firstBad == null) firstBad = bad
+  }
+  if (firstBad != null) return ParseOutcome.SemanticError(firstBad)
+  val furthest = parse(value, 0).maxOfOrNull { it.end } ?: 0
+  return ParseOutcome.SyntaxError(furthest)
+}

src/main/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/semanticdata/optionvalues/grammar/RegexTerminal.kt

@@ -18,6 +18,14 @@ class RegexTerminal(syntaticMatchStr : String, semanticMatchStr: String ) : Term
     return MatchResult(listOf(matchResult.value), offset + matchResult.value.length, listOf(this), offset + matchResult.value.length)
   }
 
+  override fun parse(value: String, offset: Int): Sequence<Parse> {
+    // The syntactic regex gives the lenient span; valid iff the semantic regex matches that same span.
+    val syn = syntaticMatch.matchAt(value, offset) ?: return emptySequence()
+    val text = syn.value
+    val valid = semanticMatch.matchAt(value, offset)?.value == text
+    return sequenceOf(Parse(offset + text.length, listOf(ParsedToken(offset, offset + text.length, text, this, valid))))
+  }
+
   override fun toString(): String {
     return "Regex(\"${semanticMatch.pattern}\")"
   }

src/main/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/semanticdata/optionvalues/grammar/Repeat.kt

@@ -62,6 +62,20 @@ class Repeat(val combinator : Combinator, val minInclusive: Int, val maxExclusiv
     return match(value, offset, combinator::SemanticMatch)
   }
 
+  override fun parse(value: String, offset: Int): Sequence<Parse> {
+    // Offer every repetition count in [minInclusive, maxExclusive] (maxExclusive is the cap on the
+    // count, mirroring the existing match() loop). Yield only once enough repetitions have happened.
+    fun extend(from: Parse, count: Int): Sequence<Parse> = sequence {
+      if (count >= minInclusive) yield(from)
+      if (count < maxExclusive) {
+        for (step in combinator.parse(value, from.end)) {
+          if (step.end > from.end) yieldAll(extend(Parse(step.end, from.tokens + step.tokens), count + 1))
+        }
+      }
+    }
+    return extend(Parse(offset, emptyList()), 0)
+  }
+
   override fun toString(): String = toStringIndented(0)
 
   override fun toStringIndented(indent: Int): String {