feat: grammar-based value completion (#494)

* feat: grammar-based value completion behind the experimental flag

The frontier built for error localization answers exactly the question
completion needs — "what was expected at this offset?" is "what could come next
here?". Adds Combinator.nextTokenChoices(prefix), which reads the Stuck values at
the end of the typed prefix and collects the enumerable choices expected there
(literal / flexible-literal terminals; numbers/regexes/whitespace contribute
nothing to suggest).

Wires it into UnitFileValueCompletionContributor, gated behind
ExperimentalSettings.useGrammarParseEngine AND validator is GrammarOptionValue:
with the flag off the existing completion path is untouched. Two behaviours:
  - completing a partial token (sets the prefix matcher so "AF_IN" -> AF_INET/6)
  - chaining through forced separators on accept (e.g. accepting "home" inserts
    "home=" and re-opens completion, never auto-inserting a content token)

Bundles the stacked PRs #477 (completion) and #478 (forced-separator chaining),
re-cut against current 242.x. New behaviour, so flag-gated; correctness of
existing validators is unchanged. Tests enable the flag and reset it in tearDown
to avoid leaking into the shared light-test project.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

* test: cover the flag-off gating and non-enumerable completion cases

The completion bundle tested the flag-on behaviours but not two things worth
guaranteeing: that grammar completion stays suppressed when the experimental
engine is off (the gating contract), and that a non-enumerable next token
(number/regex) offers nothing rather than crashing or suggesting junk.

- GrammarValueCompletionTest.testFlagOffOffersNoGrammarCompletions: flag off, the
  AF_IN partial yields no AF_* names (the flag-on path's suggestions are absent).
- NextTokenChoicesTest.testNonEnumerableNextTokenOffersNothing: an IntegerTerminal
  port grammar returns an empty choice set.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Steve Ramage <gitcommits@sjrx.net>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: 3 people

src/main/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/completion/UnitFileValueCompletionContributor.kt

@@ -1,7 +1,10 @@
 package net.sjrx.intellij.plugins.systemdunitfiles.completion
 
+import com.intellij.codeInsight.AutoPopupController
 import com.intellij.codeInsight.completion.*
+import com.intellij.codeInsight.lookup.LookupElement
 import com.intellij.codeInsight.lookup.LookupElementBuilder
+import com.intellij.openapi.progress.ProgressManager
 import com.intellij.patterns.PlatformPatterns
 import com.intellij.psi.util.PsiTreeUtil
 import com.intellij.util.ProcessingContext
@@ -11,6 +14,10 @@ import net.sjrx.intellij.plugins.systemdunitfiles.psi.UnitFileProperty
 import net.sjrx.intellij.plugins.systemdunitfiles.psi.UnitFileSectionGroups
 import net.sjrx.intellij.plugins.systemdunitfiles.semanticdata.SemanticDataRepository
 import net.sjrx.intellij.plugins.systemdunitfiles.semanticdata.fileClass
+import net.sjrx.intellij.plugins.systemdunitfiles.semanticdata.optionvalues.grammar.Combinator
+import net.sjrx.intellij.plugins.systemdunitfiles.semanticdata.optionvalues.grammar.GrammarOptionValue
+import net.sjrx.intellij.plugins.systemdunitfiles.semanticdata.optionvalues.grammar.nextTokenChoices
+import net.sjrx.intellij.plugins.systemdunitfiles.settings.ExperimentalSettings
 import java.util.function.Supplier
 import java.util.stream.Collectors
 
@@ -48,6 +55,14 @@ class UnitFileValueCompletionContributor : CompletionContributor() {
                val fileClass = section.containingFile.fileClass()
 
                val validator = sdr.getOptionValidator(fileClass, sectionName, keyName)
+
+               if (ExperimentalSettings.getInstance(property.project).state.useGrammarParseEngine &&
+                 validator is GrammarOptionValue
+               ) {
+                 addGrammarCompletions(parameters, validator, property, resultSet)
+                 return
+               }
+
                resultSet.addAllElements(
                  validator.getAutoCompleteOptions(property.project)
                    .stream()
@@ -59,4 +74,79 @@ class UnitFileValueCompletionContributor : CompletionContributor() {
            }
     )
   }
+
+  /**
+   * Experimental grammar-based completion (#467 / #343): suggest the literal/choice tokens the
+   * grammar could accept at the caret. We read the value text up to the caret, find the tightest
+   * split where the grammar expects an enumerable token whose choices match what's already typed,
+   * and set that as the prefix so the platform filters correctly (otherwise a partial token like
+   * "~AF_IN" would be the prefix and match nothing).
+   */
+  private fun addGrammarCompletions(
+    parameters: CompletionParameters,
+    validator: GrammarOptionValue,
+    property: UnitFileProperty,
+    resultSet: CompletionResultSet,
+  ) {
+    val valueStart = property.valueNode?.psi?.textRange?.startOffset ?: return
+    val caret = parameters.offset
+    if (caret < valueStart) return
+    val pre = parameters.position.containingFile.text.substring(valueStart, caret)
+    val combinator = validator.combinator
+
+    // Case 1 — completing a partial token. Find the token start: the longest non-empty trailing
+    // word for which the grammar expects an enumerable choice that STRICTLY extends it (i.e. there's
+    // more to type). This beats just advancing, so "h" completes to "home" rather than offering "="
+    // (the lenient terminal would otherwise treat "h" as a finished identifier).
+    for (split in 0 until pre.length) {
+      ProgressManager.checkCanceled()
+      val word = pre.substring(split)
+      val choices = combinator.nextTokenChoices(pre.substring(0, split))
+      if (choices.any { it.length > word.length && it.startsWith(word) }) {
+        resultSet.withPrefixMatcher(word).addAllElements(lookupElements(choices, combinator))
+        return
+      }
+    }
+
+    // Case 2 — at a fresh token boundary (e.g. empty value, or after a complete token like "~" or
+    // "root="). Offer whatever can come next, matched against an empty prefix.
+    ProgressManager.checkCanceled()
+    val choices = combinator.nextTokenChoices(pre)
+    if (choices.isNotEmpty()) {
+      resultSet.withPrefixMatcher("").addAllElements(lookupElements(choices, combinator))
+    }
+  }
+
+  private fun lookupElements(choices: Set<String>, combinator: Combinator): List<LookupElement> {
+    val handler = chainingInsertHandler(combinator)
+    return choices.map { LookupElementBuilder.create(it).withInsertHandler(handler) }
+  }
+
+  /**
+   * After a choice is accepted, walk the grammar forward: while the only thing it can accept next is
+   * a single forced separator (punctuation, e.g. "=" after a partition designator), insert it
+   * automatically, then re-open completion. So accepting "home" yields "home=" with the policy-flag
+   * popup, rather than stopping on a separator the user must type by hand.
+   */
+  private fun chainingInsertHandler(combinator: Combinator) = InsertHandler<LookupElement> { context, _ ->
+    context.commitDocument()
+    val element = context.file.findElementAt(maxOf(0, context.tailOffset - 1)) ?: return@InsertHandler
+    val property = PsiTreeUtil.getParentOfType(element, UnitFileProperty::class.java) ?: return@InsertHandler
+    val valueStart = property.valueNode?.psi?.textRange?.startOffset ?: return@InsertHandler
+    val document = context.document
+
+    var caret = context.tailOffset
+    var guard = 0
+    while (guard++ < 8 && caret in valueStart..document.textLength) {
+      val pre = document.charsSequence.subSequence(valueStart, caret).toString()
+      val separator = combinator.nextTokenChoices(pre).singleOrNull() ?: break
+      // Only auto-insert a forced punctuation separator; never a content token the user should pick.
+      if (separator.isEmpty() || separator.any { it.isLetterOrDigit() }) break
+      document.insertString(caret, separator)
+      caret += separator.length
+    }
+    context.commitDocument()
+    context.editor.caretModel.moveToOffset(caret)
+    AutoPopupController.getInstance(context.project).scheduleAutoPopup(context.editor)
+  }
 }

src/main/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/semanticdata/optionvalues/grammar/Completion.kt

@@ -0,0 +1,34 @@
+package net.sjrx.intellij.plugins.systemdunitfiles.semanticdata.optionvalues.grammar
+
+/*
+ * Grammar-based completion support (GitHub #467 / #343).
+ *
+ * The frontier we built for error localization is exactly what completion needs: "what was expected
+ * at this offset?" is the same question as "what could come next here?". [nextTokenChoices] answers
+ * it for the position at the end of [prefix].
+ *
+ * It reads the FIRST set at the end of [prefix] from the parse frontier — the Stuck values whose
+ * offset is the end of [prefix] — and collects the enumerable choices of the terminals expected
+ * there (literal choices and flexible-literal choices). Non-enumerable terminals (numbers, regexes,
+ * whitespace, EOF) contribute nothing to suggest.
+ *
+ * Pure Kotlin, no IntelliJ types. The matcher is exhaustive, so [maxSteps] bounds the work; callers
+ * running on a UI/highlighting thread should also poll cancellation between calls.
+ */
+fun Combinator.nextTokenChoices(prefix: String, maxSteps: Int = 100_000): Set<String> {
+  val choices = linkedSetOf<String>()
+  var steps = 0
+  for (step in parse(prefix, 0)) {
+    if (++steps > maxSteps) break
+    if (step is Stuck && step.offset == prefix.length) {
+      for (matcher in step.expected) {
+        when (matcher) {
+          is LiteralChoiceTerminal -> choices += matcher.choices
+          is FlexibleLiteralChoiceTerminal -> choices += matcher.choices
+          else -> {} // not enumerable — nothing concrete to suggest
+        }
+      }
+    }
+  }
+  return choices
+}

src/test/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/completion/GrammarValueCompletionTest.kt

@@ -0,0 +1,52 @@
+package net.sjrx.intellij.plugins.systemdunitfiles.completion
+
+import net.sjrx.intellij.plugins.systemdunitfiles.AbstractUnitFileTest
+import net.sjrx.intellij.plugins.systemdunitfiles.settings.ExperimentalSettings
+import org.junit.Test
+
+/**
+ * End-to-end grammar-based completion (#467 / #343), behind the experimental flag.
+ */
+class GrammarValueCompletionTest : AbstractUnitFileTest() {
+
+  // The light-test project is shared across classes; don't leak the opt-in.
+  override fun tearDown() {
+    try {
+      ExperimentalSettings.getInstance(project).state.useGrammarParseEngine = false
+    } finally {
+      super.tearDown()
+    }
+  }
+
+  private fun enableNewEngine() {
+    ExperimentalSettings.getInstance(project).state.useGrammarParseEngine = true
+  }
+
+  @Test
+  fun testCompletesPartialAddressFamily() {
+    enableNewEngine()
+    setupFileInEditor("file.service", "[Service]\nRestrictAddressFamilies=AF_IN${COMPLETION_POSITION}")
+
+    val results = basicCompletionResultStrings
+    assertContainsElements(results, "AF_INET", "AF_INET6")
+  }
+
+  @Test
+  fun testCompletesSubsequentFamilyInList() {
+    enableNewEngine()
+    setupFileInEditor("file.service", "[Service]\nRestrictAddressFamilies=AF_UNIX AF_IN${COMPLETION_POSITION}")
+
+    val results = basicCompletionResultStrings
+    assertContainsElements(results, "AF_INET", "AF_INET6")
+  }
+
+  @Test
+  fun testFlagOffOffersNoGrammarCompletions() {
+    // Gating guarantee: with the experimental engine OFF (the default), grammar completion must not
+    // fire. A GrammarOptionValue has no legacy autocomplete options, so the same partial token that
+    // the flag-on path completes to AF_INET/AF_INET6 yields nothing grammar-derived here.
+    setupFileInEditor("file.service", "[Service]\nRestrictAddressFamilies=AF_IN${COMPLETION_POSITION}")
+
+    assertDoesntContain(basicCompletionResultStrings, "AF_INET", "AF_INET6")
+  }
+}

src/test/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/completion/ImagePolicyCompletionTest.kt

@@ -0,0 +1,58 @@
+package net.sjrx.intellij.plugins.systemdunitfiles.completion
+
+import net.sjrx.intellij.plugins.systemdunitfiles.AbstractUnitFileTest
+import net.sjrx.intellij.plugins.systemdunitfiles.settings.ExperimentalSettings
+import org.junit.Test
+
+/**
+ * Grammar completion for the more complex RootImagePolicy= grammar (#467 / #343), behind the flag.
+ * Exercises a fresh boundary (empty), completing a partial partition identifier, and the position
+ * after "partition=" where policy flags are expected.
+ */
+class ImagePolicyCompletionTest : AbstractUnitFileTest() {
+
+  override fun tearDown() {
+    try {
+      ExperimentalSettings.getInstance(project).state.useGrammarParseEngine = false
+    } finally {
+      super.tearDown()
+    }
+  }
+
+  private fun enableNewEngine() {
+    ExperimentalSettings.getInstance(project).state.useGrammarParseEngine = true
+  }
+
+  @Test
+  fun testEmptyValueOffersPartitionsAndDefault() {
+    enableNewEngine()
+    setupFileInEditor("file.service", "[Service]\nRootImagePolicy=${COMPLETION_POSITION}")
+    assertContainsElements(basicCompletionResultStrings, "root", "usr", "swap", "+")
+  }
+
+  @Test
+  fun testPartialPartitionIdentifierIsCompleted() {
+    // Regression for the bug: typing "r" used to offer "=" (the lenient terminal treated "r" as a
+    // finished identifier); it should complete the identifier instead.
+    enableNewEngine()
+    setupFileInEditor("file.service", "[Service]\nRootImagePolicy=r${COMPLETION_POSITION}")
+    assertContainsElements(basicCompletionResultStrings, "root", "root-verity", "root-verity-sig")
+  }
+
+  @Test
+  fun testAfterPartitionEqualsOffersPolicyFlags() {
+    enableNewEngine()
+    setupFileInEditor("file.service", "[Service]\nRootImagePolicy=root=${COMPLETION_POSITION}")
+    assertContainsElements(basicCompletionResultStrings, "verity", "signed", "encrypted")
+  }
+
+  @Test
+  fun testAcceptingPartitionChainsTheEqualsSeparator() {
+    // Accepting a partition designator auto-inserts the forced "=" (then re-opens completion),
+    // so the user goes straight to choosing a policy flag.
+    enableNewEngine()
+    setupFileInEditor("file.service", "[Service]\nRootImagePolicy=hom${COMPLETION_POSITION}")
+    myFixture.completeBasic() // single match "home" -> auto-inserted -> handler appends "="
+    myFixture.checkResult("[Service]\nRootImagePolicy=home=${COMPLETION_POSITION}")
+  }
+}

src/test/kotlin/net/sjrx/intellij/plugins/systemdunitfiles/semanticdata/optionvalues/grammar/NextTokenChoicesTest.kt

@@ -0,0 +1,42 @@
+package net.sjrx.intellij.plugins.systemdunitfiles.semanticdata.optionvalues.grammar
+
+import net.sjrx.intellij.plugins.systemdunitfiles.semanticdata.optionvalues.ai.ConfigParseAddressFamiliesOptionValue
+import org.junit.Assert.assertFalse
+import org.junit.Assert.assertTrue
+import org.junit.Test
+
+/** Unit tests for the grammar-completion FIRST-set computation (no IntelliJ fixture needed). */
+class NextTokenChoicesTest {
+
+  private val addressFamilies = ConfigParseAddressFamiliesOptionValue().combinator
+
+  @Test
+  fun testEmptyValueOffersStartTokens() {
+    val choices = addressFamilies.nextTokenChoices("")
+    assertTrue(choices.contains("none"))
+    assertTrue(choices.contains("~"))
+    assertTrue(choices.contains("AF_INET"))
+  }
+
+  @Test
+  fun testAfterInversionOffersFamiliesNotNone() {
+    // After "~" the grammar expects an address family, not "none" or another "~".
+    val choices = addressFamilies.nextTokenChoices("~")
+    assertTrue(choices.contains("AF_INET"))
+    assertFalse(choices.contains("none"))
+    assertFalse(choices.contains("~"))
+  }
+
+  @Test
+  fun testAfterFamilyAndSpaceOffersAnotherFamily() {
+    // Context awareness: after a family and a separator, another family is expected.
+    assertTrue(addressFamilies.nextTokenChoices("AF_INET ").contains("AF_INET6"))
+  }
+
+  @Test
+  fun testNonEnumerableNextTokenOffersNothing() {
+    // Numbers/regexes/whitespace/EOF aren't enumerable, so there is nothing concrete to suggest.
+    val portGrammar = SequenceCombinator(IntegerTerminal(0, 65536), EOF())
+    assertTrue(portGrammar.nextTokenChoices("").isEmpty())
+  }
+}