Add unknown/unused directive and rule diagnostics

Improve parsing and diagnostics for Bison and Flex files. Introduces UnknownDirective and augments Bison/Flex document models to collect unknown %directives. Parsers (bisonParser/flexParser) now recognize known directive sets and record unknown directives; bisonParser also records rule alternatives with a firstSymbol (via getFirstSymbol) to aid conflict detection. computeBisonDiagnostics gained checks for unknown directives, unused non-terminals, unused tokens, and a heuristic for shift/reduce conflicts; computeFlexDiagnostics now flags unknown directives and inaccessible rules (duplicate patterns and catch-all shadowing). Tests updated/added to cover the new diagnostics and heuristics.

Author: theodevelop

server/src/parser/bisonParser.ts

@@ -6,8 +6,25 @@ import {
   DefineDeclaration,
   PrecedenceDeclaration,
   RuleDefinition,
+  RuleAlternative,
 } from './types';
 
+/**
+ * All directives recognized by GNU Bison (and common deprecated aliases).
+ * Anything starting with % that isn't in this set → unknown directive diagnostic.
+ */
+const KNOWN_BISON_DIRECTIVES = new Set([
+  'token', 'type', 'nterm', 'define', 'code',
+  'left', 'right', 'nonassoc', 'precedence',
+  'start', 'union', 'expect', 'expect-rr', 'require',
+  'language', 'skeleton', 'glr-parser', 'locations',
+  'defines', 'debug', 'param', 'parse-param', 'lex-param',
+  'printer', 'destructor', 'empty', 'prec',
+  'initial-action', 'verbose', 'no-lines', 'token-table',
+  'output', 'file-prefix', 'header', 'name-prefix',
+  'pure-parser', 'error-verbose',
+]);
+
 export function parseBisonDocument(text: string): BisonDocument {
   const lines = text.split(/\r?\n/);
   const doc: BisonDocument = {
@@ -19,6 +36,7 @@ export function parseBisonDocument(text: string): BisonDocument {
     rules: new Map(),
     separators: [],
     ruleReferences: new Map(),
+    unknownDirectives: [],
   };
 
   // Phase 1: Find %% separators (skip those inside code blocks)
@@ -163,6 +181,17 @@ export function parseBisonDocument(text: string): BisonDocument {
     }
 
     lastTokenDirectiveLine = -1;
+
+    // Unknown directive: any %word that didn't match a known pattern above
+    if (trimmed.startsWith('%') && !trimmed.startsWith('%%')) {
+      const directiveMatch = trimmed.match(/^%([a-zA-Z][a-zA-Z0-9_-]*)/);
+      if (directiveMatch && !KNOWN_BISON_DIRECTIVES.has(directiveMatch[1])) {
+        doc.unknownDirectives.push({
+          name: '%' + directiveMatch[1],
+          location: Range.create(i, 0, i, directiveMatch[0].length),
+        });
+      }
+    }
   }
 
   // Phase 3: Parse rules section
@@ -197,11 +226,21 @@ export function parseBisonDocument(text: string): BisonDocument {
           alternatives: [],
         });
       }
-      // Parse the rest of the line after ':'
+      // Parse the rest of the line after ':' as the first alternative
       const rest = trimmed.substring(ruleDefMatch[0].length);
+      const altRange = Range.create(i, 0, i, line.length);
+      const alt: RuleAlternative = { range: altRange, firstSymbol: getFirstSymbol(rest) };
+      doc.rules.get(currentRule)!.alternatives.push(alt);
       extractRuleReferences(rest, i, line, doc);
-    } else if (trimmed.startsWith('|') || currentRule) {
-      // Alternative or continuation
+    } else if (trimmed.startsWith('|') && currentRule) {
+      // New alternative: track first symbol
+      const altBody = trimmed.slice(1); // strip leading '|'
+      const altRange = Range.create(i, 0, i, line.length);
+      const alt: RuleAlternative = { range: altRange, firstSymbol: getFirstSymbol(altBody) };
+      doc.rules.get(currentRule)?.alternatives.push(alt);
+      extractRuleReferences(trimmed, i, line, doc);
+    } else if (currentRule) {
+      // Continuation of current alternative (no '|', no rule def)
       extractRuleReferences(trimmed, i, line, doc);
     }
 
@@ -228,6 +267,22 @@ export function parseBisonDocument(text: string): BisonDocument {
   return doc;
 }
 
+/**
+ * Extract the first terminal or non-terminal symbol from a production RHS.
+ * Returns undefined for empty productions (%empty) or pure action blocks.
+ */
+function getFirstSymbol(text: string): string | undefined {
+  const cleaned = text
+    .replace(/"(?:[^"\\]|\\.)*"/g, ' ')    // remove strings
+    .replace(/\{[^}]*\}/g, ' ')            // remove inline actions
+    .replace(/%prec\s+\S+/g, ' ')          // remove %prec TOKEN
+    .replace(/%empty/g, ' ')               // remove %empty
+    .replace(/\/\/.*$/g, ' ')             // remove line comments
+    .trim();
+  const m = cleaned.match(/^([a-zA-Z_][a-zA-Z0-9_.]*)/);
+  return m ? m[1] : undefined;
+}
+
 function parseTokenNames(text: string, type: string | undefined, lineNum: number, doc: BisonDocument): void {
   // Match patterns like: NAME "alias" VALUE  or just NAME
   const regex = /([A-Z_][A-Z0-9_]*)\s*(?:("(?:[^"\\]|\\.)*")\s*)?(?:(\d+)\s*)?/g;

server/src/parser/flexParser.ts

@@ -7,6 +7,16 @@ import {
   FlexRule,
 } from './types';
 
+/**
+ * All directives recognized by Flex / RE-flex.
+ * Anything starting with % that isn't in this set → unknown directive diagnostic.
+ */
+const KNOWN_FLEX_DIRECTIVES = new Set([
+  'option', 'x', 's',
+  'top', 'class',           // RE-flex extensions
+  'pointer', 'array',       // old Flex memory model
+]);
+
 /**
  * Code block types in Flex/RE-flex files:
  * - %{ ... %}    prologue block
@@ -82,6 +92,7 @@ export function parseFlexDocument(text: string): FlexDocument {
     separators: [],
     startConditionRefs: new Map(),
     abbreviationRefs: new Map(),
+    unknownDirectives: [],
   };
 
   // Build skip map for code blocks
@@ -159,6 +170,17 @@ export function parseFlexDocument(text: string): FlexDocument {
       });
       continue;
     }
+
+    // Unknown directive: any %word that didn't match a known pattern above
+    if (trimmed.startsWith('%') && !trimmed.startsWith('%%') && !trimmed.startsWith('%{') && !trimmed.startsWith('%}')) {
+      const directiveMatch = trimmed.match(/^%([a-zA-Z][a-zA-Z0-9_-]*)/);
+      if (directiveMatch && !KNOWN_FLEX_DIRECTIVES.has(directiveMatch[1])) {
+        doc.unknownDirectives.push({
+          name: '%' + directiveMatch[1],
+          location: Range.create(i, 0, i, directiveMatch[0].length),
+        });
+      }
+    }
   }
 
   // Phase 3: Parse rules section

server/src/parser/types.ts

@@ -1,5 +1,10 @@
 import { Range } from 'vscode-languageserver';
 
+export interface UnknownDirective {
+  name: string;       // e.g. "%prout"
+  location: Range;
+}
+
 export interface TokenDeclaration {
   name: string;
   type?: string;       // e.g., "int", "std::string"
@@ -31,10 +36,15 @@ export interface CodeBlock {
   range: Range;
 }
 
+export interface RuleAlternative {
+  range: Range;
+  firstSymbol?: string;  // first terminal/non-terminal of this production (for conflict detection)
+}
+
 export interface RuleDefinition {
   name: string;
   location: Range;
-  alternatives: Range[];
+  alternatives: RuleAlternative[];
 }
 
 export interface BisonDocument {
@@ -47,6 +57,7 @@ export interface BisonDocument {
   separators: number[];  // line numbers of %%
   startSymbol?: string;
   ruleReferences: Map<string, Range[]>;  // symbol name -> locations used in rules RHS
+  unknownDirectives: UnknownDirective[];
 }
 
 export interface FlexOption {
@@ -82,6 +93,7 @@ export interface FlexDocument {
   separators: number[];
   startConditionRefs: Map<string, Range[]>;  // SC name -> locations used in rules
   abbreviationRefs: Map<string, Range[]>;    // abbrev name -> locations used in rules
+  unknownDirectives: UnknownDirective[];
 }
 
 export type DocumentModel = BisonDocument | FlexDocument;

server/src/providers/diagnostics.ts

@@ -16,6 +16,16 @@ export function computeBisonDiagnostics(doc: BisonDocument, text: string): Diagn
     return diagnostics; // Can't do much more without sections
   }
 
+  // ── TASK 1: Unknown directives ──────────────────────────────────────────────
+  for (const unk of doc.unknownDirectives) {
+    diagnostics.push({
+      severity: DiagnosticSeverity.Error,
+      range: unk.location,
+      message: `Unknown Bison directive '${unk.name}'. Check the Bison manual for valid directives.`,
+      source: 'bison',
+    });
+  }
+
   // 2. Duplicate token declarations
   const tokenCounts = new Map<string, Range[]>();
   for (const [name, decl] of doc.tokens) {
@@ -94,6 +104,60 @@ export function computeBisonDiagnostics(doc: BisonDocument, text: string): Diagn
     });
   }
 
+  // ── TASK 2: Unused rules (non-terminals never referenced) ───────────────────
+  // If %start is not declared, Bison uses the first rule as the implicit start symbol
+  const effectiveStart = doc.startSymbol ?? (doc.rules.size > 0 ? [...doc.rules.keys()][0] : undefined);
+
+  for (const [name, rule] of doc.rules) {
+    // The start symbol is the grammar entry point — always "used"
+    if (name === effectiveStart) continue;
+    // If this name never appears in any rule body, it is unreachable
+    if (!doc.ruleReferences.has(name)) {
+      diagnostics.push({
+        severity: DiagnosticSeverity.Warning,
+        range: rule.location,
+        message: `Non-terminal '${name}' is defined but never referenced in any rule. It is unreachable from the grammar.`,
+        source: 'bison',
+      });
+    }
+  }
+
+  // ── TASK 3: Unused tokens ────────────────────────────────────────────────────
+  for (const [name, decl] of doc.tokens) {
+    if (!doc.ruleReferences.has(name)) {
+      diagnostics.push({
+        severity: DiagnosticSeverity.Warning,
+        range: decl.location,
+        message: `Token '${name}' is declared with %token but never used in any rule.`,
+        source: 'bison',
+      });
+    }
+  }
+
+  // ── TASK 4: Obvious shift/reduce conflicts ───────────────────────────────────
+  // Heuristic: same terminal token appears as first symbol in ≥2 alternatives
+  // of the same rule, with no %prec disambiguation tracked.
+  for (const [name, rule] of doc.rules) {
+    // Count how many alternatives start with each terminal (ALL_CAPS)
+    const firstTerminalCount = new Map<string, number>();
+    for (const alt of rule.alternatives) {
+      const sym = alt.firstSymbol;
+      if (sym && /^[A-Z_][A-Z0-9_]*$/.test(sym) && doc.tokens.has(sym)) {
+        firstTerminalCount.set(sym, (firstTerminalCount.get(sym) ?? 0) + 1);
+      }
+    }
+    for (const [token, count] of firstTerminalCount) {
+      if (count >= 2) {
+        diagnostics.push({
+          severity: DiagnosticSeverity.Warning,
+          range: rule.location,
+          message: `Potential shift/reduce conflict in rule '${name}': token '${token}' starts ${count} alternatives without precedence disambiguation (%prec / %left / %right).`,
+          source: 'bison',
+        });
+      }
+    }
+  }
+
   return diagnostics;
 }
 
@@ -112,6 +176,16 @@ export function computeFlexDiagnostics(doc: FlexDocument, text: string): Diagnos
     return diagnostics;
   }
 
+  // ── TASK 1: Unknown directives ──────────────────────────────────────────────
+  for (const unk of doc.unknownDirectives) {
+    diagnostics.push({
+      severity: DiagnosticSeverity.Error,
+      range: unk.location,
+      message: `Unknown Flex directive '${unk.name}'. Valid directives are %option, %x, %s, %top, %class.`,
+      source: 'flex',
+    });
+  }
+
   // 2. Undefined start conditions used in rules
   for (const [name, refs] of doc.startConditionRefs) {
     if (!doc.startConditions.has(name) && name !== 'INITIAL') {
@@ -180,5 +254,74 @@ export function computeFlexDiagnostics(doc: FlexDocument, text: string): Diagnos
     });
   }
 
+  // ── TASK 5: Inaccessible Flex rules ─────────────────────────────────────────
+  // Heuristic A: Exact duplicate pattern → second one is always shadowed.
+  // Heuristic B: Catch-all pattern (. or .* or .*\n etc.) before specific patterns
+  //              in the same start-condition context → subsequent rules unreachable.
+
+  // Build a canonical "context key" for a rule: sorted start conditions, or "INITIAL"
+  const contextKey = (rule: typeof doc.rules[0]): string =>
+    rule.startConditions.length > 0 ? [...rule.startConditions].sort().join(',') : 'INITIAL';
+
+  /**
+   * Extract just the regex part of a Flex rule pattern string.
+   * doc.rules[].pattern is the full trimmed line: "<SC> pattern   { action }"
+   * We strip the optional <SC> prefix, then take the first non-space token (the regex).
+   * In Flex, patterns cannot contain unescaped spaces, so the pattern ends at
+   * the first whitespace after the regex.
+   */
+  const rawPattern = (pattern: string): string => {
+    // Remove optional <SC> or <SC1,SC2> prefix
+    let p = pattern.replace(/^<[A-Z_*][A-Z0-9_,*]*>\s*/, '').trimStart();
+    // The pattern is the first "word" — Flex patterns have no unescaped spaces
+    const m = p.match(/^(\S+)/);
+    return m ? m[1] : p;
+  };
+
+  // Catch-all patterns that would shadow everything after them
+  const CATCHALL_PATTERNS = new Set(['.', '.*', '.+', '.|\\n', '(.|\n)*', '(.|\n)+', '(.|\\n)*', '(.|\\n)+']);
+
+  // Track: first seen pattern per context (for duplicate detection)
+  const seenPatterns = new Map<string, number>(); // "context|pattern" -> line number of first occurrence
+
+  // Track: catch-all line per context key
+  const catchallLine = new Map<string, number>(); // context -> line number
+
+  for (const rule of doc.rules) {
+    const ctx = contextKey(rule);
+    const pat = rawPattern(rule.pattern);
+    const lineNum = rule.location.start.line;
+    const dupKey = `${ctx}|${pat}`;
+
+    // Heuristic B: is this rule after a catch-all in the same context?
+    if (catchallLine.has(ctx) && !CATCHALL_PATTERNS.has(pat)) {
+      const catchLine = catchallLine.get(ctx)!;
+      diagnostics.push({
+        severity: DiagnosticSeverity.Warning,
+        range: rule.location,
+        message: `Flex rule '${pat}' may be inaccessible: catch-all pattern at line ${catchLine + 1} will always match first.`,
+        source: 'flex',
+      });
+    }
+
+    // Heuristic A: duplicate pattern in same context?
+    if (seenPatterns.has(dupKey)) {
+      const firstLine = seenPatterns.get(dupKey)!;
+      diagnostics.push({
+        severity: DiagnosticSeverity.Warning,
+        range: rule.location,
+        message: `Flex rule '${pat}' is inaccessible: identical pattern already defined at line ${firstLine + 1}.`,
+        source: 'flex',
+      });
+    } else {
+      seenPatterns.set(dupKey, lineNum);
+    }
+
+    // Register catch-all (only on first occurrence in this context)
+    if (CATCHALL_PATTERNS.has(pat) && !catchallLine.has(ctx)) {
+      catchallLine.set(ctx, lineNum);
+    }
+  }
+
   return diagnostics;
 }