Language Server: Add hover support for HTML character references (#1512)

https://github.com/user-attachments/assets/7eff224c-d14c-4ccb-94dd-1f54e727c3d7

Author: marcoroth

.gitignore

@@ -100,6 +100,7 @@ java/org/herb/ast/Nodes.java
 java/org/herb/ast/NodeVisitor.java
 java/org/herb/ast/Visitor.java
 javascript/packages/core/src/errors.ts
+javascript/packages/core/src/html-entities.json
 javascript/packages/core/src/node-type-guards.ts
 javascript/packages/core/src/nodes.ts
 javascript/packages/core/src/visitor.ts

javascript/packages/core/package.json

@@ -14,7 +14,8 @@
   "module": "./dist/herb-core.esm.js",
   "types": "./dist/types/index.d.ts",
   "scripts": {
-    "build": "yarn clean && rollup -c",
+    "download-html-entities": "node scripts/download-html-entities.mjs",
+    "build": "yarn download-html-entities && yarn clean && rollup -c",
     "dev": "rollup -c -w",
     "clean": "rimraf dist",
     "test": "vitest run",

javascript/packages/core/scripts/download-html-entities.mjs

@@ -0,0 +1,50 @@
+#!/usr/bin/env node
+
+import { existsSync, writeFileSync } from "fs"
+import { fileURLToPath } from "url"
+import { dirname, join } from "path"
+
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = dirname(__filename)
+
+const ENTITIES_URL = "https://html.spec.whatwg.org/entities.json"
+const OUTPUT_PATH = join(__dirname, "..", "src", "html-entities.json")
+
+async function downloadEntities() {
+  if (existsSync(OUTPUT_PATH)) {
+    console.log(`HTML entities already present at ${OUTPUT_PATH}, skipping download.`)
+    return
+  }
+
+  console.log(`Downloading HTML named character references from ${ENTITIES_URL}...`)
+
+  const response = await fetch(ENTITIES_URL)
+
+  if (!response.ok) {
+    throw new Error(`Failed to fetch entities: ${response.status} ${response.statusText}`)
+  }
+
+  const data = await response.json()
+  const entities = {}
+
+  for (const [key, value] of Object.entries(data).sort(([a], [b]) => a.localeCompare(b))) {
+    if (key.endsWith(";")) {
+      const name = key.slice(1, -1)
+      entities[name] = {
+        characters: value.characters,
+        codepoints: value.codepoints,
+      }
+    }
+  }
+
+  const count = Object.keys(entities).length
+
+  writeFileSync(OUTPUT_PATH, JSON.stringify(entities, null, 2) + "\n")
+
+  console.log(`Wrote ${count} named character references to ${OUTPUT_PATH}`)
+}
+
+downloadEntities().catch((error) => {
+  console.error(error)
+  process.exit(1)
+})

javascript/packages/core/src/html-character-references.ts

@@ -0,0 +1,41 @@
+// https://html.spec.whatwg.org/multipage/syntax.html#character-references
+// JSON source: https://html.spec.whatwg.org/entities.json
+import entities from "./html-entities.json"
+
+export interface HTMLCharacterReference {
+  characters: string
+  codepoints: number[]
+}
+
+export const HTML_NAMED_CHARACTER_REFERENCES: Record<string, HTMLCharacterReference> = entities
+
+/**
+ * Pattern that matches HTML character references: named (`&amp;`), decimal (`&#60;`), and hexadecimal (`&#x3C;`).
+ * Uses capturing groups: group 1 = hex digits, group 2 = decimal digits, group 3 = named reference name.
+ *
+ * @see https://html.spec.whatwg.org/multipage/syntax.html#character-references
+ */
+export const CHARACTER_REFERENCE_PATTERN = /&(?:#x([0-9a-fA-F]+)|#([0-9]+)|([a-zA-Z][a-zA-Z0-9]*));/g
+
+export function isNamedCharacterReference(name: string): boolean {
+  return name in HTML_NAMED_CHARACTER_REFERENCES
+}
+
+export function getNamedCharacterReference(name: string): HTMLCharacterReference | undefined {
+  return HTML_NAMED_CHARACTER_REFERENCES[name]
+}
+
+/**
+ * Checks if a string is a valid HTML character reference.
+ * Supports named (`&amp;`), decimal (`&#60;`), and hexadecimal (`&#x3C;`) references.
+ *
+ * @see https://html.spec.whatwg.org/multipage/syntax.html#character-references
+ */
+export function isValidCharacterReference(text: string): boolean {
+  if (!text.startsWith("&") || !text.endsWith(";")) return false
+
+  if (text.startsWith("&#x")) return /^&#x[0-9a-fA-F]+;$/.test(text)
+  if (text.startsWith("&#")) return /^&#[0-9]+;$/.test(text)
+
+  return isNamedCharacterReference(text.slice(1, -1))
+}

javascript/packages/core/src/index.ts

@@ -1,5 +1,6 @@
 export * from "./ast-utils.js"
 export * from "./html-constants.js"
+export * from "./html-character-references.js"
 export * from "./backend.js"
 export * from "./diagnostic.js"
 export * from "./didyoumean.js"

javascript/packages/language-server/src/hover_service.ts

@@ -4,12 +4,12 @@ import { TextDocument } from "vscode-languageserver-textdocument"
 import { Visitor } from "@herb-tools/node-wasm"
 import { IdentityPrinter } from "@herb-tools/printer"
 import { ActionViewTagHelperToHTMLRewriter } from "@herb-tools/rewriter"
-import { isERBOpenTagNode, isHTMLElementNode } from "@herb-tools/core"
+import { isERBOpenTagNode, isHTMLElementNode, getNamedCharacterReference, CHARACTER_REFERENCE_PATTERN } from "@herb-tools/core"
 import { ParserService } from "./parser_service"
 import { lspPosition, isPositionInRange, rangeSize } from "./range_utils"
 import { ACTION_VIEW_HELPERS } from "./action_view_helpers"
 
-import type { HTMLElementNode, ERBOpenTagNode } from "@herb-tools/core"
+import type { HTMLElementNode, ERBOpenTagNode, HTMLCharacterReference } from "@herb-tools/core"
 
 class ActionViewElementCollector extends Visitor {
   public elements: { node: HTMLElementNode; openTag: ERBOpenTagNode; range: Range }[] = []
@@ -84,7 +84,7 @@ export class HoverService {
     }
 
     if (!bestElement) {
-      return null
+      return this.getEntityHover(textDocument, position)
     }
 
     const elementSource = bestElement.node.element_source
@@ -153,4 +153,121 @@ export class HoverService {
 
     return IdentityPrinter.print(rewrittenNode)
   }
+
+  private getEntityHover(textDocument: TextDocument, position: Position): Hover | null {
+    const lineText = textDocument.getText(Range.create(position.line, 0, position.line + 1, 0))
+    const match = findCharacterReferenceAtPosition(lineText, position.character)
+
+    if (!match) return null
+
+    const range = Range.create(
+      position.line, match.start,
+      position.line, match.end,
+    )
+
+    return {
+      contents: {
+        kind: MarkupKind.Markdown,
+        value: formatCharacterReferenceHover(match),
+      },
+      range,
+    }
+  }
+}
+
+interface CharacterReferenceMatch {
+  reference: string
+  start: number
+  end: number
+  characters: string
+  codepoints: number[]
+  type: "named" | "decimal" | "hexadecimal"
+  name?: string
+}
+
+function findCharacterReferenceAtPosition(lineText: string, character: number): CharacterReferenceMatch | null {
+  const pattern = new RegExp(CHARACTER_REFERENCE_PATTERN.source, "g")
+  let match: RegExpExecArray | null
+
+  while ((match = pattern.exec(lineText)) !== null) {
+    const start = match.index
+    const end = start + match[0].length
+
+    if (character < start || character >= end) continue
+
+    const reference = match[0]
+
+    if (match[1]) {
+      const codepoint = parseInt(match[1], 16)
+      return {
+        reference,
+        start,
+        end,
+        characters: String.fromCodePoint(codepoint),
+        codepoints: [codepoint],
+        type: "hexadecimal",
+      }
+    }
+
+    if (match[2]) {
+      const codepoint = parseInt(match[2], 10)
+      return {
+        reference,
+        start,
+        end,
+        characters: String.fromCodePoint(codepoint),
+        codepoints: [codepoint],
+        type: "decimal",
+      }
+    }
+
+    if (match[3]) {
+      const entity = getNamedCharacterReference(match[3])
+
+      if (entity) {
+        return {
+          reference,
+          start,
+          end,
+          characters: entity.characters,
+          codepoints: entity.codepoints,
+          type: "named",
+          name: match[3],
+        }
+      }
+    }
+  }
+
+  return null
+}
+
+function formatCodepoints(codepoints: number[]): string {
+  return codepoints.map(codepoint => `U+${codepoint.toString(16).toUpperCase().padStart(4, "0")}`).join(", ")
+}
+
+function formatCharacterReferenceHover(match: CharacterReferenceMatch): string {
+  const parts: string[] = []
+
+  parts.push(`## \`${match.characters}\``)
+
+  const typeLabel = match.type === "named" ? "Named character reference"
+    : match.type === "decimal" ? "Decimal numeric character reference"
+    : "Hexadecimal numeric character reference"
+
+  parts.push(`**${typeLabel}**`)
+
+  const details: string[] = []
+  details.push(`| Character | \`${match.characters}\` |`)
+  details.push(`| Codepoint${match.codepoints.length > 1 ? "s" : ""} | ${formatCodepoints(match.codepoints)} |`)
+  details.push(`| Reference | \`${match.reference}\` |`)
+
+  if (match.name) {
+    details.push(`| Name | \`${match.name}\` |`)
+  }
+
+  parts.push(`| | |\n|---|---|\n${details.join("\n")}`)
+
+  parts.push(`[HTML spec: Character references](https://html.spec.whatwg.org/multipage/syntax.html#character-references)`)
+
+  return parts.join("\n\n")
 }