input: revise CustomHighlighter to token-name model + add example

Reshapes the trait per maintainer feedback on PR longbridge#2328:

    - fn styles(&self, range: Range<usize>, cx: &App)
    -     -> Vec<(Range<usize>, HighlightStyle)>;
    + fn tokens(&self, range: Range<usize>)
    +     -> Vec<(Range<usize>, SharedString)>;

The previous shape returned pre-resolved `HighlightStyle` and took
`&App`, forcing implementors to resolve theme tokens on the render
thread every frame. That bypassed upstream's own tree-sitter path,
which emits scope-name strings resolved through `HighlightTheme::style()`
at `highlighter.rs:719`.

Per Monaco Editor's design (referenced in review), tokenizers emit
named scopes; a separate theme map resolves names to styles. The trait
now constrains the hot path to a `SharedString` read so heavy parsing
can be arranged off-thread (subscribe to `InputEvent::Change`, cache,
hand back ranges on call) -- addressing the 200k-line performance
concern by construction.

Token names use upstream's existing scope vocabulary
(`"keyword"`, `"string"`, `"comment"`, `"variable.special"`, ...);
`.`-namespaced names fall back to their prefix via
`SyntaxColors::style`. Unrecognized names render with the default
style.

Adds a worked example at `crates/story/examples/editor.rs`
(`MarkerHighlighter`): tags `TODO` / `FIXME` / `XXX` / `HACK` /
`NOTE` markers with `keyword.special`, recomputed on
`InputEvent::Change`. A syntect or language-server consumer would
follow the same shape with a different parser inside `refresh()`.

Tests updated to the new method signature; both still verify
install / clear round-trip and tree-sitter composition through
`combine_highlights`.

Author: glani

crates/story/examples/editor.rs

@@ -13,7 +13,10 @@ use gpui_component::{
     ActiveTheme, IconName, Sizable, WindowExt,
     button::{Button, ButtonVariants as _},
     h_flex,
-    highlighter::{Diagnostic, DiagnosticSeverity, Language, LanguageConfig, LanguageRegistry},
+    highlighter::{
+        CustomHighlighter, Diagnostic, DiagnosticSeverity, Language, LanguageConfig,
+        LanguageRegistry,
+    },
     input::{
         self, CodeActionProvider, CompletionProvider, DefinitionProvider, DocumentColorProvider,
         HoverProvider, Input, InputEvent, InputState, Position, Rope, RopeExt, TabSize,
@@ -66,6 +69,57 @@ fn init() {
     );
 }
 
+/// Example consumer of [`CustomHighlighter`]: tags `TODO` / `FIXME` / `XXX`
+/// / `HACK` / `NOTE` markers anywhere in the buffer with the
+/// `keyword.special` token so they stand out against the tree-sitter
+/// comment colour.
+///
+/// Demonstrates the stateful pattern the trait expects — heavy work happens
+/// off the render thread (here: in response to `InputEvent::Change`) and
+/// the per-frame [`CustomHighlighter::tokens`] call is a pure read of
+/// pre-computed state. A real consumer plugging in syntect or a language
+/// server would follow the same shape with a different parser inside
+/// [`refresh`](MarkerHighlighter::refresh).
+#[derive(Default)]
+struct MarkerHighlighter {
+    tokens: RwLock<Vec<(Range<usize>, SharedString)>>,
+}
+
+impl MarkerHighlighter {
+    fn new() -> Self {
+        Self::default()
+    }
+
+    fn refresh(&self, text: &Rope) {
+        const MARKERS: &[&str] = &["TODO", "FIXME", "XXX", "HACK", "NOTE"];
+        let token: SharedString = "keyword.special".into();
+        let s = text.to_string();
+        let mut new_tokens: Vec<(Range<usize>, SharedString)> = Vec::new();
+        for marker in MARKERS {
+            let mut start = 0usize;
+            while let Some(rel) = s[start..].find(marker) {
+                let abs = start + rel;
+                new_tokens.push((abs..abs + marker.len(), token.clone()));
+                start = abs + marker.len();
+            }
+        }
+        new_tokens.sort_by_key(|(r, _)| r.start);
+        *self.tokens.write().unwrap() = new_tokens;
+    }
+}
+
+impl CustomHighlighter for MarkerHighlighter {
+    fn tokens(&self, range: Range<usize>) -> Vec<(Range<usize>, SharedString)> {
+        self.tokens
+            .read()
+            .unwrap()
+            .iter()
+            .filter(|(r, _)| r.start >= range.start && r.end <= range.end)
+            .cloned()
+            .collect()
+    }
+}
+
 pub struct Example {
     editor: Entity<InputState>,
     tree_state: Entity<TreeState>,
@@ -77,6 +131,7 @@ pub struct Example {
     show_whitespaces: bool,
     folding: bool,
     lsp_store: ExampleLspStore,
+    marker_highlighter: Arc<MarkerHighlighter>,
     _subscriptions: Vec<Subscription>,
     _lint_task: Task<()>,
 }
@@ -690,6 +745,7 @@ impl Example {
     pub fn new(window: &mut Window, cx: &mut Context<Self>) -> Self {
         let default_language = Lang::BuiltIn(Language::Rust);
         let lsp_store = ExampleLspStore::new();
+        let marker_highlighter = Arc::new(MarkerHighlighter::new());
 
         let editor = cx.new(|cx| {
             let mut editor = InputState::new(window, cx)
@@ -711,6 +767,14 @@ impl Example {
             editor.lsp.definition_provider = Some(lsp_store.clone());
             editor.lsp.document_color_provider = Some(lsp_store.clone());
 
+            // Install the example custom highlighter and seed it from the
+            // initial buffer text so markers are tagged on first paint.
+            marker_highlighter.refresh(editor.text());
+            editor.set_custom_highlighter(
+                Some(marker_highlighter.clone() as Arc<dyn CustomHighlighter>),
+                cx,
+            );
+
             editor
         });
 
@@ -726,7 +790,9 @@ impl Example {
         let tree_state = cx.new(|cx| TreeState::new(cx));
         Self::load_files(tree_state.clone(), PathBuf::from("./"), cx);
 
-        let _subscriptions = vec![cx.subscribe(&editor, |this, _, _: &InputEvent, cx| {
+        let _subscriptions = vec![cx.subscribe(&editor, |this, editor, _: &InputEvent, cx| {
+            let text = editor.read(cx).text().clone();
+            this.marker_highlighter.refresh(&text);
             this.lint_document(cx);
         })];
 
@@ -741,6 +807,7 @@ impl Example {
             show_whitespaces: false,
             folding: true,
             lsp_store,
+            marker_highlighter,
             _subscriptions,
             _lint_task: Task::ready(()),
         }

crates/ui/src/highlighter/custom.rs

@@ -1,8 +1,8 @@
 use std::ops::Range;
 
-use gpui::{App, HighlightStyle};
+use gpui::SharedString;
 
-/// A consumer-supplied highlighter that contributes additional styled byte
+/// A consumer-supplied highlighter that contributes additional named token
 /// ranges to the [`Input`](crate::input::InputState) element's render
 /// pipeline, alongside the built-in tree-sitter
 /// [`SyntaxHighlighter`](crate::highlighter::SyntaxHighlighter) and
@@ -13,9 +13,23 @@ use gpui::{App, HighlightStyle};
 /// - Plugging a different parser engine (syntect, regex-based tokenizers,
 ///   language server semantic tokens) for languages tree-sitter does not
 ///   cover.
-/// - Layering decorative highlights (search match emphasis, code folding
-///   indicators, scope-aware accent colors) without rebuilding the
-///   diagnostic pipeline.
+/// - Layering decorative highlights (search-match emphasis, scope-aware
+///   accent colors) without rebuilding the diagnostic pipeline.
+///
+/// # Token-name vocabulary
+///
+/// Returned token names are resolved against the active
+/// [`HighlightTheme`](crate::highlighter::HighlightTheme) — the same
+/// vocabulary the tree-sitter highlighter emits (`"keyword"`, `"string"`,
+/// `"comment"`, `"variable.special"`, …). Names with a `.`-namespace fall
+/// back to their prefix (`"keyword.modifier"` → `"keyword"`). Unrecognized
+/// names render with the default style.
+///
+/// This decouples token classification from styling: the theme is the
+/// single source of color, so theme switches propagate without implementor
+/// cooperation. It also lets a third-party highlighter share a vocabulary
+/// with the built-in tree-sitter path so multiple sources colour the same
+/// logical token consistently.
 ///
 /// # Composition
 ///
@@ -26,18 +40,24 @@ use gpui::{App, HighlightStyle};
 ///
 /// # Threading and performance
 ///
-/// [`styles`](Self::styles) is called from the render thread inside the
-/// `Input` element's per-frame highlight pass. Implementations should be
-/// `Send + Sync` and inexpensive — caching parsed state across calls is the
-/// implementor's responsibility.
+/// [`tokens`](Self::tokens) is called from the render thread on every
+/// frame the input is visible. Implementations should be `Send + Sync` and
+/// inexpensive — typically a read of pre-computed state. Heavy parsing
+/// should happen off-thread (for example, in response to a text-change
+/// event the implementor subscribes to) and cache its output for the
+/// render thread to consume.
 ///
 /// The viewport-clamping that the built-in tree-sitter path applies for
-/// long-line skipping does **not** apply to custom highlighter output;
-/// implementations are responsible for their own performance characteristics.
+/// long-line skipping does **not** apply to custom-highlighter output;
+/// implementations are responsible for their own performance
+/// characteristics.
 pub trait CustomHighlighter: Send + Sync {
-    /// Return styled byte ranges within the requested viewport range.
+    /// Return token-name-tagged byte ranges within the requested viewport
+    /// range.
     ///
-    /// Returned ranges should be a subset of `range`; ranges outside `range`
-    /// are silently dropped during composition.
-    fn styles(&self, range: Range<usize>, cx: &App) -> Vec<(Range<usize>, HighlightStyle)>;
+    /// Returned ranges should be a subset of `range`; ranges outside
+    /// `range` are silently dropped during composition. Token names are
+    /// resolved against the active theme — see the type-level docs for
+    /// the vocabulary.
+    fn tokens(&self, range: Range<usize>) -> Vec<(Range<usize>, SharedString)>;
 }

crates/ui/src/input/element.rs

@@ -1317,11 +1317,23 @@ impl TextElement {
 
         let diagnostic_styles = diagnostics.styles_for_range(&visible_byte_range, cx);
 
-        // Custom highlighter styles, layered between tree-sitter (base) and
-        // diagnostics (top). Empty Vec when no custom highlighter is set, so
-        // `combine_highlights` short-circuits.
+        // Custom highlighter tokens, resolved through the active highlight
+        // theme so the custom source shares the same colour vocabulary as
+        // the tree-sitter path. Empty Vec when no custom highlighter is set,
+        // so `combine_highlights` short-circuits.
         let custom_styles = match &custom_highlighter {
-            Some(h) => h.styles(visible_byte_range.clone(), cx),
+            Some(h) => {
+                let highlight_theme = &cx.theme().highlight_theme;
+                h.tokens(visible_byte_range.clone())
+                    .into_iter()
+                    .map(|(range, name)| {
+                        (
+                            range,
+                            highlight_theme.style(name.as_ref()).unwrap_or_default(),
+                        )
+                    })
+                    .collect::<Vec<_>>()
+            }
             None => Vec::new(),
         };
 

crates/ui/src/input/state.rs

@@ -658,11 +658,14 @@ impl InputState {
         cx.notify();
     }
 
-    /// Set a custom highlighter that contributes additional styles to the
-    /// render pipeline alongside the built-in tree-sitter highlighter.
+    /// Set a custom highlighter that contributes additional named token
+    /// ranges to the render pipeline alongside the built-in tree-sitter
+    /// highlighter.
     ///
     /// The custom highlighter does **not** replace the built-in. Both run
-    /// (when set), and their outputs are combined via
+    /// (when set); the custom output is resolved against the active
+    /// [`HighlightTheme`](crate::highlighter::HighlightTheme) and combined
+    /// with the tree-sitter and diagnostic styles via
     /// [`gpui::combine_highlights`]. Pass `None` to remove a previously-set
     /// custom highlighter.
     ///
@@ -3135,32 +3138,19 @@ ORDER BY id
     fn test_custom_highlighter_composes_with_tree_sitter(cx: &mut TestAppContext) {
         use crate::highlighter::{CustomHighlighter, HighlightTheme};
         use crate::input::mode::InputMode;
-        use gpui::{HighlightStyle, Hsla};
+        use gpui::{HighlightStyle, SharedString};
         use std::ops::Range;
         use std::sync::Arc;
 
-        // Custom highlighter that paints a single bright-red range.
-        struct RedRange {
+        // Custom highlighter that tags a fixed byte range with a token name
+        // resolvable by the active highlight theme. Picking "keyword" guarantees
+        // a non-default style on the default themes.
+        struct KeywordRange {
             range: Range<usize>,
         }
-        impl CustomHighlighter for RedRange {
-            fn styles(
-                &self,
-                _range: Range<usize>,
-                _cx: &App,
-            ) -> Vec<(Range<usize>, HighlightStyle)> {
-                vec![(
-                    self.range.clone(),
-                    HighlightStyle {
-                        color: Some(Hsla {
-                            h: 0.0,
-                            s: 1.0,
-                            l: 0.5,
-                            a: 1.0,
-                        }),
-                        ..Default::default()
-                    },
-                )]
+        impl CustomHighlighter for KeywordRange {
+            fn tokens(&self, _range: Range<usize>) -> Vec<(Range<usize>, SharedString)> {
+                vec![(self.range.clone(), "keyword".into())]
             }
         }
 
@@ -3177,20 +3167,21 @@ ORDER BY id
         });
         cx.run_until_parked();
 
-        // Custom highlighter paints bytes [0..6) (covers "SELECT").
-        let custom: Arc<dyn CustomHighlighter> = Arc::new(RedRange { range: 0..6 });
+        // Custom highlighter tags bytes [0..6) (covers "SELECT") as "keyword".
+        let custom: Arc<dyn CustomHighlighter> = Arc::new(KeywordRange { range: 0..6 });
         cx.update(|_, cx| {
             input.update(cx, |state, cx| {
                 state.set_custom_highlighter(Some(custom), cx);
             });
         });
         cx.run_until_parked();
 
-        // Mirror element.rs::highlight_lines: pull both sources from state.mode
-        // and combine them in the documented order.
+        // Mirror element.rs::highlight_lines: pull both sources from state.mode,
+        // resolve the custom tokens against the highlight theme, and combine
+        // in the documented order.
         let theme = HighlightTheme::default_dark();
-        let combined: Vec<(Range<usize>, HighlightStyle)> = cx.update(|_, cx| {
-            input.read_with(cx, |state, cx| {
+        let combined: Vec<(Range<usize>, HighlightStyle)> = cx.update(|_, _cx| {
+            input.read_with(_cx, |state, _| {
                 let visible_range = 0..text.len();
 
                 let (ts_styles, custom_styles) = match &state.mode {
@@ -3204,9 +3195,16 @@ ORDER BY id
                             .as_ref()
                             .map(|h| h.styles(&visible_range, &theme))
                             .unwrap_or_default();
-                        let custom_styles = custom_highlighter
+                        let custom_styles: Vec<(Range<usize>, HighlightStyle)> = custom_highlighter
                             .as_ref()
-                            .map(|h| h.styles(visible_range.clone(), cx))
+                            .map(|h| {
+                                h.tokens(visible_range.clone())
+                                    .into_iter()
+                                    .map(|(r, name)| {
+                                        (r, theme.style(name.as_ref()).unwrap_or_default())
+                                    })
+                                    .collect()
+                            })
                             .unwrap_or_default();
                         (ts_styles, custom_styles)
                     }
@@ -3223,6 +3221,15 @@ ORDER BY id
                     "custom highlighter produced no styles"
                 );
 
+                // The custom "keyword" token should resolve to a non-default
+                // style on the default-dark theme — proves theme integration.
+                assert!(
+                    custom_styles
+                        .iter()
+                        .any(|(_, s)| *s != HighlightStyle::default()),
+                    "custom token name 'keyword' did not resolve to a styled output"
+                );
+
                 // Compose in the same order as element.rs::highlight_lines.
                 gpui::combine_highlights(custom_styles, ts_styles).collect()
             })
@@ -3245,19 +3252,15 @@ ORDER BY id
     fn test_set_custom_highlighter_round_trip(cx: &mut TestAppContext) {
         use crate::highlighter::CustomHighlighter;
         use crate::input::mode::InputMode;
-        use gpui::HighlightStyle;
+        use gpui::SharedString;
         use std::ops::Range;
         use std::sync::Arc;
 
         struct DummyHighlighter;
 
         impl CustomHighlighter for DummyHighlighter {
-            fn styles(
-                &self,
-                range: Range<usize>,
-                _cx: &App,
-            ) -> Vec<(Range<usize>, HighlightStyle)> {
-                vec![(range, HighlightStyle::default())]
+            fn tokens(&self, range: Range<usize>) -> Vec<(Range<usize>, SharedString)> {
+                vec![(range, "keyword".into())]
             }
         }