Skip to content

Commit 5992830

Browse files
vipmaxvipmax
committed
Improve WordHighlight detection and Unicode correctness
- Make WordHighlight extraction robust at Unicode word boundaries - Use grapheme-aware traversal to avoid UTF-16 boundary issues - Add helper utilities for word-grapheme checks - Expand tests for Cyrillic, CJK, and emoji-adjacent behavior
1 parent af1729b commit 5992830

8 files changed

Lines changed: 302 additions & 20 deletions

File tree

‎anycode-base/src/code.ts‎

Lines changed: 91 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import {
1010
import Parser from 'web-tree-sitter';
1111
import History from './history';
1212
import { Selection } from './selection';
13-
import { getWasmPath } from './utils';
13+
import { getGraphemeAt, getNextGraphemeIndex, getPrevGraphemeIndex, getWasmPath, isWordGrapheme } from './utils';
1414
import type { Lang } from './lang';
1515

1616
import javascript from './langs/javascript';
@@ -61,6 +61,20 @@ export type Position = {
6161
column: number;
6262
}
6363

64+
export type WordHighlight = {
65+
text: string;
66+
token: string | null;
67+
};
68+
69+
export function areWordHighlightsEqual(
70+
a: WordHighlight | null,
71+
b: WordHighlight | null
72+
): boolean {
73+
if (a === b) return true;
74+
if (!a || !b) return false;
75+
return a.text === b.text && a.token === b.token;
76+
}
77+
6478

6579
export interface HighlighedNode {
6680
name: string | null;
@@ -977,6 +991,82 @@ export class Code {
977991
return columns;
978992
}
979993

994+
public getWordAtOffset(offset: number): WordHighlight | null {
995+
if (offset < 0 || offset > this.length()) {
996+
return null;
997+
}
998+
const pos = this.getPosition(offset);
999+
return this.getWordAtPosition(pos.line, pos.column);
1000+
}
1001+
1002+
public getWordAtPosition(lineIndex: number, columnIndex: number): WordHighlight | null {
1003+
if (lineIndex < 0 || lineIndex >= this.linesLength()) {
1004+
return null;
1005+
}
1006+
const lineText = this.line(lineIndex);
1007+
if (columnIndex < 0 || columnIndex > lineText.length) {
1008+
return null;
1009+
}
1010+
1011+
// Anchor: grapheme under cursor, otherwise grapheme on the left.
1012+
let anchor = -1;
1013+
if (columnIndex < lineText.length && isWordGrapheme(getGraphemeAt(lineText, columnIndex))) {
1014+
anchor = columnIndex;
1015+
} else if (columnIndex > 0) {
1016+
const prev = getPrevGraphemeIndex(lineText, columnIndex);
1017+
if (isWordGrapheme(getGraphemeAt(lineText, prev))) {
1018+
anchor = prev;
1019+
}
1020+
}
1021+
if (anchor === -1) {
1022+
return null;
1023+
}
1024+
1025+
// Expand to the left by grapheme clusters.
1026+
let start = anchor;
1027+
while (start > 0) {
1028+
const prev = getPrevGraphemeIndex(lineText, start);
1029+
if (prev === start || !isWordGrapheme(getGraphemeAt(lineText, prev))) {
1030+
break;
1031+
}
1032+
start = prev;
1033+
}
1034+
1035+
// Expand to the right by grapheme clusters.
1036+
let end = getNextGraphemeIndex(lineText, anchor);
1037+
while (end < lineText.length) {
1038+
if (!isWordGrapheme(getGraphemeAt(lineText, end))) {
1039+
break;
1040+
}
1041+
const next = getNextGraphemeIndex(lineText, end);
1042+
if (next === end) {
1043+
break;
1044+
}
1045+
end = next;
1046+
}
1047+
1048+
if (start === end) {
1049+
return null;
1050+
}
1051+
1052+
const text = lineText.slice(start, end);
1053+
1054+
// Find the class (name) of the token at the cursor position
1055+
let classs: string | null = null;
1056+
const nodes = this.getLineNodes(lineIndex);
1057+
let currentCharCount = 0;
1058+
for (const node of nodes) {
1059+
const nextCharCount = currentCharCount + node.text.length;
1060+
if (start >= currentCharCount && start < nextCharCount) {
1061+
classs = node.name;
1062+
break;
1063+
}
1064+
currentCharCount = nextCharCount;
1065+
}
1066+
1067+
return { text, token: classs };
1068+
}
1069+
9801070
clone() {
9811071
return new Code(this.getContent(), this.filename, this.language!);
9821072
}

‎anycode-base/src/editor.ts‎

Lines changed: 50 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { Code, Change, Position, Operation, type FoldRange } from "./code";
1+
import { Code, Change, Position, Operation, type FoldRange, WordHighlight, areWordHighlightsEqual } from "./code";
22
import { Renderer } from './renderer/Renderer';
33
import { getPosFromMouse } from './mouse';
44
import { Selection, hasDiagnosticSelection } from "./selection";
@@ -39,6 +39,7 @@ export interface EditorOptions {
3939
focusedDiffEnabled?: boolean;
4040
focusedDiffContextLines?: number;
4141
codeFoldingEnabled?: boolean;
42+
wordHighlightEnabled?: boolean;
4243
}
4344

4445
export interface EditorState {
@@ -54,6 +55,8 @@ export interface EditorState {
5455
foldRanges: FoldRange[];
5556
collapsedFoldStarts: Set<number>;
5657
codeFoldingEnabled: boolean;
58+
wordHighlightEnabled: boolean;
59+
wordHighlight: WordHighlight | null;
5760
}
5861

5962
export class AnycodeEditor {
@@ -104,6 +107,9 @@ export class AnycodeEditor {
104107
private readonly readOnly: boolean;
105108
private collapsedFoldStarts: Set<number> = new Set();
106109
private codeFoldingEnabled: boolean;
110+
111+
private wordHighlightEnabled: boolean;
112+
private wordHighlight: WordHighlight | null = null;
107113

108114
constructor(
109115
initialText = '',
@@ -116,6 +122,7 @@ export class AnycodeEditor {
116122
this.focusedDiffEnabled = options.focusedDiffEnabled ?? false;
117123
this.focusedDiffContextLines = Math.max(0, options.focusedDiffContextLines ?? 3);
118124
this.codeFoldingEnabled = options.codeFoldingEnabled ?? true;
125+
this.wordHighlightEnabled = options.wordHighlightEnabled ?? true;
119126
// Set initial cursor position
120127
if (options.line !== undefined && options.column !== undefined) {
121128
this.offset = this.code.getOffset(options.line, options.column);
@@ -318,9 +325,40 @@ export class AnycodeEditor {
318325
public setCursor(line: number, column: number): void {
319326
const offset = this.code.getOffset(line, column);
320327
this.offset = offset;
328+
this.updateWordHighlight();
321329
this.renderer.renderCursor(line, column);
322330
}
323331

332+
private updateWordHighlight() {
333+
if (!this.code) return;
334+
335+
if (!this.wordHighlightEnabled) {
336+
if (this.wordHighlight !== null) {
337+
this.wordHighlight = null;
338+
if (this.renderer) {
339+
this.renderer.renderWordHighlight(this.getEditorState());
340+
}
341+
}
342+
return;
343+
}
344+
const highlight = this.code.getWordAtOffset(this.offset);
345+
const hasChanged = !areWordHighlightsEqual(highlight, this.wordHighlight);
346+
347+
if (hasChanged) {
348+
this.wordHighlight = highlight;
349+
if (this.renderer) {
350+
this.renderer.renderWordHighlight(this.getEditorState());
351+
}
352+
}
353+
}
354+
355+
public setWordHighlightEnabled(enabled: boolean) {
356+
if (this.wordHighlightEnabled !== enabled) {
357+
this.wordHighlightEnabled = enabled;
358+
this.updateWordHighlight();
359+
}
360+
}
361+
324362
public setSelectionRange(
325363
startLine: number,
326364
startColumn: number,
@@ -332,6 +370,7 @@ export class AnycodeEditor {
332370
const endOffset = this.code.getOffset(endLine, endColumn);
333371
this.selection = new Selection(startOffset, endOffset);
334372
this.offset = endOffset;
373+
this.updateWordHighlight();
335374

336375
if (center) {
337376
this.renderer.focusCenter(this.getEditorState());
@@ -529,6 +568,8 @@ export class AnycodeEditor {
529568
foldRanges: this.code.getFoldRanges(),
530569
collapsedFoldStarts: this.collapsedFoldStarts,
531570
codeFoldingEnabled: this.codeFoldingEnabled,
571+
wordHighlightEnabled: this.wordHighlightEnabled,
572+
wordHighlight: this.wordHighlight,
532573
};
533574
}
534575

@@ -627,6 +668,7 @@ export class AnycodeEditor {
627668
//if (o == this.offset) { return; }
628669

629670
this.offset = o;
671+
this.updateWordHighlight();
630672

631673
const { line, column } = this.code.getPosition(this.offset);
632674
this.renderer.renderCursor(line, column);
@@ -1194,9 +1236,15 @@ export class AnycodeEditor {
11941236
this.code = result.ctx.code;
11951237
this.recomputeDiffs();
11961238
}
1197-
if (offsetChanged) this.offset = result.ctx.offset;
1239+
if (offsetChanged) {
1240+
this.offset = result.ctx.offset;
1241+
}
11981242
if (selectionChanged) this.selection = result.ctx.selection || null;
11991243

1244+
if (textChanged || offsetChanged) {
1245+
this.updateWordHighlight();
1246+
}
1247+
12001248
const state = this.getEditorState();
12011249

12021250
if (textChanged) {

‎anycode-base/src/renderer/DiffRenderer.ts‎

Lines changed: 27 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { AnycodeLine } from "../utils";
22
import { EditorSettings } from "../editor";
33
import { DiffInfo, ChangeType } from "../diff";
4-
import { HighlighedNode } from "../code";
4+
import { HighlighedNode, WordHighlight } from "../code";
55
import type { GhostRow, SeparatorRow, VisualRow } from "./Renderer";
66

77
export type ExpandDirection = 'up' | 'down' | 'both' | 'all';
@@ -81,7 +81,8 @@ export class DiffRenderer {
8181
text: string,
8282
settings: EditorSettings,
8383
hunkId: number,
84-
nodes?: HighlighedNode[]
84+
nodes?: HighlighedNode[],
85+
wordHighlight?: WordHighlight | null
8586
): HTMLDivElement {
8687
const ghostLine = document.createElement('div');
8788
ghostLine.className = "line line-deleted-ghost";
@@ -92,12 +93,25 @@ export class DiffRenderer {
9293
if (nodes && nodes.length > 0) {
9394
for (const { name, text: nodeText } of nodes) {
9495
const span = document.createElement('span');
96+
const classNameParts: string[] = [];
9597
if (name) {
9698
// Keep class fallback behavior consistent with normal line rendering.
9799
const parts = name.split('.').filter(Boolean);
98-
span.className = [name, ...parts].join(' ');
100+
classNameParts.push(...Array.from(new Set([name, ...parts])));
101+
}
102+
if (!name && nodeText === '\t') classNameParts.push('indent');
103+
104+
if (
105+
wordHighlight?.token &&
106+
classNameParts.includes(wordHighlight.token) &&
107+
nodeText === wordHighlight.text
108+
) {
109+
classNameParts.push('wh');
110+
}
111+
112+
if (classNameParts.length > 0) {
113+
span.className = classNameParts.join(' ');
99114
}
100-
if (!name && nodeText === '\t') span.className = 'indent';
101115
span.textContent = nodeText;
102116
ghostLine.appendChild(span);
103117
}
@@ -118,11 +132,18 @@ export class DiffRenderer {
118132
ghostRow: GhostRow,
119133
settings: EditorSettings,
120134
originalText: string,
121-
originalNodes?: HighlighedNode[]
135+
originalNodes?: HighlighedNode[],
136+
wordHighlight?: WordHighlight | null
122137
): GhostLine {
123138
const { hunkId } = ghostRow;
124139

125-
const ghostLine = this.createDeletedGhostLine(originalText, settings, hunkId, originalNodes);
140+
const ghostLine = this.createDeletedGhostLine(
141+
originalText,
142+
settings,
143+
hunkId,
144+
originalNodes,
145+
wordHighlight
146+
);
126147

127148
const emptyGutter = document.createElement('div');
128149
emptyGutter.className = 'ln';

‎anycode-base/src/renderer/LineRenderer.ts‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { HighlighedNode } from "../code";
1+
import { HighlighedNode, WordHighlight } from "../code";
22
import { AnycodeLine, objectHash } from "../utils";
33
import { EditorSettings } from "../editor";
44
import { DiffInfo } from "../diff";
@@ -23,7 +23,8 @@ export class LineRenderer {
2323
nodes: HighlighedNode[],
2424
errorLines: Map<number, string>,
2525
settings: EditorSettings,
26-
diffs?: Map<number, DiffInfo>
26+
diffs?: Map<number, DiffInfo>,
27+
wordHighlight?: WordHighlight | null
2728
): AnycodeLine {
2829
const wrapper = document.createElement('div') as AnycodeLine;
2930

@@ -50,14 +51,29 @@ export class LineRenderer {
5051
} else {
5152
for (const { name, text } of nodes) {
5253
const span = document.createElement('span');
54+
const classNameParts: string[] = [];
5355
if (name) {
5456
// Add both full token class (e.g. "function.method") and path segments
5557
// ("function", "method") so styles can gracefully fall back from specific
5658
// to general when a theme misses a deep token color.
59+
// Deduplicate classes to avoid repeating when category name has no dots.
5760
const parts = name.split('.').filter(Boolean);
58-
span.className = [name, ...parts].join(' ');
61+
classNameParts.push(...Array.from(new Set([name, ...parts])));
62+
}
63+
if (!name && text === '\t') classNameParts.push('indent');
64+
65+
// Add highlight class if it matches the wordHighlight text and is highlightable
66+
if (
67+
wordHighlight?.token &&
68+
classNameParts.includes(wordHighlight.token) &&
69+
text === wordHighlight.text
70+
) {
71+
classNameParts.push('wh');
72+
}
73+
74+
if (classNameParts.length > 0) {
75+
span.className = classNameParts.join(' ');
5976
}
60-
if (!name && text === '\t') span.className = 'indent';
6177
span.textContent = text;
6278
wrapper.appendChild(span);
6379
}
@@ -141,8 +157,9 @@ export class LineRenderer {
141157
diffs: Map<number, DiffInfo> | undefined,
142158
runLines: number[],
143159
foldIndicator: { canFold: boolean; collapsed: boolean },
160+
wordHighlight?: WordHighlight | null,
144161
): { code: AnycodeLine; gutter: HTMLDivElement; btn: HTMLDivElement; fold: HTMLDivElement } {
145-
const code = this.createLineWrapper(lineNumber, nodes, errorLines, settings, diffs);
162+
const code = this.createLineWrapper(lineNumber, nodes, errorLines, settings, diffs, wordHighlight);
146163
const gutter = this.createLineNumber(lineNumber, settings, diffs);
147164
const btn = this.createLineButtons(lineNumber, runLines, errorLines, settings);
148165
const fold = document.createElement('div');

0 commit comments

Comments
 (0)