feat(review): token-level code selection for annotations (#500)

* feat(review): token-level code selection for annotations

Add the ability to click individual syntax tokens in the diff viewer to
create annotations with precise code context. Uses Pierre v1.1.12's
onTokenClick/onTokenEnter/onTokenLeave callbacks.

- Click a token → toolbar opens with token text in header
- Click same token again → deselects
- Gutter line selection still works independently
- Token metadata (charStart, charEnd, tokenText) stored on CodeAnnotation
- Sidebar shows token badge, export includes token context

For provenance purposes, this commit was AI assisted.

* fix(review): token draft roundtrip, export precision, truncation alignment

- Persist tokenSelection in drafts so token context survives file switches
- Include char offsets in exported feedback for duplicate token disambiguation
- Escape backticks in token text to prevent broken markdown fences
- Align sidebar token truncation to 30 chars (matching toolbar header)

For provenance purposes, this commit was AI assisted.

Author: backnotprop

packages/review-editor/App.tsx

@@ -18,7 +18,7 @@ import { getAgentSwitchSettings, getEffectiveAgentName } from '@plannotator/ui/u
 import { getAIProviderSettings, saveAIProviderSettings, getPreferredModel } from '@plannotator/ui/utils/aiProvider';
 import { AISetupDialog } from '@plannotator/ui/components/AISetupDialog';
 import { needsAISetup } from '@plannotator/ui/utils/aiSetup';
-import { CodeAnnotation, CodeAnnotationType, SelectedLineRange } from '@plannotator/ui/types';
+import { CodeAnnotation, CodeAnnotationType, SelectedLineRange, TokenAnnotationMeta } from '@plannotator/ui/types';
 import { useResizablePanel } from '@plannotator/ui/hooks/useResizablePanel';
 import { useCodeAnnotationDraft } from '@plannotator/ui/hooks/useCodeAnnotationDraft';
 import { useGitAdd } from './hooks/useGitAdd';
@@ -687,7 +687,8 @@ const ReviewApp: React.FC = () => {
     type: CodeAnnotationType,
     text?: string,
     suggestedCode?: string,
-    originalCode?: string
+    originalCode?: string,
+    tokenMeta?: TokenAnnotationMeta
   ) => {
     if (!pendingSelection || !files[activeFileIndex]) return;
 
@@ -706,6 +707,11 @@ const ReviewApp: React.FC = () => {
       text,
       suggestedCode,
       originalCode,
+      ...(tokenMeta && {
+        charStart: tokenMeta.charStart,
+        charEnd: tokenMeta.charEnd,
+        tokenText: tokenMeta.tokenText,
+      }),
       createdAt: Date.now(),
       author: identity,
     };

packages/review-editor/components/AnnotationToolbar.tsx

@@ -2,7 +2,7 @@ import React, { useState, useEffect, useRef } from 'react';
 import { createPortal } from 'react-dom';
 import { ToolbarState } from '../hooks/useAnnotationToolbar';
 import { useTabIndent } from '../hooks/useTabIndent';
-import { formatLineRange } from '../utils/formatLineRange';
+import { formatLineRange, formatTokenContext } from '../utils/formatLineRange';
 import { AskAIInput } from './AskAIInput';
 import { SparklesIcon } from './SparklesIcon';
 import type { AIChatEntry } from '../hooks/useAIChat';
@@ -116,7 +116,11 @@ export const AnnotationToolbar: React.FC<AnnotationToolbarProps> = ({
         <div className="w-80">
           <div className="flex items-center justify-between mb-2" {...dragHandleProps}>
             <span className="text-xs text-muted-foreground">
-              {isEditing ? 'Edit annotation' : formatLineRange(toolbarState.range.start, toolbarState.range.end)}
+              {isEditing
+                ? 'Edit annotation'
+                : toolbarState.tokenSelection
+                  ? formatTokenContext(toolbarState.tokenSelection)
+                  : formatLineRange(toolbarState.range.start, toolbarState.range.end)}
             </span>
             <button
               onClick={onCancel}

packages/review-editor/components/DiffViewer.tsx

@@ -1,7 +1,8 @@
 import React, { useMemo, useRef, useEffect, useLayoutEffect, useCallback, useState } from 'react';
 import { FileDiff, type DiffLineAnnotation } from '@pierre/diffs/react';
 import { getSingularPatch, processFile } from '@pierre/diffs';
-import { CodeAnnotation, CodeAnnotationType, SelectedLineRange, DiffAnnotationMetadata } from '@plannotator/ui/types';
+import { CodeAnnotation, CodeAnnotationType, SelectedLineRange, DiffAnnotationMetadata, TokenAnnotationMeta } from '@plannotator/ui/types';
+import type { DiffTokenEventBaseProps } from '@pierre/diffs';
 import { useTheme } from '@plannotator/ui/components/ThemeProvider';
 import { CommentPopover } from '@plannotator/ui/components/CommentPopover';
 import { storage } from '@plannotator/ui/utils/storage';
@@ -37,6 +38,9 @@ interface PierreDiffContentProps {
   onLineSelectionEnd: (range: SelectedLineRange | null) => void;
   renderAnnotation: (annotation: { side: string; lineNumber: number; metadata?: DiffAnnotationMetadata }) => React.ReactNode;
   renderHoverUtility: (getHoveredLine: () => { lineNumber: number; side: 'deletions' | 'additions' } | undefined) => React.ReactNode;
+  onTokenClick?: (props: DiffTokenEventBaseProps, event: MouseEvent) => void;
+  onTokenEnter?: (props: DiffTokenEventBaseProps, event: PointerEvent) => void;
+  onTokenLeave?: (props: DiffTokenEventBaseProps, event: PointerEvent) => void;
 }
 
 const PierreDiffContent = React.memo(({
@@ -54,6 +58,9 @@ const PierreDiffContent = React.memo(({
   onLineSelectionEnd,
   renderAnnotation,
   renderHoverUtility,
+  onTokenClick,
+  onTokenEnter,
+  onTokenLeave,
 }: PierreDiffContentProps) => {
   return (
     <FileDiff
@@ -72,6 +79,9 @@ const PierreDiffContent = React.memo(({
         enableLineSelection: true,
         enableHoverUtility: true,
         onLineSelectionEnd,
+        onTokenClick,
+        onTokenEnter,
+        onTokenLeave,
       }}
       lineAnnotations={mergedAnnotations}
       selectedLines={pendingSelection || undefined}
@@ -94,7 +104,10 @@ const PierreDiffContent = React.memo(({
   prev.pendingSelection === next.pendingSelection &&
   prev.onLineSelectionEnd === next.onLineSelectionEnd &&
   prev.renderAnnotation === next.renderAnnotation &&
-  prev.renderHoverUtility === next.renderHoverUtility
+  prev.renderHoverUtility === next.renderHoverUtility &&
+  prev.onTokenClick === next.onTokenClick &&
+  prev.onTokenEnter === next.onTokenEnter &&
+  prev.onTokenLeave === next.onTokenLeave
 ));
 
 interface DiffViewerProps {
@@ -114,7 +127,7 @@ interface DiffViewerProps {
   selectedAnnotationId: string | null;
   pendingSelection: SelectedLineRange | null;
   onLineSelection: (range: SelectedLineRange | null) => void;
-  onAddAnnotation: (type: CodeAnnotationType, text?: string, suggestedCode?: string, originalCode?: string) => void;
+  onAddAnnotation: (type: CodeAnnotationType, text?: string, suggestedCode?: string, originalCode?: string, tokenMeta?: TokenAnnotationMeta) => void;
   onAddFileComment: (text: string) => void;
   onEditAnnotation: (id: string, text?: string, suggestedCode?: string, originalCode?: string) => void;
   onSelectAnnotation: (id: string | null) => void;
@@ -441,6 +454,19 @@ export const DiffViewer: React.FC<DiffViewerProps> = ({
     );
   }, [toolbar.handleLineSelectionEnd]);
 
+  // Token interaction handlers (code area clicks)
+  const handleTokenClick = useCallback((props: DiffTokenEventBaseProps, event: MouseEvent) => {
+    toolbar.handleTokenClick(props, event);
+  }, [toolbar.handleTokenClick]);
+
+  const handleTokenEnter = useCallback((props: DiffTokenEventBaseProps) => {
+    props.tokenElement.classList.add('pn-token-hover');
+  }, []);
+
+  const handleTokenLeave = useCallback((props: DiffTokenEventBaseProps) => {
+    props.tokenElement.classList.remove('pn-token-hover');
+  }, []);
+
   // Inject resolved colors into @pierre/diffs shadow DOM.
   // CSS custom properties don't cross the shadow boundary, so we read computed
   // values and pass them via unsafeCSS. Single state object avoids split renders.
@@ -452,6 +478,7 @@ export const DiffViewer: React.FC<DiffViewerProps> = ({
       const bg = styles.getPropertyValue('--background').trim();
       const fg = styles.getPropertyValue('--foreground').trim();
       const muted = styles.getPropertyValue('--muted').trim();
+      const primary = styles.getPropertyValue('--primary').trim();
       if (!bg || !fg) return;
 
       const fontCSS = fontFamily || fontSize ? `
@@ -486,6 +513,13 @@ export const DiffViewer: React.FC<DiffViewerProps> = ({
           [data-diff-type='split'][data-overflow='scroll'] > [data-code][data-additions] [data-content] {
             min-width: 0 !important;
           }
+          .pn-token-hover {
+            text-decoration: underline;
+            text-decoration-color: ${primary || 'oklch(0.70 0.20 280)'};
+            text-decoration-thickness: 1.5px;
+            text-underline-offset: 2px;
+            cursor: pointer;
+          }
           ${fontCSS}
         `,
       });
@@ -543,6 +577,9 @@ export const DiffViewer: React.FC<DiffViewerProps> = ({
               onLineSelectionEnd={toolbar.handleLineSelectionEnd}
               renderAnnotation={renderAnnotation}
               renderHoverUtility={renderHoverUtility}
+              onTokenClick={handleTokenClick}
+              onTokenEnter={handleTokenEnter}
+              onTokenLeave={handleTokenLeave}
             />
           </div>
         </div>

packages/review-editor/components/ReviewSidebar.tsx

@@ -328,6 +328,9 @@ export const ReviewSidebar: React.FC<ReviewSidebarProps> = /* React.memo */({
                                       {annotation.lineStart === annotation.lineEnd
                                         ? `L${annotation.lineStart}`
                                         : `L${annotation.lineStart}-${annotation.lineEnd}`}
+                                      {annotation.tokenText && (
+                                        <span className="ml-1 text-primary/70">{`\`${annotation.tokenText.length > 30 ? annotation.tokenText.slice(0, 27) + '...' : annotation.tokenText}\``}</span>
+                                      )}
                                     </span>
                                   )}
                                   {annotation.author && (

packages/review-editor/dock/ReviewStateContext.tsx

@@ -1,5 +1,5 @@
 import React, { createContext, useContext } from 'react';
-import type { CodeAnnotation, CodeAnnotationType, SelectedLineRange } from '@plannotator/ui/types';
+import type { CodeAnnotation, CodeAnnotationType, SelectedLineRange, TokenAnnotationMeta } from '@plannotator/ui/types';
 import type { AgentJobInfo } from '@plannotator/ui/types';
 import type { DiffFile } from '../types';
 import type { AIChatEntry } from '../hooks/useAIChat';
@@ -33,7 +33,7 @@ export interface ReviewState {
   selectedAnnotationId: string | null;
   pendingSelection: SelectedLineRange | null;
   onLineSelection: (range: SelectedLineRange | null) => void;
-  onAddAnnotation: (type: CodeAnnotationType, text?: string, suggestedCode?: string, originalCode?: string) => void;
+  onAddAnnotation: (type: CodeAnnotationType, text?: string, suggestedCode?: string, originalCode?: string, tokenMeta?: TokenAnnotationMeta) => void;
   onAddFileComment: (text: string) => void;
   onEditAnnotation: (id: string, text?: string, suggestedCode?: string, originalCode?: string) => void;
   onSelectAnnotation: (id: string | null) => void;

packages/review-editor/hooks/useAnnotationToolbar.ts

@@ -1,19 +1,34 @@
 import { useState, useCallback, useRef, useEffect } from 'react';
-import { CodeAnnotation, SelectedLineRange, CodeAnnotationType } from '@plannotator/ui/types';
+import { CodeAnnotation, SelectedLineRange, CodeAnnotationType, TokenAnnotationMeta } from '@plannotator/ui/types';
 import { useDismissOnOutsideAndEscape } from '@plannotator/ui/hooks/useDismissOnOutsideAndEscape';
 import { extractLinesFromPatch } from '../utils/patchParser';
+import type { DiffTokenEventBaseProps } from '@pierre/diffs';
+
+export interface TokenMeta {
+  lineNumber: number;
+  charStart: number;
+  charEnd: number;
+  tokenText: string;
+  side: 'deletions' | 'additions';
+}
+
+export interface TokenSelection {
+  anchor: TokenMeta;
+  fullText: string;
+}
 
 export interface ToolbarState {
   position: { top: number; left: number };
   range: SelectedLineRange;
+  tokenSelection?: TokenSelection;
 }
 
 interface UseAnnotationToolbarArgs {
   patch: string;
   filePath: string;
   isFocused: boolean;
   onLineSelection: (range: SelectedLineRange | null) => void;
-  onAddAnnotation: (type: CodeAnnotationType, text?: string, suggestedCode?: string, originalCode?: string) => void;
+  onAddAnnotation: (type: CodeAnnotationType, text?: string, suggestedCode?: string, originalCode?: string, tokenMeta?: TokenAnnotationMeta) => void;
   onEditAnnotation: (id: string, text?: string, suggestedCode?: string, originalCode?: string) => void;
 }
 
@@ -24,6 +39,7 @@ interface Draft {
   showSuggestedCode: boolean;
   range: SelectedLineRange;
   position: { top: number; left: number };
+  tokenSelection?: TokenSelection;
 }
 
 const draftStore = new Map<string, Draft>();
@@ -38,6 +54,7 @@ function draftKey(filePath: string, range: SelectedLineRange): string {
 export function useAnnotationToolbar({ patch, filePath, isFocused, onLineSelection, onAddAnnotation, onEditAnnotation }: UseAnnotationToolbarArgs) {
   const toolbarRef = useRef<HTMLDivElement>(null);
   const lastMousePosition = useRef<{ x: number; y: number }>({ x: 0, y: 0 });
+  const tokenAnchorRef = useRef<TokenMeta | null>(null);
 
   const [toolbarState, setToolbarState] = useState<ToolbarState | null>(null);
   const [commentText, setCommentText] = useState('');
@@ -68,6 +85,7 @@ export function useAnnotationToolbar({ patch, filePath, isFocused, onLineSelecti
         ...form,
         range,
         position: toolbarStateRef.current?.position ?? { top: 0, left: 0 },
+        tokenSelection: toolbarStateRef.current?.tokenSelection,
       });
       currentDraftKeyRef.current = key;
     } else {
@@ -94,6 +112,11 @@ export function useAnnotationToolbar({ patch, filePath, isFocused, onLineSelecti
     return () => saveDraft();
   }, [saveDraft]);
 
+  // Clear token anchor on file switch
+  useEffect(() => {
+    tokenAnchorRef.current = null;
+  }, [filePath]);
+
   const resetForm = useCallback(() => {
     setToolbarState(null);
     setCommentText('');
@@ -109,19 +132,15 @@ export function useAnnotationToolbar({ patch, filePath, isFocused, onLineSelecti
     lastMousePosition.current = { x: e.clientX, y: e.clientY };
   }, []);
 
-  // Handle line selection end
-  const handleLineSelectionEnd = useCallback((range: SelectedLineRange | null) => {
-    if (!range) {
-      setToolbarState(null);
-      onLineSelection(null);
-      return;
-    }
-
-    // Save current draft before switching
+  // Shared: save current draft, restore form for new range, set toolbar state, notify parent
+  const openToolbar = useCallback((
+    range: SelectedLineRange,
+    position: { top: number; left: number },
+    tokenSelection?: TokenSelection,
+  ) => {
     saveDraft();
     setEditingAnnotationId(null);
 
-    // Restore draft for new range or start fresh
     const draft = draftStore.get(draftKey(filePath, range));
     if (draft) {
       setCommentText(draft.commentText);
@@ -133,18 +152,10 @@ export function useAnnotationToolbar({ patch, filePath, isFocused, onLineSelecti
       setShowSuggestedCode(false);
     }
 
-    const mousePos = lastMousePosition.current;
-    setToolbarState({
-      position: {
-        top: mousePos.y + 10,
-        left: mousePos.x,
-      },
-      range,
-    });
+    setToolbarState({ position, range, tokenSelection });
     currentDraftKeyRef.current = draftKey(filePath, range);
     restoreDraftKeyByFilePath.delete(filePath);
 
-    // Pre-extract original code from selected lines
     const side = range.side === 'additions' ? 'new' : 'old';
     const start = Math.min(range.start, range.end);
     const end = Math.max(range.start, range.end);
@@ -153,6 +164,20 @@ export function useAnnotationToolbar({ patch, filePath, isFocused, onLineSelecti
     onLineSelection(range);
   }, [patch, filePath, onLineSelection, saveDraft]);
 
+  // Handle line selection end (gutter clicks)
+  const handleLineSelectionEnd = useCallback((range: SelectedLineRange | null) => {
+    tokenAnchorRef.current = null;
+
+    if (!range) {
+      setToolbarState(null);
+      onLineSelection(null);
+      return;
+    }
+
+    const mousePos = lastMousePosition.current;
+    openToolbar(range, { top: mousePos.y + 10, left: mousePos.x });
+  }, [onLineSelection, openToolbar]);
+
   // Handle annotation submission (create or update)
   const handleSubmitAnnotation = useCallback(() => {
     const hasComment = commentText.trim().length > 0;
@@ -166,7 +191,13 @@ export function useAnnotationToolbar({ patch, filePath, isFocused, onLineSelecti
     if (editingAnnotationId) {
       onEditAnnotation(editingAnnotationId, text, code, original);
     } else {
-      onAddAnnotation('comment', text, code, original);
+      const tokenSel = toolbarState.tokenSelection;
+      const tokenMeta = tokenSel ? {
+        charStart: tokenSel.anchor.charStart,
+        charEnd: tokenSel.anchor.charEnd,
+        tokenText: tokenSel.fullText,
+      } : undefined;
+      onAddAnnotation('comment', text, code, original, tokenMeta);
     }
 
     clearDraft();
@@ -239,6 +270,7 @@ export function useAnnotationToolbar({ patch, filePath, isFocused, onLineSelecti
       setToolbarState({
         position: draft.position,
         range: draft.range,
+        tokenSelection: draft.tokenSelection,
       });
       currentDraftKeyRef.current = key;
       restoreDraftKeyByFilePath.delete(filePath);
@@ -251,6 +283,35 @@ export function useAnnotationToolbar({ patch, filePath, isFocused, onLineSelecti
     }
   }, [filePath, isFocused, onLineSelection, patch]);
 
+  // Handle single token click — opens toolbar for one token
+  const handleTokenClick = useCallback((props: DiffTokenEventBaseProps, event: MouseEvent) => {
+    const clickedToken: TokenMeta = {
+      lineNumber: props.lineNumber,
+      charStart: props.lineCharStart,
+      charEnd: props.lineCharEnd,
+      tokenText: props.tokenText,
+      side: props.side,
+    };
+
+    // Same token clicked twice → deselect
+    const anchor = tokenAnchorRef.current;
+    if (anchor && anchor.lineNumber === clickedToken.lineNumber
+      && anchor.charStart === clickedToken.charStart
+      && anchor.side === clickedToken.side) {
+      tokenAnchorRef.current = null;
+      setToolbarState(null);
+      onLineSelection(null);
+      return;
+    }
+
+    tokenAnchorRef.current = clickedToken;
+    openToolbar(
+      { start: clickedToken.lineNumber, end: clickedToken.lineNumber, side: clickedToken.side },
+      { top: event.clientY + 10, left: event.clientX },
+      { anchor: clickedToken, fullText: clickedToken.tokenText },
+    );
+  }, [onLineSelection, openToolbar]);
+
   return {
     // State
     toolbarState,
@@ -271,6 +332,7 @@ export function useAnnotationToolbar({ patch, filePath, isFocused, onLineSelecti
     // Handlers
     handleMouseMove,
     handleLineSelectionEnd,
+    handleTokenClick,
     handleSubmitAnnotation,
     handleDismiss,
     handleCancel,