feat(canvas): comment-mode annotation overlay with element and text targets

Canvas feedback was whole-document chat: describing the target in prose was
the only way to point at something. Add a comment mode to freeform canvases:
a host-authored overlay in the sandbox iframe captures element clicks and
text selections as structured, LLM-locatable targets (stable-attribute
selector, bounded text, attributes), queued annotations render as numbered
pins in the canvas and editable comment chips above the composer, and on
submit they fold into the generation instruction as a numbered
[Annotations] block the agent resolves into targeted edits of the
checked-out scratch file.

Selector inference follows the toolbar's product-tours approach (stable
attribute allowlist, nth-of-type fallback) in a dependency-free form —
precision is secondary since the consumer is the agent locating spots in
source it wrote itself.

Generated-By: PostHog Code
Task-Id: 2b3d176e-3023-41d2-92e8-81eda7c12a8c

Author: adboio

packages/core/src/canvas/freeformSchemas.ts

@@ -164,6 +164,51 @@ export type CanvasAnalyticsConfig = z.infer<typeof canvasAnalyticsConfigSchema>;
 export const canvasThemeSchema = z.enum(["light", "dark"]);
 export type CanvasTheme = z.infer<typeof canvasThemeSchema>;
 
+// ---------------------------------------------------------------------------
+// Annotation targets captured by the comment-mode overlay (structured canvas
+// feedback). Deliberately LLM-locatable rather than machine-precise: the
+// consumer is the generation agent locating the spot in source IT wrote, so
+// the bounded text + stable attributes matter more than selector exactness.
+// ---------------------------------------------------------------------------
+export const canvasElementTargetSchema = z.object({
+  type: z.literal("element"),
+  // Best-effort CSS selector for the rendered element (stable attributes
+  // preferred, nth-of-type fallback).
+  selector: z.string(),
+  tag: z.string(),
+  // Bounded innerText snippet — usually the strongest locator.
+  text: z.string(),
+  ariaLabel: z.string().nullable().optional(),
+  // The stable attributes present on the element (data-attr, id, role, …).
+  attributes: z.record(z.string(), z.string()),
+});
+export type CanvasElementTarget = z.infer<typeof canvasElementTargetSchema>;
+
+export const canvasTextRangeTargetSchema = z.object({
+  type: z.literal("text-range"),
+  // The selected text, bounded.
+  text: z.string(),
+  ancestorSelector: z.string(),
+  ancestorTag: z.string(),
+});
+export type CanvasTextRangeTarget = z.infer<typeof canvasTextRangeTargetSchema>;
+
+export const canvasAnnotationTargetSchema = z.discriminatedUnion("type", [
+  canvasElementTargetSchema,
+  canvasTextRangeTargetSchema,
+]);
+export type CanvasAnnotationTarget = z.infer<
+  typeof canvasAnnotationTargetSchema
+>;
+
+// A queued annotation the host asks the overlay to render as a numbered pin.
+export const canvasAnnotationPinSchema = z.object({
+  id: z.string(),
+  n: z.number(),
+  selector: z.string(),
+});
+export type CanvasAnnotationPin = z.infer<typeof canvasAnnotationPinSchema>;
+
 // host -> iframe
 export const hostToCanvasMessageSchema = z.discriminatedUnion("type", [
   // First frame: hand the iframe its source + the run mode. The iframe does not
@@ -199,6 +244,19 @@ export const hostToCanvasMessageSchema = z.discriminatedUnion("type", [
     result: z.unknown().optional(),
     error: z.string().optional(),
   }),
+  // Toggle the annotation ("comment mode") overlay: crosshair cursor, hover
+  // highlight, and click-to-capture targets.
+  z.object({
+    channel: z.literal(CANVAS_CHANNEL),
+    type: z.literal("set-annotation-mode"),
+    enabled: z.boolean(),
+  }),
+  // The queued annotations to render as numbered pins over their targets.
+  z.object({
+    channel: z.literal(CANVAS_CHANNEL),
+    type: z.literal("annotation-pins"),
+    pins: z.array(canvasAnnotationPinSchema),
+  }),
 ]);
 export type HostToCanvasMessage = z.infer<typeof hostToCanvasMessageSchema>;
 
@@ -254,5 +312,12 @@ export const canvasToHostMessageSchema = z.discriminatedUnion("type", [
     type: z.literal("navigate"),
     nav: canvasNavIntentSchema,
   }),
+  // A comment-mode click captured an annotation target. The comment itself is
+  // typed host-side; only the target crosses the boundary.
+  z.object({
+    channel: z.literal(CANVAS_CHANNEL),
+    type: z.literal("annotation-target"),
+    target: canvasAnnotationTargetSchema,
+  }),
 ]);
 export type CanvasToHostMessage = z.infer<typeof canvasToHostMessageSchema>;

packages/ui/src/features/canvas/freeform/CanvasFrameHost.tsx

@@ -56,6 +56,9 @@ export function CanvasFrameHost() {
                 onError={slot.inputs.onError}
                 onRendered={slot.inputs.onRendered}
                 onNavigate={slot.inputs.onNavigate}
+                annotationMode={slot.inputs.annotationMode}
+                annotationPins={slot.inputs.annotationPins}
+                onAnnotationTarget={slot.inputs.onAnnotationTarget}
               />
             </ErrorBoundary>
           </div>

packages/ui/src/features/canvas/freeform/CanvasFramePlaceholder.tsx

@@ -1,5 +1,7 @@
 import type {
   CanvasAnalyticsConfig,
+  CanvasAnnotationPin,
+  CanvasAnnotationTarget,
   CanvasNavIntent,
 } from "@posthog/core/canvas/freeformSchemas";
 import { useEffect, useLayoutEffect, useMemo, useRef } from "react";
@@ -19,6 +21,9 @@ export function CanvasFramePlaceholder({
   onError,
   onRendered,
   onNavigate,
+  annotationMode,
+  annotationPins,
+  onAnnotationTarget,
 }: {
   dashboardId: string;
   code: string;
@@ -27,6 +32,9 @@ export function CanvasFramePlaceholder({
   onError?: (message: string, stack?: string) => void;
   onRendered?: () => void;
   onNavigate?: (intent: CanvasNavIntent) => void;
+  annotationMode?: boolean;
+  annotationPins?: CanvasAnnotationPin[];
+  onAnnotationTarget?: (target: CanvasAnnotationTarget) => void;
 }) {
   const ref = useRef<HTMLDivElement>(null);
   const refreshKey = useCanvasRefreshNonce(`dashboard:${dashboardId}`);
@@ -45,6 +53,9 @@ export function CanvasFramePlaceholder({
       onError,
       onRendered,
       onNavigate,
+      annotationMode,
+      annotationPins,
+      onAnnotationTarget,
     }),
     [
       code,
@@ -54,6 +65,9 @@ export function CanvasFramePlaceholder({
       onError,
       onRendered,
       onNavigate,
+      annotationMode,
+      annotationPins,
+      onAnnotationTarget,
     ],
   );
 

packages/ui/src/features/canvas/freeform/FreeformCanvas.tsx

@@ -1,5 +1,7 @@
 import {
   type CanvasAnalyticsConfig,
+  type CanvasAnnotationPin,
+  type CanvasAnnotationTarget,
   type CanvasNavIntent,
   type CanvasToHostMessage,
   canvasToHostMessageSchema,
@@ -51,6 +53,12 @@ export interface FreeformCanvasProps {
    * reloads the frame via the existing reload path.
    */
   refreshKey?: number;
+  /** Comment mode: the overlay captures element/text targets on click. */
+  annotationMode?: boolean;
+  /** Queued annotations rendered as numbered pins over their targets. */
+  annotationPins?: CanvasAnnotationPin[];
+  /** Called when the overlay captures a target (the comment is typed host-side). */
+  onAnnotationTarget?: (target: CanvasAnnotationTarget) => void;
 }
 
 // Renders a freeform-React canvas inside a null-origin sandboxed iframe and
@@ -65,6 +73,9 @@ export function FreeformCanvas({
   onNavigate,
   analytics,
   refreshKey = 0,
+  annotationMode = false,
+  annotationPins,
+  onAnnotationTarget,
 }: FreeformCanvasProps) {
   const iframeRef = useRef<HTMLIFrameElement>(null);
   // The canvas mirrors the host's light/dark theme. Passed via `init` (not the
@@ -92,25 +103,32 @@ export function FreeformCanvas({
     onError,
     onRendered,
     onNavigate,
+    onAnnotationTarget,
     code,
     mode,
     analytics,
     theme,
+    annotationMode,
+    annotationPins,
   });
   latest.current = {
     onDataRequest,
     onError,
     onRendered,
     onNavigate,
+    onAnnotationTarget,
     code,
     mode,
     analytics,
     theme,
+    annotationMode,
+    annotationPins,
   };
 
   const postInit = useCallback(() => {
     const p = latest.current;
-    iframeRef.current?.contentWindow?.postMessage(
+    const target = iframeRef.current?.contentWindow;
+    target?.postMessage(
       {
         channel: "posthog-canvas",
         type: "init",
@@ -121,6 +139,24 @@ export function FreeformCanvas({
       },
       "*",
     );
+    // A reload resets the overlay's state; re-sync comment mode + pins so they
+    // survive srcDoc changes exactly like the theme survives via init.
+    target?.postMessage(
+      {
+        channel: "posthog-canvas",
+        type: "set-annotation-mode",
+        enabled: p.annotationMode,
+      },
+      "*",
+    );
+    target?.postMessage(
+      {
+        channel: "posthog-canvas",
+        type: "annotation-pins",
+        pins: p.annotationPins ?? [],
+      },
+      "*",
+    );
   }, []);
 
   // The iframe reloads only when srcDoc changes (mode / analytics host); on
@@ -182,6 +218,9 @@ export function FreeformCanvas({
           // msg.nav is already allowlist-validated by safeParse below.
           latest.current.onNavigate?.(msg.nav);
           break;
+        case "annotation-target":
+          latest.current.onAnnotationTarget?.(msg.target);
+          break;
       }
     };
 
@@ -232,6 +271,30 @@ export function FreeformCanvas({
     );
   }, [theme]);
 
+  // Live comment-mode + pin updates (no remount; postInit re-syncs on reload).
+  useEffect(() => {
+    if (!readyRef.current) return;
+    iframeRef.current?.contentWindow?.postMessage(
+      {
+        channel: "posthog-canvas",
+        type: "set-annotation-mode",
+        enabled: annotationMode,
+      },
+      "*",
+    );
+  }, [annotationMode]);
+  useEffect(() => {
+    if (!readyRef.current) return;
+    iframeRef.current?.contentWindow?.postMessage(
+      {
+        channel: "posthog-canvas",
+        type: "annotation-pins",
+        pins: annotationPins ?? [],
+      },
+      "*",
+    );
+  }, [annotationPins]);
+
   return (
     <iframe
       ref={iframeRef}

packages/ui/src/features/canvas/freeform/FreeformCanvasView.tsx

@@ -2,12 +2,16 @@ import {
   ArrowCounterClockwiseIcon,
   ArrowUUpLeftIcon,
   ArrowUUpRightIcon,
+  CursorClickIcon,
   ShapesIcon,
   SidebarSimpleIcon,
   SpinnerGapIcon,
   WarningIcon,
 } from "@phosphor-icons/react";
-import type { CanvasAnalyticsConfig } from "@posthog/core/canvas/freeformSchemas";
+import type {
+  CanvasAnalyticsConfig,
+  CanvasAnnotationTarget,
+} from "@posthog/core/canvas/freeformSchemas";
 import { useHostTRPC } from "@posthog/host-router/react";
 import {
   Button,
@@ -23,6 +27,10 @@ import {
   isCanvasGenerationRunning,
 } from "@posthog/ui/features/canvas/freeform/canvasGenerationStatus";
 import { useChannels } from "@posthog/ui/features/canvas/hooks/useChannels";
+import {
+  useCanvasAnnotations,
+  useCanvasAnnotationsStore,
+} from "@posthog/ui/features/canvas/stores/canvasAnnotationsStore";
 import { useCanvasChatPanelStore } from "@posthog/ui/features/canvas/stores/canvasChatPanelStore";
 import {
   useFreeformChatStore,
@@ -216,6 +224,32 @@ export function FreeformCanvasView({
   // Routes the canvas's allowlisted nav intents within this channel.
   const onNavigate = useCanvasNavigation(channelId);
 
+  // Comment mode: the overlay captures element/text targets; each becomes a
+  // queued annotation (chip + pin) that rides the next edit instruction.
+  const [annotationMode, setAnnotationMode] = useState(false);
+  const annotations = useCanvasAnnotations(dashboardId);
+  const addAnnotation = useCanvasAnnotationsStore((s) => s.add);
+  const onAnnotationTarget = useCallback(
+    (target: CanvasAnnotationTarget) => {
+      addAnnotation(dashboardId, target);
+      // Reveal the composer so the comment can be typed right away.
+      setCollapsed(false);
+    },
+    [addAnnotation, dashboardId, setCollapsed],
+  );
+  const annotationPins = useMemo(
+    () =>
+      annotations.map((a, i) => ({
+        id: a.id,
+        n: i + 1,
+        selector:
+          a.target.type === "element"
+            ? a.target.selector
+            : a.target.ancestorSelector,
+      })),
+    [annotations],
+  );
+
   // The edit composer's editor handle, so self-repair can prefill it.
   const editorRef = useRef<EditorHandle>(null);
   const askAgentToFix = () => {
@@ -309,6 +343,26 @@ export function FreeformCanvasView({
                   {isResetting ? "Resetting…" : "Reset to default"}
                 </Button>
               )}
+              {showCanvas && (
+                <Tooltip
+                  content={
+                    annotationMode
+                      ? "Exit comment mode"
+                      : "Comment on elements — click anything in the canvas"
+                  }
+                >
+                  <Button
+                    size="icon"
+                    variant={annotationMode ? "primary" : "default"}
+                    aria-label="Comment mode"
+                    className="ml-1"
+                    disabled={isGenerating}
+                    onClick={() => setAnnotationMode((v) => !v)}
+                  >
+                    <CursorClickIcon size={16} />
+                  </Button>
+                </Tooltip>
+              )}
             </Flex>
             <Flex align="center" gap="2">
               {isGenerating && effectiveTaskId ? (
@@ -383,6 +437,9 @@ export function FreeformCanvasView({
                 onError={onError}
                 onRendered={onRendered}
                 onNavigate={onNavigate}
+                annotationMode={annotationMode}
+                annotationPins={annotationPins}
+                onAnnotationTarget={onAnnotationTarget}
               />
             </Box>
           ) : (