feat(desktop): add canvas context tab

Signed-off-by: framethespace <68256458+framethespace@users.noreply.github.com>

Author: framethespace

README.md

@@ -45,6 +45,27 @@
 
 ---
 
+## Recent Desktop Additions
+
+The items in this section are the latest workflow upgrades added on top of the
+core Open CoDesign experience, so they stay separate from the baseline repo
+feature list below.
+
+- **Pinned `Canvas` tab with full Excalidraw UI** — sketch wireframes, widgets, motion notes, and layout ideas directly in the app before the first generation
+- **Canvas-to-model context export** — the canvas is packaged into prompt context automatically as a summary plus SVG exports, with frame-aware exports when the scene is too large
+- **Imported canvas images are sent separately too** — reference images dropped into the canvas also show up as standalone chat attachments for clearer model context
+- **Visible send confirmation** — the composer and chat now show when canvas context was actually included, so you can confirm it at a glance
+- **Canvas autosave and save-on-submit** — rough sketches are persisted per design, with an extra flush when you send a prompt
+- **Smart follow-up reuse** — canvas context is only resent on later turns when the canvas changed since the last successful generation
+
+<p align="center">
+  <img src="./website/Excalidraw-canvas.png" alt="Open CoDesign canvas tab with an Excalidraw wireframe and imported UI references" width="1000" />
+</p>
+
+For implementation details, see [`apps/desktop/CANVAS_CONTEXT.md`](./apps/desktop/CANVAS_CONTEXT.md).
+
+---
+
 ## What it is
 
 Turn a prompt into a polished prototype, slide deck, or marketing asset, locally, with the model you already use.

apps/desktop/CANVAS_CONTEXT.md

@@ -0,0 +1,40 @@
+# Canvas Context
+
+The desktop app now includes a pinned `Canvas` tab beside `Files`. It embeds a
+full Excalidraw surface that can be used before the first generation, so users
+can sketch wireframes, widgets, animation notes, and layout ideas before asking
+the model to build or edit the UI.
+
+## How It Works
+
+- The renderer mounts Excalidraw in `CanvasSketchView.tsx`.
+- Canvas state is stored per design through `canvas:v1:*` IPC handlers in
+  `src/main/canvas-ipc.ts`.
+- Scene JSON is persisted under the app user-data directory, alongside a small
+  list of imported local files.
+- Imported images are also surfaced as regular chat attachments so the model
+  receives both the scene context and the original image files.
+
+## Prompt Context Export
+
+Before generation, the store converts the current scene into prompt attachments:
+
+- `canvas-summary.md` with a compact summary of visible elements and labels
+- one `canvas.svg` export for the whole scene, or
+- up to four frame-specific SVG exports when Excalidraw frames are present
+
+These artifacts are written to a temp directory and attached automatically when
+the canvas contains visible content.
+
+## Current Limitation
+
+The current generation pipeline is still text-first. In practice that means the
+model receives SVG and markdown artifacts derived from the Excalidraw scene,
+plus any imported source images, rather than true bitmap-vision analysis of the
+canvas itself.
+
+## Testing Note
+
+Vitest uses a local Excalidraw shim so renderer tests stay deterministic and do
+not depend on the full browser/runtime behavior of the published Excalidraw
+bundle.

apps/desktop/package.json

@@ -18,6 +18,7 @@
     "test": "vitest run --passWithNoTests"
   },
   "dependencies": {
+    "@excalidraw/excalidraw": "^0.18.1",
     "@open-codesign/artifacts": "workspace:*",
     "@open-codesign/core": "workspace:*",
     "@open-codesign/exporters": "workspace:*",

apps/desktop/src/main/canvas-ipc.ts

@@ -0,0 +1,143 @@
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { basename, join } from 'node:path';
+import { LocalInputFile } from '@open-codesign/shared';
+import { app, ipcMain } from './electron-runtime';
+
+interface CanvasStatePayload {
+  sceneJson: string | null;
+  importedFiles: Array<{
+    path: string;
+    name: string;
+    size: number;
+  }>;
+}
+
+function canvasStateDir(designId: string): string {
+  return join(app.getPath('userData'), 'canvas-state', designId);
+}
+
+function canvasScenePath(designId: string): string {
+  return join(canvasStateDir(designId), 'scene.excalidraw.json');
+}
+
+function canvasImportsPath(designId: string): string {
+  return join(canvasStateDir(designId), 'imports.json');
+}
+
+function canvasExportDir(designId: string): string {
+  return join(app.getPath('temp'), 'open-codesign-canvas-context', designId);
+}
+
+async function readTextIfPresent(path: string): Promise<string | null> {
+  try {
+    return await readFile(path, 'utf8');
+  } catch (error) {
+    const code = (error as { code?: unknown })?.code;
+    if (code === 'ENOENT') return null;
+    throw error;
+  }
+}
+
+function requireSchemaV1(raw: unknown, channel: string): Record<string, unknown> {
+  if (typeof raw !== 'object' || raw === null) {
+    throw new Error(`${channel} expects an object payload`);
+  }
+  const record = raw as Record<string, unknown>;
+  if (record['schemaVersion'] !== 1) {
+    throw new Error(`${channel} requires schemaVersion: 1`);
+  }
+  return record;
+}
+
+function requireDesignId(record: Record<string, unknown>, channel: string): string {
+  const designId = record['designId'];
+  if (typeof designId !== 'string' || designId.trim().length === 0) {
+    throw new Error(`${channel} requires a non-empty designId`);
+  }
+  return designId;
+}
+
+function parseImportedFiles(raw: unknown): CanvasStatePayload['importedFiles'] {
+  if (!Array.isArray(raw)) return [];
+  return raw
+    .map((entry) => LocalInputFile.parse(entry))
+    .map((file) => ({ path: file.path, name: file.name, size: file.size }));
+}
+
+function sanitizeFileName(name: string): string {
+  const clean = basename(name).replace(/[^\w.\-]+/g, '-');
+  return clean.length > 0 ? clean : 'canvas-context.txt';
+}
+
+export function registerCanvasIpc(): void {
+  ipcMain.handle('canvas:v1:load-state', async (_event: unknown, raw: unknown) => {
+    const record = requireSchemaV1(raw, 'canvas:v1:load-state');
+    const designId = requireDesignId(record, 'canvas:v1:load-state');
+    const [sceneJson, importsJson] = await Promise.all([
+      readTextIfPresent(canvasScenePath(designId)),
+      readTextIfPresent(canvasImportsPath(designId)),
+    ]);
+
+    let importedFiles: CanvasStatePayload['importedFiles'] = [];
+    if (importsJson) {
+      try {
+        importedFiles = parseImportedFiles(JSON.parse(importsJson));
+      } catch {
+        importedFiles = [];
+      }
+    }
+
+    return {
+      sceneJson,
+      importedFiles,
+    } satisfies CanvasStatePayload;
+  });
+
+  ipcMain.handle('canvas:v1:save-state', async (_event: unknown, raw: unknown) => {
+    const record = requireSchemaV1(raw, 'canvas:v1:save-state');
+    const designId = requireDesignId(record, 'canvas:v1:save-state');
+    const sceneJson = record['sceneJson'];
+    if (sceneJson !== null && typeof sceneJson !== 'string') {
+      throw new Error('canvas:v1:save-state requires sceneJson to be a string or null');
+    }
+
+    const importedFiles = parseImportedFiles(record['importedFiles']);
+    await mkdir(canvasStateDir(designId), { recursive: true });
+    await Promise.all([
+      writeFile(canvasScenePath(designId), sceneJson ?? '', 'utf8'),
+      writeFile(canvasImportsPath(designId), JSON.stringify(importedFiles, null, 2), 'utf8'),
+    ]);
+    return { ok: true as const };
+  });
+
+  ipcMain.handle('canvas:v1:write-context-files', async (_event: unknown, raw: unknown) => {
+    const record = requireSchemaV1(raw, 'canvas:v1:write-context-files');
+    const designId = requireDesignId(record, 'canvas:v1:write-context-files');
+    const files = record['files'];
+    if (!Array.isArray(files)) {
+      throw new Error('canvas:v1:write-context-files requires files[]');
+    }
+    await mkdir(canvasExportDir(designId), { recursive: true });
+    const stamp = Date.now().toString(36);
+    const written = await Promise.all(
+      files.map(async (entry, index) => {
+        if (typeof entry !== 'object' || entry === null) {
+          throw new Error('canvas:v1:write-context-files received an invalid file entry');
+        }
+        const file = entry as Record<string, unknown>;
+        const name = sanitizeFileName(
+          typeof file['name'] === 'string' ? file['name'] : `canvas-context-${index + 1}.txt`,
+        );
+        const content = typeof file['content'] === 'string' ? file['content'] : '';
+        const path = join(canvasExportDir(designId), `${stamp}-${index + 1}-${name}`);
+        await writeFile(path, content, 'utf8');
+        return LocalInputFile.parse({
+          path,
+          name,
+          size: Buffer.byteLength(content, 'utf8'),
+        });
+      }),
+    );
+    return written;
+  });
+}

apps/desktop/src/renderer/src/components/CanvasSketchView.tsx

@@ -0,0 +1,136 @@
+import '@excalidraw/excalidraw/index.css';
+import { Excalidraw, serializeAsJSON } from '@excalidraw/excalidraw';
+import type { ExcalidrawElement } from '@excalidraw/excalidraw/element/types';
+import type { LocalInputFile } from '@open-codesign/shared';
+import type { ComponentProps } from 'react';
+import { useEffect, useMemo, useRef } from 'react';
+import { useCodesignStore } from '../store';
+
+type AppState = Parameters<typeof serializeAsJSON>[1];
+type BinaryFiles = Parameters<typeof serializeAsJSON>[2];
+type ExcalidrawImperativeAPI = Parameters<
+  NonNullable<ComponentProps<typeof Excalidraw>['excalidrawAPI']>
+>[0];
+
+function extractLocalInputFile(file: File): LocalInputFile | null {
+  const path = (file as File & { path?: string }).path;
+  if (typeof path !== 'string' || path.length === 0) return null;
+  return {
+    path,
+    name: file.name,
+    size: file.size,
+  };
+}
+
+export function CanvasSketchView() {
+  const currentDesignId = useCodesignStore((s) => s.currentDesignId);
+  const canvasSceneLoaded = useCodesignStore((s) => s.canvasSceneLoaded);
+  const canvasSeed = useCodesignStore((s) => s.canvasSeed);
+  const ensureCurrentDesign = useCodesignStore((s) => s.ensureCurrentDesign);
+  const loadCanvasStateForCurrentDesign = useCodesignStore((s) => s.loadCanvasStateForCurrentDesign);
+
+  const apiRef = useRef<ExcalidrawImperativeAPI | null>(null);
+  const saveTimerRef = useRef<number | null>(null);
+
+  useEffect(() => {
+    if (!window.codesign?.snapshots) return;
+    if (!currentDesignId) {
+      void ensureCurrentDesign();
+      return;
+    }
+    if (!canvasSceneLoaded) {
+      void loadCanvasStateForCurrentDesign();
+    }
+  }, [currentDesignId, canvasSceneLoaded, ensureCurrentDesign, loadCanvasStateForCurrentDesign]);
+
+  useEffect(() => {
+    const flushCanvasState = () => {
+      const api = apiRef.current;
+      if (!api) return;
+      const sceneJson = serializeAsJSON(
+        api.getSceneElementsIncludingDeleted(),
+        api.getAppState(),
+        api.getFiles(),
+        'local',
+      );
+      void useCodesignStore.getState().persistCanvasState(sceneJson);
+    };
+
+    const handleBeforeUnload = () => {
+      flushCanvasState();
+    };
+
+    window.addEventListener('beforeunload', handleBeforeUnload);
+    return () => {
+      if (saveTimerRef.current !== null) {
+        window.clearTimeout(saveTimerRef.current);
+      }
+      window.removeEventListener('beforeunload', handleBeforeUnload);
+      flushCanvasState();
+    };
+  }, []);
+
+  const initialData = useMemo(() => {
+    const canvasScene = useCodesignStore.getState().canvasScene;
+    if (!canvasScene) return null;
+    return {
+      elements: canvasScene.elements,
+      appState: canvasScene.appState,
+      files: canvasScene.files,
+    };
+  }, [canvasSeed, currentDesignId]);
+
+  if (!currentDesignId || !canvasSceneLoaded) {
+    return (
+      <div className="h-full flex items-center justify-center bg-[var(--color-background)] text-[var(--text-sm)] text-[var(--color-text-muted)]">
+        Loading canvas...
+      </div>
+    );
+  }
+
+  return (
+    <div className="h-full w-full bg-[var(--color-background)]">
+      <Excalidraw
+        key={`${currentDesignId}:${canvasSeed}`}
+        initialData={initialData}
+        excalidrawAPI={(api) => {
+          apiRef.current = api;
+        }}
+        generateIdForFile={async (file) => {
+          const localFile = extractLocalInputFile(file);
+          if (localFile) {
+            useCodesignStore.getState().addCanvasImportedFile(localFile);
+          }
+          return [
+            file.name,
+            file.size,
+            file.lastModified,
+            Math.random().toString(36).slice(2, 8),
+          ].join('-');
+        }}
+        onChange={(
+          elements: readonly ExcalidrawElement[],
+          appState: AppState,
+          files: BinaryFiles,
+        ) => {
+          useCodesignStore.getState().updateCanvasScene({
+            elements,
+            appState,
+            files,
+          });
+          if (saveTimerRef.current !== null) {
+            window.clearTimeout(saveTimerRef.current);
+          }
+          saveTimerRef.current = window.setTimeout(() => {
+            const api = apiRef.current;
+            const latestElements = api?.getSceneElementsIncludingDeleted() ?? elements;
+            const latestAppState = api?.getAppState() ?? appState;
+            const latestFiles = api?.getFiles() ?? files;
+            const sceneJson = serializeAsJSON(latestElements, latestAppState, latestFiles, 'local');
+            void useCodesignStore.getState().persistCanvasState(sceneJson);
+          }, 350);
+        }}
+      />
+    </div>
+  );
+}