Merge pull request framer#406 from framer/global-search/index-changes

Global search: Indexer watches for change

Author: elmarburke

plugins/global-search/src/App.tsx

@@ -28,7 +28,7 @@ export function App() {
 
     return (
         <ErrorBoundary FallbackComponent={ErrorScene}>
-            <IndexerProvider projectId={projectInfo.id}>
+            <IndexerProvider projectId={projectInfo.id} projectName={projectInfo.name}>
                 {activeScene === "dev-tools" && <DevToolsScene />}
                 {activeScene === "search" && <SearchScene />}
             </IndexerProvider>

plugins/global-search/src/components/SearchScene.tsx

@@ -8,6 +8,7 @@ import type { RootNodeType } from "../utils/indexer/types"
 import { useIndexer } from "../utils/indexer/useIndexer"
 import { entries } from "../utils/object"
 import { getPluginUiOptions } from "../utils/plugin-ui"
+import { useMinimumDuration } from "../utils/useMinimumDuration"
 import { NoResults } from "./NoResults"
 import { ResultsList } from "./Results"
 import { SearchInput } from "./SearchInput"
@@ -20,6 +21,7 @@ export function SearchScene() {
     const [query, setQuery] = useState("")
     const { searchOptions, optionsMenuItems } = useOptionsMenuItems()
     const deferredQuery = useDeferredValue(query)
+    const isIndexingWithMinimumDuration = useMinimumDuration(isIndexing, 500)
 
     const { results, hasResults, error: filterError } = useAsyncFilter(deferredQuery, searchOptions, db, dataVersion)
 
@@ -46,15 +48,15 @@ export function SearchScene() {
                     )}
                 >
                     <SearchInput value={query} onChange={handleQueryChange} />
-                    {isIndexing && (
-                        // TODO: Discuss if we should add a tooltip to explain what's this.
-                        <span
-                            title="Indexing..."
-                            className="animate-[fade-in_150ms_forwards] [animation-delay:500ms] opacity-0"
-                        >
-                            <IconSpinner className="text-black dark:text-white animate-[spin_0.8s_linear_infinite]" />
-                        </span>
-                    )}
+
+                    <span
+                        title="Indexing..."
+                        className="aria-hidden:opacity-0 transition"
+                        aria-hidden={!isIndexingWithMinimumDuration}
+                    >
+                        <IconSpinner className="text-black dark:text-white animate-[spin_0.8s_linear_infinite]" />
+                    </span>
+
                     <Menu items={optionsMenuItems}>
                         <IconEllipsis className="text-framer-text-tertiary-light dark:text-framer-text-tertiary-dark" />
                     </Menu>

plugins/global-search/src/utils/db.ts

@@ -11,6 +11,7 @@ interface GlobalSearchDB extends DBSchema {
             type: string
             rootNodeType: string
             addedInIndexRun: number
+            rootNodeIdVersion: [string, number]
         }
     }
 }
@@ -19,19 +20,27 @@ export class GlobalSearchDatabase implements ResumableAsyncIterable<IndexEntry>
     private db: IDBPDatabase<GlobalSearchDB> | null = null
     private readonly dbName: string
 
-    constructor(projectId: string) {
-        this.dbName = `global-search-${projectId}`
+    constructor(projectId: string, projectName: string) {
+        this.dbName = `global-search-${projectName}-${projectId}`
     }
 
     async open(): Promise<IDBPDatabase<GlobalSearchDB>> {
         if (this.db) return this.db
 
-        this.db = await openDB<GlobalSearchDB>(this.dbName, 1, {
-            upgrade(db) {
-                const entriesStore = db.createObjectStore("entries", { keyPath: "id" })
-                entriesStore.createIndex("rootNodeType", "rootNodeType")
-                entriesStore.createIndex("type", "type")
-                entriesStore.createIndex("addedInIndexRun", "addedInIndexRun")
+        this.db = await openDB<GlobalSearchDB>(this.dbName, 2, {
+            upgrade(db, oldVersion, _newVersion, transaction) {
+                if (oldVersion < 1) {
+                    const entriesStore = db.createObjectStore("entries", { keyPath: "id" })
+                    entriesStore.createIndex("rootNodeType", "rootNodeType")
+                    entriesStore.createIndex("type", "type")
+                    entriesStore.createIndex("addedInIndexRun", "addedInIndexRun")
+                }
+
+                if (oldVersion < 2) {
+                    const entriesStore = transaction.objectStore("entries")
+                    // Add compound index for [rootNodeId, addedInIndexRun] - this is all we need
+                    entriesStore.createIndex("rootNodeIdVersion", ["rootNodeId", "addedInIndexRun"])
+                }
             },
         })
 
@@ -109,4 +118,25 @@ export class GlobalSearchDatabase implements ResumableAsyncIterable<IndexEntry>
 
         await tx.done
     }
+
+    /**
+     * Removes entries for a specific root node from a specific index run version.
+     *
+     * This is used for incremental updates when a specific canvas root changes.
+     */
+    async clearEntriesForRootNodeAndSpecificVersion(rootNodeId: string, version: number): Promise<void> {
+        const db = await this.open()
+        const tx = db.transaction("entries", "readwrite")
+        const store = tx.objectStore("entries")
+        const index = store.index("rootNodeIdVersion")
+
+        // Use compound index to efficiently find entries with exact [rootNodeId, version] match
+        let cursor = await index.openCursor(IDBKeyRange.only([rootNodeId, version]))
+        while (cursor) {
+            await cursor.delete()
+            cursor = await cursor.continue()
+        }
+
+        await tx.done
+    }
 }

plugins/global-search/src/utils/indexer/IndexerProvider.tsx

@@ -8,16 +8,25 @@ import { GlobalSearchIndexer } from "./indexer"
  * Creates database and indexer instances and provides them to the children.
  * The database manages its own version and the indexer depends on the database.
  */
-export function IndexerProvider({ children, projectId }: { children: React.ReactNode; projectId: string }) {
+export function IndexerProvider({
+    children,
+    projectId,
+    projectName,
+}: {
+    children: React.ReactNode
+    projectId: string
+    projectName: string
+}) {
     const dbRef = useRef<GlobalSearchDatabase>()
-    dbRef.current ??= new GlobalSearchDatabase(projectId)
+    dbRef.current ??= new GlobalSearchDatabase(projectId, projectName)
     const db = dbRef.current
 
     const indexerRef = useRef<GlobalSearchIndexer>()
     indexerRef.current ??= new GlobalSearchIndexer(db)
     const indexer = indexerRef.current
 
     const [isIndexing, setIsIndexing] = useState(false)
+    const [isCanvasRootChanging, setIsCanvasRootChanging] = useState(false)
     const [dataVersion, setDataVersion] = useState(0)
 
     useEffect(() => {
@@ -42,12 +51,14 @@ export function IndexerProvider({ children, projectId }: { children: React.React
         const onAborted = () => {
             startTransition(() => {
                 setIsIndexing(false)
+                setIsCanvasRootChanging(false)
             })
         }
 
         const onError = ({ error }: IndexerEvents["error"]) => {
             startTransition(() => {
                 setIsIndexing(false)
+                setIsCanvasRootChanging(false)
                 console.error(error)
             })
         }
@@ -58,13 +69,28 @@ export function IndexerProvider({ children, projectId }: { children: React.React
             })
         }
 
+        const onCanvasRootChangeStarted = () => {
+            startTransition(() => {
+                setIsCanvasRootChanging(true)
+            })
+        }
+
+        const onCanvasRootChangeCompleted = () => {
+            startTransition(() => {
+                setIsCanvasRootChanging(false)
+                setDataVersion(prev => prev + 1)
+            })
+        }
+
         const unsubscribes = [
             indexer.on("progress", onProgress),
             indexer.on("restarted", onRestarted),
             indexer.on("aborted", onAborted),
             indexer.on("started", onStarted),
             indexer.on("completed", onCompleted),
             indexer.on("error", onError),
+            indexer.on("canvasRootChangeStarted", onCanvasRootChangeStarted),
+            indexer.on("canvasRootChangeCompleted", onCanvasRootChangeCompleted),
         ]
 
         void indexer.start()
@@ -75,8 +101,8 @@ export function IndexerProvider({ children, projectId }: { children: React.React
     }, [indexer, db])
 
     const data = useMemo(
-        () => ({ isIndexing, indexerInstance: indexer, db, dataVersion }),
-        [isIndexing, indexer, db, dataVersion]
+        () => ({ isIndexing: isIndexing || isCanvasRootChanging, indexerInstance: indexer, db, dataVersion }),
+        [isIndexing, isCanvasRootChanging, indexer, db, dataVersion]
     )
 
     return <IndexerContext.Provider value={data}>{children}</IndexerContext.Provider>

plugins/global-search/src/utils/indexer/indexer.ts

@@ -1,5 +1,6 @@
 import {
     type AnyNode,
+    type CanvasRootNode,
     type Collection,
     framer,
     isComponentNode,
@@ -42,11 +43,12 @@ async function getNodeName(node: AnyNode): Promise<string | null> {
 
 export interface IndexerEvents extends EventMap {
     error: { error: Error }
-    progress: { processed: number; total?: number }
     started: { indexRun: number }
     completed: never
     restarted: never
     aborted: never
+    canvasRootChangeStarted: never
+    canvasRootChangeCompleted: never
 }
 
 export class GlobalSearchIndexer {
@@ -58,6 +60,8 @@ export class GlobalSearchIndexer {
     // A smaller batch size will make showing results faster, but will also make the UI more laggy.
     private batchSize = 100
     private abortRequested = false
+    private canvasSubscription: (() => void) | null = null
+    private currentCanvasRootChangeAbortController: AbortController | null = null
 
     constructor(private db: GlobalSearchDatabase) {}
 
@@ -165,35 +169,84 @@ export class GlobalSearchIndexer {
         }
     }
 
+    private async handleCanvasRootChange(rootNode: CanvasRootNode) {
+        if (this.abortRequested) return
+
+        this.currentCanvasRootChangeAbortController?.abort()
+
+        const abortController = new AbortController()
+        this.currentCanvasRootChangeAbortController = abortController
+
+        this.eventEmitter.emit("canvasRootChangeStarted")
+
+        try {
+            if (abortController.signal.aborted) return
+
+            const lastIndexRun = await this.db.getLastIndexRun()
+            const currentIndexRun = lastIndexRun + 1
+            await this.processNodes(currentIndexRun, [rootNode], abortController.signal)
+            await this.db.clearEntriesForRootNodeAndSpecificVersion(rootNode.id, lastIndexRun)
+        } catch (error) {
+            this.eventEmitter.emit("error", { error: error instanceof Error ? error : new Error(String(error)) })
+        } finally {
+            if (this.currentCanvasRootChangeAbortController === abortController) {
+                this.currentCanvasRootChangeAbortController = null
+            }
+            this.eventEmitter.emit("canvasRootChangeCompleted")
+        }
+    }
+
+    private async processNodes(
+        currentIndexRun: number,
+        rootNodes: readonly CanvasRootNode[],
+        abortSignal?: AbortSignal
+    ) {
+        const validRootNodes = rootNodes.filter(rootNode => isComponentNode(rootNode) || isWebPageNode(rootNode))
+
+        for await (const batch of this.crawlNodes(currentIndexRun, validRootNodes)) {
+            if (this.abortRequested || abortSignal?.aborted) break
+            await this.db.upsertEntries(batch)
+        }
+    }
+
+    private async processCollections(currentIndexRun: number) {
+        const collections = await framer.getCollections()
+
+        for await (const batch of this.crawlCollections(currentIndexRun, collections)) {
+            if (this.abortRequested) break
+            await this.db.upsertEntries(batch)
+        }
+    }
+
     async start() {
         // XXX: The indexer has no "locking mechanism" to prevent multiple instances from running at the same time in multiple tabs.
         try {
             const lastIndexRun = await this.db.getLastIndexRun()
             const currentIndexRun = lastIndexRun + 1
 
-            const [pages, components] = await Promise.all([
+            const [pages, components, canvasRoot] = await Promise.all([
                 framer.getNodesWithType("WebPageNode"),
                 framer.getNodesWithType("ComponentNode"),
+                framer.getCanvasRoot(),
             ])
 
             this.abortRequested = false
             this.eventEmitter.emit("started", { indexRun: currentIndexRun })
 
-            for await (const batch of this.crawlNodes(currentIndexRun, [...pages, ...components])) {
-                // this isn't a unnecassary static expression, as the value could change during the async loop
-                // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
-                if (this.abortRequested) break
-                await this.db.upsertEntries(batch)
-            }
+            this.canvasSubscription ??= framer.subscribeToCanvasRoot(rootNode => {
+                void this.handleCanvasRootChange(rootNode)
+            })
 
-            const collections = await framer.getCollections()
+            // Remove the current open canvas root from the list of root nodes to index
+            // as it's already being indexed by the canvas root watcher
+            const rootNodesWithoutCurrentRoot = [...pages, ...components].filter(
+                rootNode => rootNode.id !== canvasRoot.id
+            )
 
-            for await (const batch of this.crawlCollections(currentIndexRun, collections)) {
-                // this isn't a unnecassary static expression, as the value could change during the async loop
-                // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
-                if (this.abortRequested) break
-                await this.db.upsertEntries(batch)
-            }
+            await Promise.all([
+                this.processNodes(currentIndexRun, rootNodesWithoutCurrentRoot),
+                this.processCollections(currentIndexRun),
+            ])
 
             // this isn't a unnecassary static expression, as the value could change during the async loop
             // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
@@ -207,13 +260,24 @@ export class GlobalSearchIndexer {
     }
 
     async restart() {
-        this.abortRequested = true
+        this.abort()
+
         this.eventEmitter.emit("restarted")
         return this.start()
     }
 
     abort() {
         this.abortRequested = true
+
+        this.currentCanvasRootChangeAbortController?.abort()
+        this.currentCanvasRootChangeAbortController = null
+
+        if (this.canvasSubscription) {
+            this.canvasSubscription()
+            this.canvasSubscription = null
+        }
+
+        this.eventEmitter.emit("aborted")
     }
 
     /**