fix: Improve initial file tree load performance

packages/web/src/app/api/(client)/client.ts

@@ -18,6 +18,8 @@ import {
 import {
     GetFilesRequest,
     GetFilesResponse,
+    GetFolderContentsRequest,
+    GetFolderContentsResponse,
     GetTreeRequest,
     GetTreeResponse,
 } from "@/features/fileTree/types";
@@ -101,4 +103,12 @@ export const getFiles = async (body: GetFilesRequest): Promise<GetFilesResponse
         body: JSON.stringify(body),
     }).then(response => response.json());
     return result as GetFilesResponse | ServiceError;
+}
+
+export const getFolderContents = async (body: GetFolderContentsRequest): Promise<GetFolderContentsResponse | ServiceError> => {
+    const result = await fetch("/api/folder_contents", {
+        method: "POST",
+        body: JSON.stringify(body),
+    }).then(response => response.json());
+    return result as GetFolderContentsResponse | ServiceError;
 }

packages/web/src/app/api/(server)/folder_contents/route.ts

@@ -0,0 +1,23 @@
+'use server';
+
+import { getFolderContents } from "@/features/fileTree/api";
+import { getFolderContentsRequestSchema } from "@/features/fileTree/types";
+import { schemaValidationError, serviceErrorResponse } from "@/lib/serviceError";
+import { isServiceError } from "@/lib/utils";
+import { NextRequest } from "next/server";
+
+export const POST = async (request: NextRequest) => {
+    const body = await request.json();
+    const parsed = await getFolderContentsRequestSchema.safeParseAsync(body);
+    if (!parsed.success) {
+        return serviceErrorResponse(schemaValidationError(parsed.error));
+    }
+
+    const response = await getFolderContents(parsed.data);
+    if (isServiceError(response)) {
+        return serviceErrorResponse(response);
+    }
+
+    return Response.json(response);
+}
+

packages/web/src/app/api/(server)/tree/route.ts

@@ -3,7 +3,7 @@
 import { getTree } from "@/features/fileTree/api";
 import { getTreeRequestSchema } from "@/features/fileTree/types";
 import { schemaValidationError, serviceErrorResponse } from "@/lib/serviceError";
-import { isServiceError } from "@/lib/utils";
+import { isServiceError, measure } from "@/lib/utils";
 import { NextRequest } from "next/server";
 
 export const POST = async (request: NextRequest) => {
@@ -13,7 +13,7 @@ export const POST = async (request: NextRequest) => {
         return serviceErrorResponse(schemaValidationError(parsed.error));
     }
 
-    const response = await getTree(parsed.data);
+    const { data: response } = await measure(() => getTree(parsed.data), 'getTree');
     if (isServiceError(response)) {
         return serviceErrorResponse(response);
     }

packages/web/src/features/fileTree/api.ts

@@ -8,17 +8,19 @@ import { Repo } from '@sourcebot/db';
 import { createLogger } from '@sourcebot/shared';
 import path from 'path';
 import { simpleGit } from 'simple-git';
-import { FileTreeItem, FileTreeNode } from './types';
+import { FileTreeItem } from './types';
+import { buildFileTree, isPathValid, normalizePath } from './utils';
+import { compareFileTreeItems } from './utils';
 
 const logger = createLogger('file-tree');
 
 /**
  * Returns the tree of files (blobs) and directories (trees) for a given repository,
  * at a given revision.
  */
-export const getTree = async (params: { repoName: string, revisionName: string }) => sew(() =>
+export const getTree = async (params: { repoName: string, revisionName: string, paths: string[] }) => sew(() =>
     withOptionalAuthV2(async ({ org, prisma }) => {
-        const { repoName, revisionName } = params;
+        const { repoName, revisionName, paths } = params;
         const repo = await prisma.repo.findFirst({
             where: {
                 name: repoName,
@@ -33,21 +35,30 @@ export const getTree = async (params: { repoName: string, revisionName: string }
         const { path: repoPath } = getRepoPath(repo);
 
         const git = simpleGit().cwd(repoPath);
+        if (!paths.every(path => isPathValid(path))) {
+            return notFound();
+        }
 
-        let result: string;
+        const normalizedPaths = paths.map(path => normalizePath(path));
+
+        let result: string = '';
         try {
-            result = await git.raw([
+
+            const command = [
                 // Disable quoting of non-ASCII characters in paths
                 '-c', 'core.quotePath=false',
                 'ls-tree',
                 revisionName,
-                // recursive
-                '-r',
-                // include trees when recursing
-                '-t',
                 // format as output as {type},{path}
                 '--format=%(objecttype),%(path)',
-            ]);
+                // include tree nodes
+                '-t',
+                '--',
+                '.',
+                ...normalizedPaths,
+            ];
+
+            result = await git.raw(command);
         } catch (error) {
             logger.error('git ls-tree failed.', { error });
             return unexpectedError('git ls-tree command failed.');
@@ -90,31 +101,12 @@ export const getFolderContents = async (params: { repoName: string, revisionName
         }
 
         const { path: repoPath } = getRepoPath(repo);
+        const git = simpleGit().cwd(repoPath);
 
-        // @note: we don't allow directory traversal
-        // or null bytes in the path.
-        if (path.includes('..') || path.includes('\0')) {
+        if (!isPathValid(path)) {
             return notFound();
         }
-
-        // Normalize the path by...
-        let normalizedPath = path;
-
-        // ... adding a trailing slash if it doesn't have one.
-        // This is important since ls-tree won't return the contents
-        // of a directory if it doesn't have a trailing slash.
-        if (!normalizedPath.endsWith('/')) {
-            normalizedPath = `${normalizedPath}/`;
-        }
-
-        // ... removing any leading slashes. This is needed since
-        // the path is relative to the repository's root, so we
-        // need a relative path.
-        if (normalizedPath.startsWith('/')) {
-            normalizedPath = normalizedPath.slice(1);
-        }
-
-        const git = simpleGit().cwd(repoPath);
+        const normalizedPath = normalizePath(path);
 
         let result: string;
         try {
@@ -145,6 +137,9 @@ export const getFolderContents = async (params: { repoName: string, revisionName
             }
         });
 
+        // Sort the contents in place, first by type (trees before blobs), then by name.
+        contents.sort(compareFileTreeItems);
+
         return contents;
     }));
 
@@ -199,60 +194,6 @@ export const getFiles = async (params: { repoName: string, revisionName: string
 
     }));
 
-const buildFileTree = (flatList: { type: string, path: string }[]): FileTreeNode => {
-    const root: FileTreeNode = {
-        name: 'root',
-        path: '',
-        type: 'tree',
-        children: [],
-    };
-
-    for (const item of flatList) {
-        const parts = item.path.split('/');
-        let current: FileTreeNode = root;
-
-        for (let i = 0; i < parts.length; i++) {
-            const part = parts[i];
-            const isLeaf = i === parts.length - 1;
-            const nodeType = isLeaf ? item.type : 'tree';
-            let next = current.children.find((child: FileTreeNode) => child.name === part && child.type === nodeType);
-
-            if (!next) {
-                next = {
-                    name: part,
-                    path: item.path,
-                    type: nodeType,
-                    children: [],
-                };
-                current.children.push(next);
-            }
-            current = next;
-        }
-    }
-
-    const sortTree = (node: FileTreeNode): FileTreeNode => {
-        if (node.type === 'blob') {
-            return node;
-        }
-
-        const sortedChildren = node.children
-            .map(sortTree)
-            .sort((a: FileTreeNode, b: FileTreeNode) => {
-                if (a.type !== b.type) {
-                    return a.type === 'tree' ? -1 : 1;
-                }
-                return a.name.localeCompare(b.name, undefined, { sensitivity: 'base' });
-            });
-
-        return {
-            ...node,
-            children: sortedChildren,
-        };
-    };
-
-    return sortTree(root);
-}
-
 // @todo: this is duplicated from the `getRepoPath` function in the
 // backend's `utils.ts` file. Eventually we should move this to a shared
 // package.

packages/web/src/features/fileTree/components/fileTreePanel.tsx

@@ -9,18 +9,18 @@ import { ResizablePanel } from "@/components/ui/resizable";
 import { Separator } from "@/components/ui/separator";
 import { Skeleton } from "@/components/ui/skeleton";
 import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip";
-import { unwrapServiceError } from "@/lib/utils";
+import { measure, unwrapServiceError } from "@/lib/utils";
 import { useQuery } from "@tanstack/react-query";
 import { SearchIcon } from "lucide-react";
-import { useRef } from "react";
+import { useCallback, useEffect, useRef, useState } from "react";
 import { useHotkeys } from "react-hotkeys-hook";
 import {
     GoSidebarExpand as CollapseIcon,
     GoSidebarCollapse as ExpandIcon
 } from "react-icons/go";
 import { ImperativePanelHandle } from "react-resizable-panels";
 import { PureFileTreePanel } from "./pureFileTreePanel";
-
+import { FileTreeNode } from "../types";
 
 interface FileTreePanelProps {
     order: number;
@@ -30,7 +30,6 @@ const FILE_TREE_PANEL_DEFAULT_SIZE = 20;
 const FILE_TREE_PANEL_MIN_SIZE = 10;
 const FILE_TREE_PANEL_MAX_SIZE = 30;
 
-
 export const FileTreePanel = ({ order }: FileTreePanelProps) => {
     const {
         state: {
@@ -41,17 +40,79 @@ export const FileTreePanel = ({ order }: FileTreePanelProps) => {
 
     const { repoName, revisionName, path } = useBrowseParams();
 
+    const [tree, setTree] = useState<FileTreeNode | null>(null);
+    const [openPaths, setOpenPaths] = useState<Set<string>>(new Set());
+
     const fileTreePanelRef = useRef<ImperativePanelHandle>(null);
-    const { data, isPending, isError } = useQuery({
-        queryKey: ['tree', repoName, revisionName],
-        queryFn: () => unwrapServiceError(
-            getTree({
-                repoName,
-                revisionName: revisionName ?? 'HEAD',
-            })
-        ),
+
+    const { data, isError } = useQuery({
+        queryKey: ['tree', repoName, revisionName, ...Array.from(openPaths)],
+        queryFn: async () => {
+            const result = await measure(async () => unwrapServiceError(
+                getTree({
+                    repoName,
+                    revisionName: revisionName ?? 'HEAD',
+                    paths: Array.from(openPaths),
+                })
+            ), 'getTree');
+
+            return result.data;
+        }
     });
-
+
+    useEffect(() => {
+        if (!data) {
+            return;
+        }
+        setTree(data.tree);
+    }, [data]);
+
+    // Whenever the repo name or revision name changes, we will need to
+    // reset the open paths since they no longer reference the same repository/revision.
+    useEffect(() => {
+        setOpenPaths(new Set());
+    }, [repoName, revisionName]);
+
+    // When the path changes (e.g., the user clicks a reference in the explore panel),
+    // we want this to be open and visible in the file tree.
+    useEffect(() => {
+        const pathParts = path.split('/').filter(Boolean);
+
+        setOpenPaths(current => {
+            const next = new Set<string>(current);
+            for (let i = 0; i < pathParts.length; i++) {
+                next.add(pathParts.slice(0, i + 1).join('/'));
+            }
+            return next;
+        });
+    }, [path]);
+
+    // When the user clicks a file tree node, we will want to either
+    // add or remove it from the open paths depending on if it's already open or not.
+    const onNodeClicked = useCallback((node: FileTreeNode) => {
+        if (!openPaths.has(node.path)) {
+            setOpenPaths(current => {
+                const next = new Set(current);
+                next.add(node.path);
+                return next;
+            })
+        } else {
+            setOpenPaths(current => {
+                const next = new Set(current);
+                next.delete(node.path);
+                return next;
+            })
+        }
+    }, [openPaths]);
+
+    // @debug: format the tree for console output.
+    // useEffect(() => {
+    //     if (!tree) {
+    //         return;
+    //     }
+    //     console.debug(__debugFormatTreeForConsole(tree));
+    // }, [tree]);
+
     useHotkeys("mod+b", () => {
         if (isFileTreePanelCollapsed) {
             fileTreePanelRef.current?.expand();
@@ -122,7 +183,7 @@ export const FileTreePanel = ({ order }: FileTreePanelProps) => {
                         </Tooltip>
                     </div>
                     <Separator orientation="horizontal" className="w-full mb-2" />
-                    {isPending ? (
+                    {!tree ? (
                         <FileTreePanelSkeleton />
                     ) :
                         isError ? (
@@ -131,8 +192,10 @@ export const FileTreePanel = ({ order }: FileTreePanelProps) => {
                             </div>
                         ) : (
                             <PureFileTreePanel
-                                tree={data.tree}
+                                tree={tree}
+                                openPaths={openPaths}
                                 path={path}
+                                onNodeClicked={onNodeClicked}
                             />
                         )}
                 </div>
@@ -323,4 +386,19 @@ const FileTreePanelSkeleton = () => {
             </div>
         </div>
     )
-}
+}
+
+const __debugFormatTreeForConsole = (node: FileTreeNode): string => {
+    const lines: string[] = [];
+    const walk = (current: FileTreeNode, prefix: string, isLast: boolean, isRoot: boolean) => {
+        const label = current.name || current.path;
+        const connector = isRoot ? "" : (isLast ? "`-- " : "|-- ");
+        lines.push(`${prefix}${connector}${label}`);
+        const nextPrefix = isRoot ? "" : `${prefix}${isLast ? "    " : "|   "}`;
+        current.children.forEach((child, index) => {
+            walk(child, nextPrefix, index === current.children.length - 1, false);
+        });
+    };
+    walk(node, "", true, true);
+    return lines.join("\n");
+};