fix: paginate Explorer CRE load and batch-hydrate all_cres

Fix production H12 timeouts from GET /all_cres?per_page=1000 by batching
N+1 link hydration in the DB layer, capping per_page at 100, scoping
DataProvider to Explorer routes with incremental page loads, and using
ensureFullExplorerData for graph views.

Closes #930. Related: #847, #848.

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: northdpole

application/database/db.py

@@ -10,7 +10,7 @@
 
 from pprint import pprint
 
-from collections import Counter
+from collections import Counter, defaultdict
 from itertools import permutations
 from typing import Any, Dict, List, Optional, Tuple, cast
 from neomodel.exceptions import (
@@ -1449,7 +1449,6 @@ def get_CREs(
         include_only: Optional[List[str]] = None,
         internal_id: Optional[str] = None,
     ) -> List[cre_defs.CRE]:
-        cres: List[cre_defs.CRE] = []
         query = CRE.query
         if not external_id and not name and not description and not internal_id:
             logger.error(
@@ -1484,65 +1483,128 @@ def get_CREs(
             )
             return []
 
-        for matching_cre in dbcres:
-            cre = CREfromDB(matching_cre)
-            cre.links = self.__make_cre_links(
-                cre=matching_cre, include_only_nodes=include_only
-            )
-            cre.links.extend(self.__make_cre_internal_links(cre=matching_cre))
-            cres.append(cre)
-        return cres
+        return self._hydrate_cres_batch(dbcres, include_only_nodes=include_only)
 
-    def __make_cre_internal_links(self, cre: CRE) -> List[cre_defs.Link]:
-        links = []
-        internal_links = (
+    def _hydrate_cres_batch(
+        self,
+        dbcres: List[CRE],
+        include_only_nodes: Optional[List[str]] = None,
+    ) -> List[cre_defs.CRE]:
+        if not dbcres:
+            return []
+
+        cre_ids = [cre.id for cre in dbcres]
+        node_links_by_cre: Dict[str, List[Links]] = defaultdict(list)
+        node_ids: set = set()
+        for link in self.session.query(Links).filter(Links.cre.in_(cre_ids)).all():
+            node_links_by_cre[link.cre].append(link)
+            node_ids.add(link.node)
+
+        nodes_by_id: Dict[str, Node] = {}
+        if node_ids:
+            nodes_by_id = {
+                node.id: node
+                for node in self.session.query(Node).filter(Node.id.in_(node_ids)).all()
+            }
+
+        internal_links_by_cre: Dict[str, List[InternalLinks]] = defaultdict(list)
+        linked_cre_ids: set = set()
+        for internal_link in (
             self.session.query(InternalLinks)
             .filter(
-                sqla.or_(InternalLinks.cre == cre.id, InternalLinks.group == cre.id)
+                sqla.or_(
+                    InternalLinks.cre.in_(cre_ids),
+                    InternalLinks.group.in_(cre_ids),
+                )
             )
             .all()
-        )
+        ):
+            linked_cre_ids.add(internal_link.cre)
+            linked_cre_ids.add(internal_link.group)
+            if internal_link.cre in cre_ids:
+                internal_links_by_cre[internal_link.cre].append(internal_link)
+            if internal_link.group in cre_ids:
+                internal_links_by_cre[internal_link.group].append(internal_link)
+
+        cres_by_id: Dict[str, CRE] = {cre.id: cre for cre in dbcres}
+        extra_cre_ids = linked_cre_ids - set(cre_ids)
+        if extra_cre_ids:
+            for linked in (
+                self.session.query(CRE).filter(CRE.id.in_(extra_cre_ids)).all()
+            ):
+                cres_by_id[linked.id] = linked
 
-        if len(internal_links) == 0:
+        result: List[cre_defs.CRE] = []
+        for matching_cre in dbcres:
+            cre = CREfromDB(matching_cre)
+            cre.links = self._assemble_cre_node_links(
+                node_links_by_cre.get(matching_cre.id, []),
+                nodes_by_id,
+                include_only_nodes,
+            )
+            seen_internal: set = set()
+            internal_rows = []
+            for row in internal_links_by_cre.get(matching_cre.id, []):
+                key = (row.cre, row.group, row.type)
+                if key not in seen_internal:
+                    seen_internal.add(key)
+                    internal_rows.append(row)
+            cre.links.extend(
+                self._assemble_cre_internal_links(
+                    matching_cre, internal_rows, cres_by_id
+                )
+            )
+            result.append(cre)
+        return result
+
+    def _assemble_cre_internal_links(
+        self,
+        cre: CRE,
+        internal_links: List[InternalLinks],
+        cres_by_id: Dict[str, CRE],
+    ) -> List[cre_defs.Link]:
+        links: List[cre_defs.Link] = []
+        if not internal_links:
             logger.debug(
                 f"CRE {cre.name}:{cre.external_id}:{cre.id} has no internal links"
             )
 
         for internal_link in internal_links:
-
-            linked_cre_query = self.session.query(CRE)
             link_type = cre_defs.LinkTypes.from_str(internal_link.type)
 
             if internal_link.cre == cre.id:
-                # if we are the lower cre in this relationship, we need to flip the "Contains" linktypes
-                linked_cre = linked_cre_query.filter(
-                    CRE.id == internal_link.group
-                ).first()  # get the higher cre so we can add the link
+                linked_cre = cres_by_id.get(internal_link.group)
+                if not linked_cre:
+                    continue
                 if link_type == cre_defs.LinkTypes.Contains:
                     links.append(
                         cre_defs.Link(
                             ltype=cre_defs.LinkTypes.PartOf,
                             document=CREfromDB(linked_cre),
                         )
                     )
-                elif (
-                    link_type == cre_defs.LinkTypes.Related
-                ):  # if it's not a "Contains" link, it's a "Related" link
+                elif link_type == cre_defs.LinkTypes.Related:
                     links.append(
                         cre_defs.Link(ltype=link_type, document=CREfromDB(linked_cre))
                     )
                 continue
-            # if we are are the higher cre then we don't need to do anything, relationship types are always "higher"->"lower"
-            linked_cre = linked_cre_query.filter(CRE.id == internal_link.cre).first()
-            links.append(cre_defs.Link(ltype=link_type, document=CREfromDB(linked_cre)))
+
+            linked_cre = cres_by_id.get(internal_link.cre)
+            if linked_cre:
+                links.append(
+                    cre_defs.Link(ltype=link_type, document=CREfromDB(linked_cre))
+                )
         return links
 
-    def __make_cre_links(
-        self, cre: CRE, include_only_nodes: List[str]
+    def _assemble_cre_node_links(
+        self,
+        node_links: List[Links],
+        nodes_by_id: Dict[str, Node],
+        include_only_nodes: Optional[List[str]],
     ) -> List[cre_defs.Link]:
-        links = []
-        for link in self.session.query(Links).filter(Links.cre == cre.id).all():
-            node = self.session.query(Node).filter(Node.id == link.node).first()
+        links: List[cre_defs.Link] = []
+        for link in node_links:
+            node = nodes_by_id.get(link.node)
             if node and (not include_only_nodes or node.name in include_only_nodes):
                 links.append(
                     cre_defs.Link(
@@ -1647,13 +1709,11 @@ def export(self, dir: str = None, dry_run: bool = False) -> List[cre_defs.Docume
     def all_cres_with_pagination(
         self, page: int = 1, per_page: int = 10
     ) -> List[cre_defs.CRE]:
-        result: List[cre_defs.CRE] = []
         cres = self.session.query(CRE).paginate(
             page=int(page), per_page=per_page, error_out=False
         )
         total_pages = cres.pages
-        for cre in cres.items:
-            result.extend(self.get_CREs(external_id=cre.external_id))
+        result = self._hydrate_cres_batch(list(cres.items))
         return result, page, total_pages
 
     def get_cre_path(self, fromID: str, toID: str) -> List[cre_defs.Document]:
@@ -2202,10 +2262,7 @@ def get_root_cres(self):
             )
             .all()
         )
-        result = []
-        for c in cres:
-            result.extend(self.get_CREs(external_id=c.external_id))
-        return result
+        return self._hydrate_cres_batch(list(cres))
 
     def get_embeddings_by_doc_type(self, doc_type: str) -> Dict[str, List[float]]:
         res = {}

application/frontend/src/App.tsx

@@ -6,7 +6,6 @@ import { QueryClient, QueryClientProvider } from 'react-query';
 import { BrowserRouter } from 'react-router-dom';
 
 import { GlobalFilterState, filterContext } from './hooks/applyFilters';
-import { DataProvider } from './providers/DataProvider';
 import { MainContentArea } from './scaffolding';
 
 const queryClient = new QueryClient();
@@ -15,12 +14,10 @@ const App = () => (
   <div className="app">
     <filterContext.Provider value={GlobalFilterState}>
       <QueryClientProvider client={queryClient}>
-        <DataProvider>
-          <BrowserRouter>
-            <Toaster />
-            <MainContentArea />
-          </BrowserRouter>
-        </DataProvider>
+        <BrowserRouter>
+          <Toaster />
+          <MainContentArea />
+        </BrowserRouter>
       </QueryClientProvider>
     </filterContext.Provider>
   </div>

application/frontend/src/const.ts

@@ -1,4 +1,5 @@
 export const TWO_DAYS_MILLISECONDS = 1.728e8;
+export const EXPLORER_CRE_PAGE_SIZE = 20;
 
 export const TYPE_IS_PART_OF = 'Is Part Of';
 export const TYPE_LINKED_TO = 'Linked To';

application/frontend/src/pages/Explorer/ExplorerLayout.tsx

@@ -0,0 +1,19 @@
+import React from 'react';
+
+import { DataProvider } from '../../providers/DataProvider';
+
+type ExplorerLayoutProps = {
+  children: React.ReactNode;
+};
+
+export const ExplorerLayout = ({ children }: ExplorerLayoutProps) => <DataProvider>{children}</DataProvider>;
+
+export function withExplorerLayout<P extends object>(Component: React.ComponentType<P>): React.FC<P> {
+  const Wrapped = (props: P) => (
+    <ExplorerLayout>
+      <Component {...props} />
+    </ExplorerLayout>
+  );
+  Wrapped.displayName = `ExplorerLayout(${Component.displayName || Component.name || 'Component'})`;
+  return Wrapped;
+}

application/frontend/src/pages/Explorer/explorer.tsx

@@ -13,11 +13,12 @@ import { GraphDebugPanel } from './GraphDebugPanel';
 import { LinkedStandards } from './LinkedStandards';
 
 export const Explorer = () => {
-  const { dataLoading, dataTree, dataStore } = useDataStore();
+  const { dataLoading, dataTree, dataStore, hasMore, isLoadingMore, loadNextPage } = useDataStore();
   const [loading, setLoading] = useState<boolean>(false);
   const [filter, setFilter] = useState('');
   const [filteredTree, setFilteredTree] = useState<TreeDocument[]>();
   const [debugMode, setDebugMode] = useState<boolean>(false);
+  const loadMoreRef = React.useRef<HTMLDivElement | null>(null);
 
   const applyHighlight = (text, term) => {
     if (!term) return text;
@@ -84,6 +85,25 @@ export const Explorer = () => {
     setLoading(dataLoading);
   }, [dataLoading]);
 
+  useEffect(() => {
+    const sentinel = loadMoreRef.current;
+    if (!sentinel || !hasMore) {
+      return;
+    }
+
+    const observer = new IntersectionObserver(
+      (entries) => {
+        if (entries.some((entry) => entry.isIntersecting)) {
+          loadNextPage();
+        }
+      },
+      { rootMargin: '200px' }
+    );
+
+    observer.observe(sentinel);
+    return () => observer.disconnect();
+  }, [hasMore, loadNextPage, filteredTree]);
+
   function processNode(item) {
     if (!item) {
       return <></>;
@@ -198,6 +218,8 @@ export const Explorer = () => {
             return processNode(item);
           })}
         </List>
+        <div ref={loadMoreRef} style={{ height: 1 }} />
+        {isLoadingMore && hasMore && <p className="explorer-load-more">Loading more requirements…</p>}
       </main>
     </>
   );

application/frontend/src/pages/Explorer/visuals/circles/circles.tsx

@@ -10,7 +10,7 @@ import { Button, Icon } from 'semantic-ui-react';
 export const ExplorerCircles = () => {
   const { height, width } = useWindowDimensions();
   const [useFullScreen, setUseFullScreen] = useState(false);
-  const { dataLoading, dataTree } = useDataStore();
+  const { dataLoading, dataTree, ensureFullExplorerData, fullLoadProgress } = useDataStore();
   const [breadcrumb, setBreadcrumb] = useState<string[]>([]);
   const svgRef = React.useRef(null);
 
@@ -21,6 +21,10 @@ export const ExplorerCircles = () => {
   const zoomToRef = React.useRef<any>(null);
   const margin = 20;
 
+  useEffect(() => {
+    ensureFullExplorerData();
+  }, [ensureFullExplorerData]);
+
   const defaultSize = width > height ? height - 100 : width;
   const size = useFullScreen ? width : defaultSize;
 
@@ -447,7 +451,10 @@ export const ExplorerCircles = () => {
           <g transform={`translate(${size / 2},${size / 2})`}></g>
         </svg>
       </div>
-      <LoadingAndErrorIndicator loading={dataLoading} error={null} />
+      <LoadingAndErrorIndicator loading={dataLoading || !!fullLoadProgress} error={null} />
+      {fullLoadProgress && (
+        <p className="explorer-full-load-progress">Loading graph data ({fullLoadProgress})…</p>
+      )}
     </div>
   );
 };

application/frontend/src/pages/Explorer/visuals/force-graph/forceGraph.tsx

@@ -22,7 +22,8 @@ export const ExplorerForceGraph = () => {
   const [ignoreTypes, setIgnoreTypes] = useState(['same']);
   const [maxCount, setMaxCount] = useState(0);
   const [maxNodeSize, setMaxNodeSize] = useState(0);
-  const { dataLoading, dataTree, getStoreKey, dataStore } = useDataStore();
+  const { dataLoading, dataTree, getStoreKey, dataStore, ensureFullExplorerData, fullLoadProgress } =
+    useDataStore();
   const fgRef = useRef<ForceGraphMethods>();
   // ADDING STATE FOR FILTERING LOGIC
   const [filterTypeA, setFilterTypeA] = useState('');
@@ -32,6 +33,10 @@ export const ExplorerForceGraph = () => {
   const [creOptions, setCreOptions] = useState<DropdownOption[]>([]);
   const [combinedOptions, setCombinedOptions] = useState<DropdownOption[]>([]);
 
+  useEffect(() => {
+    ensureFullExplorerData();
+  }, [ensureFullExplorerData]);
+
   // Adding a show all checkbox
   const [showAll, setShowAll] = useState(true);
 
@@ -431,7 +436,10 @@ export const ExplorerForceGraph = () => {
   }, [graphData]);
   return (
     <div>
-      <LoadingAndErrorIndicator loading={dataLoading} error={null} />
+      <LoadingAndErrorIndicator loading={dataLoading || !!fullLoadProgress} error={null} />
+      {fullLoadProgress && (
+        <p className="explorer-full-load-progress">Loading graph data ({fullLoadProgress})…</p>
+      )}
 
       <Checkbox
         label="Contains"