ENG-990 Use all pages query from supabase not using roam (#521)

* fix a query bug, use supabase t get all nodes

* use getnodesbytype

Author: sid597

apps/roam/src/utils/getAllDiscourseNodesSince.ts

@@ -75,21 +75,21 @@ export const getAllDiscourseNodesSince = async (
   const result: RoamDiscourseNodeData[] = [];
 
   const query = `[
-  :find ?node-title ?uid ?nodeCreateTime ?nodeEditTime ?author_local_id ?author_name
-  :keys text source_local_id created last_modified author_local_id author_name
-  :in $ ?since 
-  :where
-    [?node :node/title ?node-title]
-    [?node :block/uid ?uid]
-    [?node :create/time ?nodeCreateTime]
-    [?node :edit/time ?nodeEditTime]
-    [?node :create/user ?user-eid]
-    [?user-eid :user/uid ?author_local_id]
-    [?node :edit/user ?eu]
-    [(get-else $ ?eu :user/display-name "Unknown-person") ?author_name]
-    [(> ?nodeEditTime ?since)]
-]`;
 
+    :find ?node-title ?uid ?nodeCreateTime ?nodeEditTime ?author_local_id ?author_name
+    :keys text source_local_id created last_modified author_local_id author_name
+    :in $ ?since 
+    :where
+      [?node :node/title ?node-title]
+      [?node :block/uid ?uid]
+      [?node :create/time ?nodeCreateTime]
+      [?node :create/user ?user-eid]
+      [?user-eid :user/uid ?author_local_id]
+      [(get-else $ ?user-eid :user/display-name "Unknown-Creator") ?author_name]
+      [(get-else $ ?node :edit/time 0) ?nodeEditTime]
+      [(get-else $ ?node :edit/time ?nodeCreateTime) ?filterTime]
+      [(> ?filterTime ?since)]
+  ]`;
   const allNodes = (await Promise.resolve(
     window.roamAlphaAPI.data.backend.q(query, sinceMs),
   )) as unknown as RoamDiscourseNodeData[];

apps/roam/src/utils/hyde.ts

@@ -1,10 +1,11 @@
-import { getLoggedInClient } from "./supabaseContext";
+import { getLoggedInClient, getSupabaseContext } from "./supabaseContext";
 import { Result } from "./types";
 import normalizePageTitle from "roamjs-components/queries/normalizePageTitle";
 import findDiscourseNode from "./findDiscourseNode";
 import { nextApiRoot } from "@repo/utils/execContext";
 import { DiscourseNode } from "./getDiscourseNodes";
 import getExtensionAPI from "roamjs-components/util/extensionApiContext";
+import { getAllNodes } from "@repo/database/lib/queries";
 
 type ApiEmbeddingResponse = {
   data: Array<{
@@ -455,26 +456,30 @@ export const performHydeSearch = async ({
   );
 
   if (useAllPagesForSuggestions) {
-    // TODO: Use Supabase to get all pages
-    candidateNodesForHyde = (await getAllPageByUidAsync())
-      .map(([pageName, pageUid]) => {
-        if (!pageUid || pageUid === blockUid) return null;
-        const node = findDiscourseNode(pageUid);
-        if (
-          !node ||
-          node.backedBy === "default" ||
-          !validTypes.includes(node.type) ||
-          existingUids.has(pageUid)
-        ) {
-          return null;
-        }
+    const context = await getSupabaseContext();
+    if (!context) return [];
+    const supabase = await getLoggedInClient();
+    const spaceId = context.spaceId;
+    if (!supabase) return [];
+
+    candidateNodesForHyde = (
+      await getAllNodes({
+        supabase,
+        spaceId,
+        fields: { content: ["source_local_id", "text"] },
+        ofTypes: validTypes,
+        pagination: { limit: 10000 },
+      })
+    )
+      .map((c) => {
+        const node = findDiscourseNode(c.Content?.source_local_id || "");
         return {
-          uid: pageUid,
-          text: pageName,
-          type: node.type,
-        } as SuggestedNode;
+          uid: c.Content?.source_local_id || "",
+          text: c.Content?.text || "",
+          type: node ? node.type : "",
+        };
       })
-      .filter((n): n is SuggestedNode => n !== null);
+      .filter((n) => n.uid && n.text && n.type);
   } else {
     const referenced: { uid: string; text: string }[] = [];
     if (shouldGrabFromReferencedPages) {

packages/database/src/lib/queries.ts

@@ -325,7 +325,7 @@ const composeConceptQuery = ({
     q += `, relations:concept_in_relations!inner(${args.join(",\n")})`;
   }
   let query = supabase.from("my_concepts").select(q);
-  if (scope.type === 'nodes') {
+  if (scope.type === "nodes") {
     query = query.eq("arity", 0);
   } else if (scope.type === 'relations') {
     query = query.gt("arity", 0);