Eng 1343 f10a sync node schema definitions (#720)

* ENG-1343 upload obsidian node schema to supabase

Author: maparent

apps/obsidian/src/utils/conceptConversion.ts

@@ -1,10 +1,10 @@
 /* eslint-disable @typescript-eslint/naming-convention */
-import { TFile } from "obsidian";
-import { DiscourseNode } from "~/types";
-import { SupabaseContext } from "./supabaseContext";
-import { LocalConceptDataInput } from "@repo/database/inputTypes";
-import { ObsidianDiscourseNodeData } from "./syncDgNodesToSupabase";
-import { Json } from "@repo/database/dbTypes";
+import type { TFile } from "obsidian";
+import type { DiscourseNode } from "~/types";
+import type { SupabaseContext } from "./supabaseContext";
+import type { LocalConceptDataInput } from "@repo/database/inputTypes";
+import type { ObsidianDiscourseNodeData } from "./syncDgNodesToSupabase";
+import type { Json } from "@repo/database/dbTypes";
 
 /**
  * Get extra data (author, timestamps) from file metadata
@@ -33,16 +33,22 @@ export const discourseNodeSchemaToLocalConcept = ({
   node: DiscourseNode;
   accountLocalId: string;
 }): LocalConceptDataInput => {
-  const now = new Date().toISOString();
+  const { description, template, id, name, created, modified, ...otherData } =
+    node;
   return {
     space_id: context.spaceId,
-    name: node.name,
-    source_local_id: node.id,
+    name: name,
+    source_local_id: id,
     is_schema: true,
     author_local_id: accountLocalId,
-    created: now,
-    // TODO: get the template or any other info to put into literal_content jsonb
-    last_modified: now,
+    created: new Date(created).toISOString(),
+    last_modified: new Date(modified).toISOString(),
+    description: description,
+    literal_content: {
+      label: name,
+      template: template,
+      source_data: otherData,
+    },
   };
 };
 
@@ -59,22 +65,19 @@ export const discourseNodeInstanceToLocalConcept = ({
   accountLocalId: string;
 }): LocalConceptDataInput => {
   const extraData = getNodeExtraData(nodeData.file, accountLocalId);
-  console.log(nodeData.frontmatter);
-  const concept = {
+  const { nodeInstanceId, nodeTypeId, ...otherData } = nodeData.frontmatter;
+  return {
     space_id: context.spaceId,
-    name: nodeData.file.basename,
-    source_local_id: nodeData.nodeInstanceId,
-    schema_represented_by_local_id: nodeData.nodeTypeId,
+    name: nodeData.file.path,
+    source_local_id: nodeInstanceId as string,
+    schema_represented_by_local_id: nodeTypeId as string,
     is_schema: false,
     literal_content: {
-      ...nodeData.frontmatter,
-    } as unknown as Json,
+      label: nodeData.file.basename,
+      source_data: otherData as unknown as Json,
+    },
     ...extraData,
   };
-  console.log(
-    `[discourseNodeInstanceToLocalConcept] Converting concept: source_local_id=${nodeData.nodeInstanceId}, name="${nodeData.file.basename}"`,
-  );
-  return concept;
 };
 
 export const relatedConcepts = (concept: LocalConceptDataInput): string[] => {
@@ -135,6 +138,7 @@ export const orderConceptsByDependency = (
       ...missing,
       ...orderConceptsRec(ordered, first, conceptById),
     ]);
+    if (missing.size > 0) console.error(`missing: ${[...missing]}`);
   }
   return { ordered, missing: Array.from(missing) };
 };

apps/obsidian/src/utils/syncDgNodesToSupabase.ts

@@ -1,21 +1,22 @@
 /* eslint-disable @typescript-eslint/naming-convention */
 import { Notice, TFile } from "obsidian";
-import { DGSupabaseClient } from "@repo/database/lib/client";
-import { Json } from "@repo/database/dbTypes";
+import type { DGSupabaseClient } from "@repo/database/lib/client";
+import type { Json } from "@repo/database/dbTypes";
 import {
   getSupabaseContext,
   getLoggedInClient,
-  SupabaseContext,
+  type SupabaseContext,
 } from "./supabaseContext";
 import { default as DiscourseGraphPlugin } from "~/index";
 import { upsertNodesToSupabaseAsContentWithEmbeddings } from "./upsertNodesAsContentWithEmbeddings";
 import {
   orderConceptsByDependency,
   discourseNodeInstanceToLocalConcept,
+  discourseNodeSchemaToLocalConcept,
 } from "./conceptConversion";
-import { LocalConceptDataInput } from "@repo/database/inputTypes";
+import type { LocalConceptDataInput } from "@repo/database/inputTypes";
 
-const DEFAULT_TIME = new Date("1970-01-01");
+const DEFAULT_TIME = "1970-01-01";
 export type ChangeType = "title" | "content";
 
 export type ObsidianDiscourseNodeData = {
@@ -101,7 +102,10 @@ const deleteNodesFromSupabase = async (
     if (conceptDeleteError) {
       result.success = false;
       result.errors.concept = conceptDeleteError;
-      console.error("Failed to delete concepts from Supabase:", conceptDeleteError);
+      console.error(
+        "Failed to delete concepts from Supabase:",
+        conceptDeleteError,
+      );
     }
 
     const { error: contentDeleteError } = await supabaseClient
@@ -159,7 +163,8 @@ const ensureNodeInstanceId = async (
 
   return nodeInstanceId;
 };
-const getLastSyncTime = async (
+
+const getLastContentSyncTime = async (
   supabaseClient: DGSupabaseClient,
   spaceId: number,
 ): Promise<Date> => {
@@ -170,7 +175,22 @@ const getLastSyncTime = async (
     .order("last_modified", { ascending: false })
     .limit(1)
     .maybeSingle();
-  return new Date(data?.last_modified || DEFAULT_TIME);
+  return new Date((data?.last_modified || DEFAULT_TIME) + "Z");
+};
+
+const getLastSchemaSyncTime = async (
+  supabaseClient: DGSupabaseClient,
+  spaceId: number,
+): Promise<Date> => {
+  const { data } = await supabaseClient
+    .from("Concept")
+    .select("last_modified")
+    .eq("space_id", spaceId)
+    .eq("is_schema", true)
+    .order("last_modified", { ascending: false })
+    .limit(1)
+    .maybeSingle();
+  return new Date((data?.last_modified || DEFAULT_TIME) + "Z");
 };
 
 type DiscourseNodeInVault = {
@@ -195,7 +215,6 @@ const mergeChangeTypes = (
   return Array.from(merged);
 };
 
-
 /**
  * Step 1: Collect all discourse nodes from the vault
  * Filters markdown files that have nodeTypeId in frontmatter
@@ -362,7 +381,10 @@ const buildChangedNodesFromNodes = async ({
     nodeInstanceIds,
   );
 
-  const lastSyncTime = await getLastSyncTime(supabaseClient, context.spaceId);
+  const lastSyncTime = await getLastContentSyncTime(
+    supabaseClient,
+    context.spaceId,
+  );
   const changedNodes: ObsidianDiscourseNodeData[] = [];
 
   for (const node of nodes) {
@@ -466,12 +488,20 @@ export const createOrUpdateDiscourseEmbedding = async (
       throw new Error("accountLocalId not found in plugin settings");
     }
 
-    await syncChangedNodesToSupabase({
-      changedNodes: allNodeInstances,
+    await upsertNodesToSupabaseAsContentWithEmbeddings({
+      obsidianNodes: allNodeInstances,
+      supabaseClient,
+      context,
+      accountLocalId,
       plugin,
+    });
+
+    await convertDgToSupabaseConcepts({
+      nodesSince: allNodeInstances,
       supabaseClient,
       context,
       accountLocalId,
+      plugin,
     });
 
     console.debug("Sync completed successfully");
@@ -486,43 +516,59 @@ const convertDgToSupabaseConcepts = async ({
   supabaseClient,
   context,
   accountLocalId,
+  plugin,
 }: {
   nodesSince: ObsidianDiscourseNodeData[];
   supabaseClient: DGSupabaseClient;
   context: SupabaseContext;
   accountLocalId: string;
+  plugin: DiscourseGraphPlugin;
 }): Promise<void> => {
-  // TODO: handling schema (node types and relations) will be handled in the future by ENG-1181
-  // Schema upsert will need allNodeTypes parameter when enabled
+  const lastSchemaSync = (
+    await getLastSchemaSyncTime(supabaseClient, context.spaceId)
+  ).getTime();
+  const newNodeTypes = (plugin.settings.nodeTypes ?? []).filter(
+    (n) => n.modified > lastSchemaSync,
+  );
 
-  const nodeInstanceToLocalConcepts = nodesSince.map((node) => {
-    return discourseNodeInstanceToLocalConcept({
+  const nodesTypesToLocalConcepts = newNodeTypes.map((nodeType) =>
+    discourseNodeSchemaToLocalConcept({
+      context,
+      node: nodeType,
+      accountLocalId,
+    }),
+  );
+
+  const nodeInstanceToLocalConcepts = nodesSince.map((node) =>
+    discourseNodeInstanceToLocalConcept({
       context,
       nodeData: node,
       accountLocalId,
-    });
-  });
+    }),
+  );
 
   const conceptsToUpsert: LocalConceptDataInput[] = [
-    // ...nodesTypesToLocalConcepts,
+    ...nodesTypesToLocalConcepts,
     ...nodeInstanceToLocalConcepts,
   ];
 
-  const { ordered } = orderConceptsByDependency(conceptsToUpsert);
+  if (conceptsToUpsert.length > 0) {
+    const { ordered } = orderConceptsByDependency(conceptsToUpsert);
 
-  const { error } = await supabaseClient.rpc("upsert_concepts", {
-    data: ordered as Json,
-    v_space_id: context.spaceId,
-  });
+    const { error } = await supabaseClient.rpc("upsert_concepts", {
+      data: ordered as Json,
+      v_space_id: context.spaceId,
+    });
 
-  if (error) {
-    const errorMessage =
-      error instanceof Error
-        ? error.message
-        : typeof error === "string"
-          ? error
-          : JSON.stringify(error, null, 2);
-    throw new Error(`upsert_concepts failed: ${errorMessage}`);
+    if (error) {
+      const errorMessage =
+        error instanceof Error
+          ? error.message
+          : typeof error === "string"
+            ? error
+            : JSON.stringify(error, null, 2);
+      throw new Error(`upsert_concepts failed: ${errorMessage}`);
+    }
   }
 };
 
@@ -543,33 +589,29 @@ const syncChangedNodesToSupabase = async ({
   context: SupabaseContext;
   accountLocalId: string;
 }): Promise<void> => {
-  if (changedNodes.length === 0) {
-    console.debug("No nodes to sync");
-    return;
+  if (changedNodes.length > 0) {
+    await upsertNodesToSupabaseAsContentWithEmbeddings({
+      obsidianNodes: changedNodes,
+      supabaseClient,
+      context,
+      accountLocalId,
+      plugin,
+    });
   }
 
-  await upsertNodesToSupabaseAsContentWithEmbeddings({
-    obsidianNodes: changedNodes,
-    supabaseClient,
-    context,
-    accountLocalId,
-    plugin,
-  });
-
   // Only upsert concepts for nodes with title changes or new files
   // (concepts store the title, so content-only changes don't affect them)
   const nodesNeedingConceptUpsert = changedNodes.filter((node) =>
     node.changeTypes.includes("title"),
   );
 
-  if (nodesNeedingConceptUpsert.length > 0) {
-    await convertDgToSupabaseConcepts({
-      nodesSince: nodesNeedingConceptUpsert,
-      supabaseClient,
-      context,
-      accountLocalId,
-    });
-  }
+  await convertDgToSupabaseConcepts({
+    nodesSince: nodesNeedingConceptUpsert,
+    supabaseClient,
+    context,
+    accountLocalId,
+    plugin,
+  });
 };
 
 /**
@@ -635,10 +677,7 @@ export const syncSpecificFiles = async (
   const changeTypesByPath = new Map<string, ChangeType[]>();
   for (const filePath of filePaths) {
     const existing = changeTypesByPath.get(filePath) ?? [];
-    changeTypesByPath.set(
-      filePath,
-      mergeChangeTypes(existing, ["content"]),
-    );
+    changeTypesByPath.set(filePath, mergeChangeTypes(existing, ["content"]));
   }
 
   await syncDiscourseNodeChanges(plugin, changeTypesByPath);