[ENG-1176] Sync DG node from Obsidian to database (#665)

* current progress

* redo the auth so that only one user acc is created

* sync all nodes on load

* current progress

* revert changes

* clean up sync nodes

* enable cors

* address PR comments

Author: trangdoan982

apps/obsidian/src/utils/conceptConversion.ts

@@ -0,0 +1,140 @@
+/* eslint-disable @typescript-eslint/naming-convention */
+import { TFile } from "obsidian";
+import { DiscourseNode } from "~/types";
+import { SupabaseContext } from "./supabaseContext";
+import { LocalConceptDataInput } from "@repo/database/inputTypes";
+import { ObsidianDiscourseNodeData } from "./syncDgNodesToSupabase";
+import { Json } from "@repo/database/dbTypes";
+
+/**
+ * Get extra data (author, timestamps) from file metadata
+ */
+const getNodeExtraData = (
+  file: TFile,
+  accountLocalId: string,
+): {
+  author_local_id: string;
+  created: string;
+  last_modified: string;
+} => {
+  return {
+    author_local_id: accountLocalId,
+    created: new Date(file.stat.ctime).toISOString(),
+    last_modified: new Date(file.stat.mtime).toISOString(),
+  };
+};
+
+export const discourseNodeSchemaToLocalConcept = ({
+  context,
+  node,
+  accountLocalId,
+}: {
+  context: SupabaseContext;
+  node: DiscourseNode;
+  accountLocalId: string;
+}): LocalConceptDataInput => {
+  const now = new Date().toISOString();
+  return {
+    space_id: context.spaceId,
+    name: node.name,
+    represented_by_local_id: node.id,
+    is_schema: true,
+    author_local_id: accountLocalId,
+    created: now,
+    // TODO: get the template or any other info to put into literal_content jsonb
+    last_modified: now,
+  };
+};
+
+/**
+ * Convert discourse node instance (file) to LocalConceptDataInput
+ */
+export const discourseNodeInstanceToLocalConcept = ({
+  context,
+  nodeData,
+  accountLocalId,
+}: {
+  context: SupabaseContext;
+  nodeData: ObsidianDiscourseNodeData;
+  accountLocalId: string;
+}): LocalConceptDataInput => {
+  const extraData = getNodeExtraData(nodeData.file, accountLocalId);
+  console.log(nodeData.frontmatter);
+  const concept = {
+    space_id: context.spaceId,
+    name: nodeData.file.basename,
+    represented_by_local_id: nodeData.nodeInstanceId,
+    schema_represented_by_local_id: nodeData.nodeTypeId,
+    is_schema: false,
+    literal_content: {
+      ...nodeData.frontmatter,
+    } as unknown as Json,
+    ...extraData,
+  };
+  console.log(
+    `[discourseNodeInstanceToLocalConcept] Converting concept: represented_by_local_id=${nodeData.nodeInstanceId}, name="${nodeData.file.basename}"`,
+  );
+  return concept;
+};
+
+export const relatedConcepts = (concept: LocalConceptDataInput): string[] => {
+  const relations = Object.values(
+    concept.local_reference_content || {},
+  ).flat() as string[];
+  if (concept.schema_represented_by_local_id) {
+    relations.push(concept.schema_represented_by_local_id);
+  }
+  // remove duplicates
+  return [...new Set(relations)];
+};
+
+/**
+ * Recursively order concepts by dependency
+ */
+const orderConceptsRec = (
+  ordered: LocalConceptDataInput[],
+  concept: LocalConceptDataInput,
+  remainder: { [key: string]: LocalConceptDataInput },
+): Set<string> => {
+  const relatedConceptIds = relatedConcepts(concept);
+  let missing: Set<string> = new Set();
+  while (relatedConceptIds.length > 0) {
+    const relatedConceptId = relatedConceptIds.shift()!;
+    const relatedConcept = remainder[relatedConceptId];
+    if (relatedConcept === undefined) {
+      missing.add(relatedConceptId);
+    } else {
+      missing = new Set([
+        ...missing,
+        ...orderConceptsRec(ordered, relatedConcept, remainder),
+      ]);
+      delete remainder[relatedConceptId];
+    }
+  }
+  ordered.push(concept);
+  delete remainder[concept.represented_by_local_id!];
+  return missing;
+};
+
+export const orderConceptsByDependency = (
+  concepts: LocalConceptDataInput[],
+): { ordered: LocalConceptDataInput[]; missing: string[] } => {
+  if (concepts.length === 0) return { ordered: concepts, missing: [] };
+  const conceptById: { [key: string]: LocalConceptDataInput } =
+    Object.fromEntries(
+      concepts
+        .filter((c) => c.represented_by_local_id)
+        .map((c) => [c.represented_by_local_id!, c]),
+    );
+  const ordered: LocalConceptDataInput[] = [];
+  let missing: Set<string> = new Set();
+  while (Object.keys(conceptById).length > 0) {
+    const first = Object.values(conceptById)[0];
+    if (!first) break;
+    missing = new Set([
+      ...missing,
+      ...orderConceptsRec(ordered, first, conceptById),
+    ]);
+  }
+  return { ordered, missing: Array.from(missing) };
+};

apps/obsidian/src/utils/registerCommands.ts

@@ -6,6 +6,8 @@ import { BulkIdentifyDiscourseNodesModal } from "~/components/BulkIdentifyDiscou
 import { createDiscourseNode } from "./createNode";
 import { VIEW_TYPE_MARKDOWN, VIEW_TYPE_TLDRAW_DG_PREVIEW } from "~/constants";
 import { createCanvas } from "~/components/canvas/utils/tldraw";
+import { createOrUpdateDiscourseEmbedding } from "./syncDgNodesToSupabase";
+import { Notice } from "obsidian";
 
 export const registerCommands = (plugin: DiscourseGraphPlugin) => {
   plugin.addCommand({
@@ -130,4 +132,28 @@ export const registerCommands = (plugin: DiscourseGraphPlugin) => {
     icon: "layout-dashboard", // Using Lucide icon as per style guide
     callback: () => createCanvas(plugin),
   });
+
+  plugin.addCommand({
+    id: "sync-discourse-nodes-to-supabase",
+    name: "Sync Discourse Nodes to Supabase",
+    checkCallback: (checking: boolean) => {
+      if (!plugin.settings.syncModeEnabled) {
+        new Notice("Sync mode is not enabled", 3000);
+        return false;
+      }
+      if (!checking) {
+        void createOrUpdateDiscourseEmbedding(plugin)
+          .then(() => {
+            new Notice("Discourse nodes synced successfully", 3000);
+          })
+          .catch((error) => {
+            const errorMessage =
+              error instanceof Error ? error.message : String(error);
+            new Notice(`Sync failed: ${errorMessage}`, 5000);
+            console.error("Manual sync failed:", error);
+          });
+      }
+      return true;
+    },
+  });
 };