[ENG-1848] Add Roam full markdown content variant for shared nodes

Author: sid597

apps/roam/src/utils/convertRoamNodeToFullContent.fixture.ts

@@ -0,0 +1,58 @@
+import type { TreeNode } from "roamjs-components/types";
+import {
+  FULL_CONTENT_FORMAT,
+  type CrossAppNodeContent,
+} from "@repo/database/crossAppNodeContract";
+import { buildFullMarkdown } from "./convertRoamNodeToFullContent";
+
+/**
+ * Reference fixture for ENG-1848 ("tests or fixtures cover representative Roam
+ * block content becoming `full` markdown"). The Roam app has no unit-test
+ * runner, so this exercises the real `buildFullMarkdown` transform on an
+ * in-memory Roam page tree and types the result against the ENG-1847 contract's
+ * `full` variant. Downstream importer validation (ENG-1857) can assert against
+ * `roamClaimFullMarkdownFixture.full.value`, which evaluates to:
+ *
+ *   # Sleep improves memory consolidation
+ *
+ *   - Multiple studies show that sleep after learning strengthens memory traces.
+ *   - Supporting evidence:
+ *       - [[EVD]] - Rasch & Born 2013
+ */
+
+const block = (text: string, children: TreeNode[] = []): TreeNode => ({
+  text,
+  children,
+  order: 0,
+  parents: [],
+  uid: "",
+  heading: 0,
+  open: true,
+  viewType: "bullet",
+  blockViewType: "outline",
+  editTime: new Date(0),
+  textAlign: "left",
+  props: { imageResize: {}, iframe: {} },
+});
+
+const title = "Sleep improves memory consolidation";
+
+const blocks: TreeNode[] = [
+  block(
+    "Multiple studies show that sleep after learning strengthens memory traces.",
+  ),
+  block("Supporting evidence:", [block("[[EVD]] - Rasch & Born 2013")]),
+];
+
+export const roamClaimFullMarkdownFixture: {
+  title: string;
+  blocks: TreeNode[];
+  full: CrossAppNodeContent["full"];
+} = {
+  title,
+  blocks,
+  full: {
+    format: FULL_CONTENT_FORMAT,
+    value: buildFullMarkdown({ title, blocks }),
+  },
+};

apps/roam/src/utils/convertRoamNodeToFullContent.ts

@@ -0,0 +1,81 @@
+import { toMarkdown } from "./pageToMarkdown";
+import { type RoamDiscourseNodeData } from "./getAllDiscourseNodesSince";
+import { type DiscourseNode } from "./getDiscourseNodes";
+import getFullTreeByParentUid from "roamjs-components/queries/getFullTreeByParentUid";
+import getPageViewType from "roamjs-components/queries/getPageViewType";
+import type { TreeNode, ViewType } from "roamjs-components/types";
+import type { LocalContentDataInput } from "@repo/database/inputTypes";
+
+/**
+ * Builds the `full` cross-app content variant for Roam discourse nodes.
+ *
+ * Per the shared cross-app node contract (ENG-1847,
+ * `@repo/database/crossAppNodeContract`), every shared node must persist a
+ * `full` variant: a self-sufficient markdown body the destination app can
+ * materialize without querying Roam. Roam previously emitted only the `direct`
+ * title content; this fills that gap (ENG-1848, F2/F3). The body reuses the
+ * existing `toMarkdown` page serializer with block-refs and embeds inlined for
+ * self-sufficiency, prefixed with the node title as an H1 — matching the
+ * contract's Roam fixture. Known MVP0 markdown-fidelity limits live on F3.
+ */
+
+const FULL_MARKDOWN_OPTS = {
+  refs: true,
+  embeds: true,
+  simplifiedFilename: false,
+  removeSpecialCharacters: false,
+  maxFilenameLength: 64,
+  linkType: "alias",
+  allNodes: [] as DiscourseNode[],
+};
+
+export const buildFullMarkdown = ({
+  title,
+  blocks,
+  viewType = "bullet",
+}: {
+  title: string;
+  blocks: TreeNode[];
+  viewType?: ViewType;
+}): string => {
+  const body = blocks
+    .filter((block) => !!block.text || !!block.children?.length)
+    .map((block) =>
+      toMarkdown({ c: block, v: viewType, i: 0, opts: FULL_MARKDOWN_OPTS }),
+    )
+    .join("\n")
+    .trim();
+  return body ? `# ${title}\n\n${body}\n` : `# ${title}\n`;
+};
+
+export const convertRoamNodeToFullContent = ({
+  nodes,
+}: {
+  nodes: RoamDiscourseNodeData[];
+}): LocalContentDataInput[] =>
+  nodes.flatMap((node) => {
+    try {
+      const title = node.node_title ?? node.text;
+      const blocks = getFullTreeByParentUid(node.source_local_id).children;
+      const viewType = getPageViewType(title) || "bullet";
+      return [
+        {
+          author_local_id: node.author_local_id,
+          source_local_id: node.source_local_id,
+          created: new Date(node.created || Date.now()).toISOString(),
+          last_modified: new Date(
+            node.last_modified || Date.now(),
+          ).toISOString(),
+          text: buildFullMarkdown({ title, blocks, viewType }),
+          variant: "full",
+          scale: "document",
+        },
+      ];
+    } catch (error) {
+      console.error(
+        `convertRoamNodeToFullContent: failed to build full markdown for ${node.source_local_id}:`,
+        error,
+      );
+      return [];
+    }
+  });

apps/roam/src/utils/syncDgNodesToSupabase.ts

@@ -18,6 +18,7 @@ import {
 } from "./conceptConversion";
 import { fetchEmbeddingsForNodes } from "./upsertNodesAsContentWithEmbeddings";
 import { convertRoamNodeToLocalContent } from "./upsertNodesAsContentWithEmbeddings";
+import { convertRoamNodeToFullContent } from "./convertRoamNodeToFullContent";
 import type { DGSupabaseClient } from "@repo/database/lib/client";
 import { intersection } from "@repo/utils/setOperations";
 import type { Json, Enums } from "@repo/database/dbTypes";
@@ -618,6 +619,9 @@ export const upsertNodesToSupabaseAsContentWithEmbeddings = async (
   const allNodeInstancesAsLocalContent = convertRoamNodeToLocalContent({
     nodes: roamNodes,
   });
+  const fullContent = convertRoamNodeToFullContent({
+    nodes: roamNodes,
+  });
 
   let nodesWithEmbeddings: LocalContentDataInput[];
   try {
@@ -658,7 +662,9 @@ export const upsertNodesToSupabaseAsContentWithEmbeddings = async (
     }
   };
 
-  await uploadBatches(chunk(nodesWithEmbeddings, BATCH_SIZE));
+  await uploadBatches(
+    chunk([...nodesWithEmbeddings, ...fullContent], BATCH_SIZE),
+  );
 };
 
 const getAllUsers = async (): Promise<LocalAccountDataInput[]> => {