ENG-1475 Export relations to imported nodes

apps/obsidian/src/utils/conceptConversion.ts

@@ -231,13 +231,6 @@ export const relationInstanceToLocalConcept = ({
     return null;
   }
 
-  if (
-    sourceNode.frontmatter.importedFromRid ||
-    destinationNode.frontmatter.importedFromRid
-  )
-    return null; // punt relation to imported nodes for now.
-  // otherwise put the importedFromRid in source, dest.
-
   /* eslint-disable @typescript-eslint/naming-convention */
   const literal_content: Record<string, Json> = {};
   if (importedFromRid) literal_content.importedFromRid = importedFromRid;
@@ -252,8 +245,12 @@ export const relationInstanceToLocalConcept = ({
     last_modified: new Date(lastModified ?? created).toISOString(),
     literal_content,
     local_reference_content: {
-      source,
-      destination,
+      source:
+        (sourceNode.frontmatter.importedFromRid as string | undefined) ??
+        source,
+      destination:
+        (destinationNode.frontmatter.importedFromRid as string | undefined) ??
+        destination,
     },
     /* eslint-enable @typescript-eslint/naming-convention */
   };

apps/obsidian/src/utils/publishNode.ts

@@ -182,7 +182,6 @@ export const publishNodeRelations = async ({
         (fm.publishedToGroups as string[]) || [];
       if (!publishedToGroups.includes(myGroup)) return;
     }
-    if (fm.importedFromRid) return; // temporary, should be removed after eng-1475
     relevantNodeTypeById[id] = fm.nodeTypeId as string;
   });
   relations.map((relation) => {
@@ -199,6 +198,7 @@ export const publishNodeRelations = async ({
       resourceIds.add(triple.id);
     }
   });
+  if (resourceIds.size === 0) return;
   const publishResponse = await client.from("ResourceAccess").upsert(
     [...resourceIds.values()].map((sourceLocalId: string) => ({
       /* eslint-disable @typescript-eslint/naming-convention */

apps/obsidian/src/utils/syncDgNodesToSupabase.ts

@@ -365,6 +365,7 @@ const buildChangedNodesFromNodes = async ({
   const changedNodes: ObsidianDiscourseNodeData[] = [];
 
   for (const node of nodes) {
+    if (node.frontmatter.importedFromRid) continue;
     const existingTitle = existingTitleMap.get(node.nodeInstanceId);
     const detectedChangeTypes = detectNodeChanges(
       node,
@@ -423,7 +424,7 @@ export const syncAllNodesAndRelations = async (
     }
     console.debug("Supabase client:", supabaseClient);
 
-    const allNodes = await collectDiscourseNodesFromVault(plugin);
+    const allNodes = await collectDiscourseNodesFromVault(plugin, true);
 
     const changedNodeInstances = relationsOnly
       ? []
@@ -433,7 +434,6 @@ export const syncAllNodesAndRelations = async (
           context,
         });
 
-    console.log("changedNodeInstances", changedNodeInstances);
     console.debug(`Found ${changedNodeInstances.length} nodes to sync`);
 
     const accountLocalId = plugin.settings.accountLocalId;
@@ -495,7 +495,7 @@ const convertDgToSupabaseConcepts = async ({
   const nodeTypes = plugin.settings.nodeTypes ?? [];
   const relationTypes = plugin.settings.relationTypes ?? [];
   const discourseRelations = plugin.settings.discourseRelations ?? [];
-  allNodes = allNodes ?? (await collectDiscourseNodesFromVault(plugin));
+  allNodes = allNodes ?? (await collectDiscourseNodesFromVault(plugin, true));
   const allNodesById = Object.fromEntries(
     allNodes.map((n) => [n.nodeInstanceId, n]),
   );

packages/database/src/dbTypes.ts

@@ -1586,6 +1586,20 @@ export type Database = {
         }
         Returns: string
       }
+      rid_or_local_id_to_concept_db_id: {
+        Args: { default_space_id: number; rid: string }
+        Returns: number
+      }
+      rid_to_space_id_and_local_id: {
+        Args: { rid: string }
+        Returns: Database["public"]["CompositeTypes"]["accessible_resource"]
+        SetofOptions: {
+          from: "*"
+          to: "accessible_resource"
+          isOneToOne: true
+          isSetofReturn: false
+        }
+      }
       schema_of_concept:
         | {
             Args: { concept: Database["public"]["Tables"]["Concept"]["Row"] }
@@ -1967,3 +1981,4 @@ export const Constants = {
     },
   },
 } as const
+

packages/database/supabase/migrations/20260221193625_rid_functions.sql

@@ -0,0 +1,162 @@
+CREATE OR REPLACE FUNCTION public.rid_to_space_id_and_local_id(rid VARCHAR)
+RETURNS public.accessible_resource STRICT STABLE
+SET search_path = ''
+LANGUAGE plpgsql AS $$
+DECLARE
+    uri VARCHAR;
+    source_local_id VARCHAR;
+    source_id BIGINT;
+BEGIN
+source_local_id := split_part(rid, '/', -1);
+IF length(source_local_id) = length(rid) THEN
+    RETURN (null, 'Not a Rid')::public.accessible_resource;
+END IF;
+uri := substr(rid, 1, length(rid) - length(source_local_id) - 1);
+IF rid ~ '^orn:\w+\.\w+:.*$' THEN
+    uri := concat(split_part(split_part(uri, ':', 2), '.', 1), ':', split_part(uri, ':', 3));
+ELSE
+    IF rid ~ '^orn:\w+:.*$' THEN
+        uri := substr(uri, 5);
+    END IF;
+END IF;
+SELECT id INTO source_id FROM public."Space" where url=uri;
+IF source_id IS NULL THEN
+    RETURN (null, concat('Cannot find ', uri))::public.accessible_resource;
+END IF;
+RETURN (source_id, source_local_id);
+END;
+$$;
+
+CREATE OR REPLACE FUNCTION public.rid_or_local_id_to_concept_db_id(rid VARCHAR, default_space_id BIGINT)
+RETURNS BIGINT STRICT STABLE
+SET search_path = ''
+LANGUAGE plpgsql AS $$
+DECLARE r public.accessible_resource;
+BEGIN
+    r := (SELECT public.rid_to_space_id_and_local_id(rid));
+    IF r.space_id IS NULL THEN
+        RETURN  (SELECT id FROM public."Concept" WHERE space_id = default_space_id AND source_local_id = rid);
+    ELSE
+        RETURN (SELECT id FROM public."Concept" WHERE space_id = r.space_id AND source_local_id = r.source_local_id);
+    END IF;
+END;
+$$;
+
+CREATE OR REPLACE FUNCTION public._local_concept_to_db_concept(data public.concept_local_input)
+RETURNS public."Concept" STABLE
+SET search_path = ''
+LANGUAGE plpgsql
+AS $$
+DECLARE
+  concept public."Concept"%ROWTYPE;
+  reference_content JSONB := jsonb_build_object();
+  key varchar;
+  value JSONB;
+  ref_single_val BIGINT;
+  ref_array_val BIGINT[];
+BEGIN
+  -- not fan of going through json, but not finding how to populate a record by a different shape record
+  concept := jsonb_populate_record(NULL::public."Concept", to_jsonb(data));
+  IF data.author_local_id IS NOT NULL THEN
+    SELECT id FROM public."PlatformAccount"
+      WHERE account_local_id = data.author_local_id INTO concept.author_id;
+  END IF;
+  IF data.represented_by_id IS NOT NULL THEN
+    SELECT space_id, source_local_id FROM public."Content"
+      WHERE id = data.represented_by_id INTO concept.space_id, concept.source_local_id;
+  END IF;
+  IF data.space_url IS NOT NULL THEN
+    SELECT id FROM public."Space"
+    WHERE url = data.space_url INTO concept.space_id;
+  END IF;
+  IF concept.source_local_id = '' THEN
+    concept.source_local_id := NULL;
+  END IF;
+  IF data.represented_by_local_id = '' THEN
+    data.represented_by_local_id := NULL;
+  END IF;
+  IF data.schema_represented_by_local_id IS NOT NULL THEN
+    SELECT public.rid_or_local_id_to_concept_db_id(
+        data.schema_represented_by_local_id, concept.space_id) INTO concept.schema_id;
+  END IF;
+  concept.source_local_id = COALESCE(concept.source_local_id, data.represented_by_local_id); -- legacy input field
+  concept.reference_content := coalesce(data.reference_content, '{}'::jsonb);
+  IF data.local_reference_content IS NOT NULL THEN
+    FOR key, value IN SELECT * FROM jsonb_each(data.local_reference_content) LOOP
+      IF jsonb_typeof(value) = 'array' THEN
+        WITH el AS (SELECT jsonb_array_elements_text(value) as x),
+        el2 AS (SELECT public.rid_or_local_id_to_concept_db_id(x, concept.space_id) AS id FROM el)
+        SELECT array_agg(DISTINCT el2.id) INTO STRICT ref_array_val
+            FROM el2 WHERE el2.id IS NOT NULL;
+        reference_content := jsonb_set(reference_content, ARRAY[key], to_jsonb(ref_array_val));
+      ELSIF jsonb_typeof(value) = 'string' THEN
+        SELECT public.rid_or_local_id_to_concept_db_id(value #>> '{}', concept.space_id) INTO STRICT ref_single_val;
+        reference_content := jsonb_set(reference_content, ARRAY[key], to_jsonb(ref_single_val));
+      ELSE
+        RAISE EXCEPTION 'Invalid value in local_reference_content % %', value, jsonb_typeof(value);
+      END IF;
+    END LOOP;
+    concept.reference_content := concept.reference_content || reference_content;
+  END IF;
+  RETURN concept;
+END;
+$$;
+
+CREATE OR REPLACE FUNCTION public.upsert_concepts(v_space_id bigint, data jsonb)
+RETURNS SETOF BIGINT
+SET search_path = ''
+LANGUAGE plpgsql
+AS $$
+DECLARE
+  v_platform public."Platform";
+  local_concept public.concept_local_input;
+  db_concept public."Concept"%ROWTYPE;
+  concept_row JSONB;
+  concept_id BIGINT;
+BEGIN
+  SELECT platform INTO STRICT v_platform FROM public."Space" WHERE id=v_space_id;
+  FOR concept_row IN SELECT * FROM jsonb_array_elements(data)
+  LOOP
+    -- first set defaults
+    local_concept := jsonb_populate_record(NULL::public.concept_local_input, '{"epistemic_status": "unknown", "literal_content":{},"reference_content":{},"is_schema":false}');
+    -- then input values
+    local_concept := jsonb_populate_record(local_concept, concept_row);
+    local_concept.space_id := v_space_id;
+    BEGIN
+        db_concept := public._local_concept_to_db_concept(local_concept);
+        -- cannot use db_concept.* because of refs.
+        INSERT INTO public."Concept" (
+        epistemic_status, name, description, author_id, created, last_modified, space_id, schema_id, literal_content, is_schema, source_local_id, reference_content
+        ) VALUES (
+        db_concept.epistemic_status, db_concept.name, db_concept.description, db_concept.author_id, db_concept.created, db_concept.last_modified, db_concept.space_id, db_concept.schema_id, db_concept.literal_content, db_concept.is_schema, db_concept.source_local_id, db_concept.reference_content
+        )
+        ON CONFLICT (space_id, source_local_id) DO UPDATE SET
+            epistemic_status = db_concept.epistemic_status,
+            name = db_concept.name,
+            description = db_concept.description,
+            author_id = db_concept.author_id,
+            created = db_concept.created,
+            last_modified = db_concept.last_modified,
+            schema_id = db_concept.schema_id,
+            literal_content = db_concept.literal_content,
+            is_schema = db_concept.is_schema,
+            reference_content = db_concept.reference_content
+        -- If the syntax allowed two conflict clauses, I would add
+        -- ON CONFLICT (space_id, name) DO NOTHING
+        -- but since not, I have to handle it as an exception.
+        RETURNING id INTO concept_id;
+        RETURN NEXT concept_id;
+    EXCEPTION
+        WHEN unique_violation THEN
+            -- a distinct unique constraint failed
+            RAISE WARNING 'Concept with space_id: % and name % already exists', v_space_id, local_concept.name;
+            RETURN NEXT -1; -- Return a special value to indicate conflict
+        WHEN OTHERS THEN
+            -- Null value; probably due to a missing concept.
+            RAISE WARNING 'Error in concept upsert: (%) %', SQLSTATE, SQLERRM;
+            RETURN NEXT -2; -- Return a special value to indicate error
+    END;
+  END LOOP;
+  RAISE DEBUG 'Completed upsert_concepts successfully';
+END;
+$$;

packages/database/supabase/schemas/concept.sql

@@ -240,6 +240,49 @@ $$;
 COMMENT ON FUNCTION public.author_of_concept(public.my_concepts)
 IS 'Computed one-to-one: returns the PlatformAccount which authored a given Concept.';
 
+CREATE OR REPLACE FUNCTION public.rid_to_space_id_and_local_id(rid VARCHAR)
+RETURNS public.accessible_resource STRICT STABLE
+SET search_path = ''
+LANGUAGE plpgsql AS $$
+DECLARE
+    uri VARCHAR;
+    source_local_id VARCHAR;
+    source_id BIGINT;
+BEGIN
+source_local_id := split_part(rid, '/', -1);
+IF length(source_local_id) = length(rid) THEN
+    RETURN (null, 'Not a Rid')::public.accessible_resource;
+END IF;
+uri := substr(rid, 1, length(rid) - length(source_local_id) - 1);
+IF rid ~ '^orn:\w+\.\w+:.*$' THEN
+    uri := concat(split_part(split_part(uri, ':', 2), '.', 1), ':', split_part(uri, ':', 3));
+ELSE
+    IF rid ~ '^orn:\w+:.*$' THEN
+        uri := substr(uri, 5);
+    END IF;
+END IF;
+SELECT id INTO source_id FROM public."Space" where url=uri;
+IF source_id IS NULL THEN
+    RETURN (null, concat('Cannot find ', uri))::public.accessible_resource;
+END IF;
+RETURN (source_id, source_local_id);
+END;
+$$;
+
+CREATE OR REPLACE FUNCTION public.rid_or_local_id_to_concept_db_id(rid VARCHAR, default_space_id BIGINT)
+RETURNS BIGINT STRICT STABLE
+SET search_path = ''
+LANGUAGE plpgsql AS $$
+DECLARE r public.accessible_resource;
+BEGIN
+    r := (SELECT public.rid_to_space_id_and_local_id(rid));
+    IF r.space_id IS NULL THEN
+        RETURN  (SELECT id FROM public."Concept" WHERE space_id = default_space_id AND source_local_id = rid);
+    ELSE
+        RETURN (SELECT id FROM public."Concept" WHERE space_id = r.space_id AND source_local_id = r.source_local_id);
+    END IF;
+END;
+$$;
 
 CREATE TYPE public.concept_local_input AS (
     -- concept columns
@@ -292,37 +335,34 @@ BEGIN
     SELECT id FROM public."Space"
     WHERE url = data.space_url INTO concept.space_id;
   END IF;
-  IF data.schema_represented_by_local_id IS NOT NULL THEN
-    SELECT cpt.id FROM public."Concept" cpt
-      WHERE cpt.source_local_id = data.schema_represented_by_local_id
-      AND cpt.space_id = concept.space_id INTO concept.schema_id;
-  END IF;
   IF concept.source_local_id = '' THEN
     concept.source_local_id := NULL;
   END IF;
   IF data.represented_by_local_id = '' THEN
     data.represented_by_local_id := NULL;
   END IF;
+  IF data.schema_represented_by_local_id IS NOT NULL THEN
+    SELECT public.rid_or_local_id_to_concept_db_id(
+        data.schema_represented_by_local_id, concept.space_id) INTO concept.schema_id;
+  END IF;
   concept.source_local_id = COALESCE(concept.source_local_id, data.represented_by_local_id); -- legacy input field
+  concept.reference_content := coalesce(data.reference_content, '{}'::jsonb);
   IF data.local_reference_content IS NOT NULL THEN
     FOR key, value IN SELECT * FROM jsonb_each(data.local_reference_content) LOOP
       IF jsonb_typeof(value) = 'array' THEN
         WITH el AS (SELECT jsonb_array_elements_text(value) as x),
-        ela AS (SELECT array_agg(x) AS a FROM el)
-        SELECT array_agg(DISTINCT cpt.id) INTO STRICT ref_array_val
-            FROM public."Concept" AS cpt
-            JOIN ela ON (true) WHERE cpt.source_local_id = ANY(ela.a) AND cpt.space_id=concept.space_id;
+        el2 AS (SELECT public.rid_or_local_id_to_concept_db_id(x, concept.space_id) AS id FROM el)
+        SELECT array_agg(DISTINCT el2.id) INTO STRICT ref_array_val
+            FROM el2 WHERE el2.id IS NOT NULL;
         reference_content := jsonb_set(reference_content, ARRAY[key], to_jsonb(ref_array_val));
       ELSIF jsonb_typeof(value) = 'string' THEN
-        SELECT cpt.id INTO STRICT ref_single_val
-            FROM public."Concept" AS cpt
-            WHERE cpt.source_local_id = (value #>> '{}') AND cpt.space_id=concept.space_id;
+        SELECT public.rid_or_local_id_to_concept_db_id(value #>> '{}', concept.space_id) INTO STRICT ref_single_val;
         reference_content := jsonb_set(reference_content, ARRAY[key], to_jsonb(ref_single_val));
       ELSE
         RAISE EXCEPTION 'Invalid value in local_reference_content % %', value, jsonb_typeof(value);
       END IF;
     END LOOP;
-    SELECT reference_content INTO concept.reference_content;
+    concept.reference_content := concept.reference_content || reference_content;
   END IF;
   RETURN concept;
 END;
@@ -351,8 +391,8 @@ BEGIN
     -- then input values
     local_concept := jsonb_populate_record(local_concept, concept_row);
     local_concept.space_id := v_space_id;
-    db_concept := public._local_concept_to_db_concept(local_concept);
     BEGIN
+        db_concept := public._local_concept_to_db_concept(local_concept);
         -- cannot use db_concept.* because of refs.
         INSERT INTO public."Concept" (
         epistemic_status, name, description, author_id, created, last_modified, space_id, schema_id, literal_content, is_schema, source_local_id, reference_content
@@ -370,14 +410,20 @@ BEGIN
             literal_content = db_concept.literal_content,
             is_schema = db_concept.is_schema,
             reference_content = db_concept.reference_content
-        -- ON CONFLICT (space_id, name) DO NOTHING... why can't I specify two conflict clauses?
+        -- If the syntax allowed two conflict clauses, I would add
+        -- ON CONFLICT (space_id, name) DO NOTHING
+        -- but since not, I have to handle it as an exception.
         RETURNING id INTO concept_id;
         RETURN NEXT concept_id;
     EXCEPTION
         WHEN unique_violation THEN
             -- a distinct unique constraint failed
             RAISE WARNING 'Concept with space_id: % and name % already exists', v_space_id, local_concept.name;
             RETURN NEXT -1; -- Return a special value to indicate conflict
+        WHEN OTHERS THEN
+            -- Null value; probably due to a missing concept.
+            RAISE WARNING 'Error in concept upsert: (%) %', SQLSTATE, SQLERRM;
+            RETURN NEXT -2; -- Return a special value to indicate error
     END;
   END LOOP;
   RAISE DEBUG 'Completed upsert_concepts successfully';