Add Collection class

Author: apdavison

fairgraph/__init__.py

@@ -25,6 +25,7 @@
 from .embedded import EmbeddedMetadata
 from .kgproxy import KGProxy
 from .kgquery import KGQuery
+from .collection import Collection
 from . import client, errors, openminds, utility
 
 __version__ = "0.12.2"

fairgraph/client.py

@@ -461,6 +461,8 @@ def create_new_instance(
         """
         if "'@id': None" in str(data):
             raise ValueError("payload contains undefined ids")
+        if instance_id:
+            UUID(instance_id)
         if self.migrated is False:
             data = deepcopy(data)
             adapt_namespaces_4to3(data)
@@ -488,6 +490,7 @@ def update_instance(self, instance_id: str, data: JsonLdDocument) -> JsonLdDocum
             instance_id (UUID): the instance's persistent identifier.
             data (dict): a JSON-LD document that modifies some or all of the data of the existing instance.
         """
+        UUID(instance_id)
         if self.migrated is False:
             data = deepcopy(data)
             adapt_namespaces_4to3(data)
@@ -507,6 +510,7 @@ def replace_instance(self, instance_id: str, data: JsonLdDocument) -> JsonLdDocu
             instance_id (UUID): the instance's persistent identifier.
             data (dict): a JSON-LD document that will replace the existing instance.
         """
+        UUID(instance_id)
         if self.migrated is False:
             data = deepcopy(data)
             adapt_namespaces_4to3(data)
@@ -522,6 +526,7 @@ def delete_instance(self, instance_id: str, ignore_not_found: bool = True, ignor
         """
         Delete a KG instance.
         """
+        UUID(instance_id)
         response = self._kg_client.instances.delete(instance_id)
         # response is None if no errors
         if response:  # error

fairgraph/collection.py

@@ -0,0 +1,108 @@
+"""
+This module provides the Collection class, an extension to the openMINDS Collection
+that knows how to upload metadata to the KG.
+"""
+
+# Copyright 2018-2024 CNRS
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+#     http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from importlib import import_module
+import os
+from time import sleep
+from uuid import uuid4
+from warnings import warn
+
+from openminds import Collection as OMCollection
+from .utility import ActivityLog
+from .errors import AuthenticationError
+
+
+class Collection(OMCollection):
+    """
+    A collection of metadata nodes that can be saved to
+    and loaded from disk, and uploaded to the KG.
+
+    Args
+    ----
+
+    *nodes (LinkedMetadata):
+        Nodes to store in the collection when creating it.
+        Child nodes that are referenced from the explicitly
+        listed nodes will also be added.
+    """
+
+    def load(self, *paths):
+        import_module("fairgraph.openminds")
+        super().load(*paths)
+
+    def upload(self, client, default_space=None, space_map=None, verbosity=0):
+        # if not self.complete:
+        #     raise Exception("Collection contains local ids. Run `generate_ids()` and then re-save the collection")
+        #     # self.generate_ids(lambda node: client.uri_from_uuid(uuid4()))
+
+        nodes_to_save = [
+            node
+            for node in self.sort_nodes_for_upload()
+            if not node.id.startswith("https://openminds.om-i.org/instances")
+        ]
+        activity_log = ActivityLog()
+
+        if verbosity == 1:
+            try:
+                tqdm = import_module("tqdm")
+            except ImportError:
+                warn("Unable to show progress bar, please install tqdm")
+            else:
+                nodes_to_save = tqdm.tqdm(nodes_to_save)
+
+        if os.path.exists(".kg_upload_log.txt"):
+            with open(".kg_upload_log.txt") as fp:
+                skip = fp.read().strip().split("\n")
+        else:
+            skip = None
+
+        for i, node in enumerate(nodes_to_save):
+            if not (skip and node.id in skip):
+                if verbosity == 2:
+                    print(f"[{100*i//len(nodes_to_save)}%] Saving {node.__class__.__name__} {node.id}")
+                if space_map:
+                    target_space = space_map.get(node.__class__, default_space)
+                else:
+                    target_space = default_space
+                original_node_id = node.id
+                try:
+                    node.save(
+                        client, space=target_space, recursive=False, ignore_duplicates=True, activity_log=activity_log
+                    )
+                except AuthenticationError as err:
+                    # client.refresh()
+                    print(err)
+                    break
+                except Exception as err:
+                    if "500" in str(err):
+                        sleep(5)
+                        node.save(
+                            client,
+                            space=target_space,
+                            recursive=False,
+                            ignore_duplicates=True,
+                            activity_log=activity_log,
+                        )
+                    else:
+                        raise
+                else:
+                    with open(".kg_upload_log.txt", "a") as fp:
+                        fp.write(f"{original_node_id}\n")
+
+        return activity_log

fairgraph/kgobject.py

@@ -89,13 +89,14 @@ def __init__(
 
         self._raw_remote_data = None
         self.remote_data = {}
-        if self.id:
+        if self.id and self.id.startswith("http"):
             # we store the original remote data in `_raw_remote_data`
             # and a normalized version in `remote_data`
             self._raw_remote_data = data  # for debugging
             if data:
                 self.remote_data = normalize_data(
-                    self.to_jsonld(include_empty_properties=True, embed_linked_nodes=False), self.context
+                    self.to_jsonld(include_empty_properties=True, embed_linked_nodes=False),
+                    data.get("@context", self.context)
                 )
 
     def __repr__(self):
@@ -352,7 +353,8 @@ def from_alias(
     def uuid(self) -> Union[str, None]:
         # todo: consider using client._kg_client.uuid_from_absolute_id
         if self.id is not None:
-            return self.id.split("/")[-1]
+            value = self.id.split("/")[-1]
+            return str(UUID(value))
         else:
             return None
 
@@ -548,7 +550,7 @@ def diff(self, other):
     def exists(self, client: KGClient, ignore_duplicates: bool = False, in_spaces: Optional[List[str]] = None) -> bool:
         """Check if this object already exists in the KnowledgeGraph"""
 
-        if self.id:
+        if self.id and self.id.startswith("http"):
             # Since the KG now allows user-specified IDs we can't assume that the presence of
             # an id means the object exists
             data = client.instance_from_full_uri(
@@ -601,6 +603,7 @@ def exists(self, client: KGClient, ignore_duplicates: bool = False, in_spaces: O
 
                 if instances:
                     if len(instances) > 1 and not ignore_duplicates:
+                        # we might want to consider running a second query with "equals" rather than "contains"
                         raise Exception(
                             f"Existence query is not specific enough. Type: {self.__class__.__name__}; filters: {query_filter}"
                         )
@@ -792,9 +795,13 @@ def save(
             # create new
             local_data = normalize_data(self.to_jsonld(embed_linked_nodes=False), self.context)
             logger.info("  - creating instance with data {}".format(local_data))
+            if self.id and self.id.startswith("http"):
+                instance_id = self.uuid
+            else:
+                instance_id = None
             try:
                 instance_data = client.create_new_instance(
-                    local_data, space or self.__class__.default_space, instance_id=self.uuid
+                    local_data, space or self.__class__.default_space, instance_id=instance_id
                 )
             except (AuthorizationError, ResourceExistsError) as err:
                 if ignore_auth_errors:

fairgraph/utility.py

@@ -220,6 +220,10 @@ def normalize_data(data: Union[None, JSONdict], context: Dict[str, Any]) -> Unio
             pass
         elif value is None:
             pass
+        elif expanded_key == "@id":
+            if value.startswith("http"):
+                # do not take local ids, e.g., those starting with "_"
+                normalized[expanded_key] = value
         elif expanded_key == "@type":
             normalized[expanded_key] = value
         elif isinstance(value, (list, tuple)):
@@ -327,6 +331,15 @@ def __init__(
     def __repr__(self):
         return f"{self.type}: {self.cls}({self.id}) in '{self.space}'"
 
+    def as_dict(self):
+        return {
+            "cls": self.cls,
+            "id": self.id,
+            "delta": self.delta,
+            "space": self.space,
+            "type_": self.type
+        }
+
 
 class ActivityLog:
     """