get file from which a shape originates from in rdf model service from directory'

ssssarah · ssssarah · commit fd5cfc77ed52 · 2023-10-26T16:36:34.000+02:00
diff --git a/kgforge/core/archetypes/model.py b/kgforge/core/archetypes/model.py
@@ -190,8 +190,8 @@ def mapping(self, entity: str, source: str, type: Callable) -> Mapping:
 
     # Validation.
 
-    def schema_id(self, type: str) -> URIRef:
-        # POLICY Should retrieve the schema id of the given type.
+    def schema_source(self, type: str) -> str:
+        # POLICY Should retrieve the schema source of the given type.
         not_supported()
 
     def validate(self, data: Union[Resource, List[Resource]],
diff --git a/kgforge/specializations/models/rdf/rdf_model_service.py b/kgforge/specializations/models/rdf/rdf_model_service.py
@@ -33,20 +33,24 @@
 
 
 class RdfModelService:
-    shape_to_source: Dict[URIRef, str]
-    class_to_shape: Dict[str, URIRef]
 
-    def __init__(self, graph: Graph, context_iri: Optional[str] = None) -> None:
+    def __init__(
+            self, graph: Graph,
+            shape_to_source: Dict[URIRef, str],
+            class_to_shape:  Dict[str, URIRef],
+            context_iri: Optional[str] = None,
+    ) -> None:
 
         if context_iri is None:
             raise ConfigurationError("RdfModel requires a context")
         self._graph = graph
         self._context_cache = dict()
-        self.shape_to_source, self.class_to_shape = self._build_shapes_map()
+        self.shape_to_source = shape_to_source
+        self.class_to_shape = class_to_shape
         self.context = Context(self.resolve_context(context_iri), context_iri)
         self.types_to_shapes: Dict[str, URIRef] = self._build_types_to_shapes()
 
-    def shape_source(self, schema_iri: URIRef) -> str:
+    def get_shape_source(self, schema_iri: URIRef) -> str:
         return self.shape_to_source[schema_iri]
 
     def sparql(self, query: str) -> List[Resource]:
@@ -100,11 +104,6 @@ def generate_context(self) -> Dict:
         """Generates a JSON-LD context with the classes and terms present in the SHACL graph."""
         raise NotImplementedError()
 
-    @abstractmethod
-    def _build_shapes_map(self) -> Tuple[Dict[URIRef, str], Dict[str, URIRef]]:
-        """Queries the source and returns a map of owl:Class to sh:NodeShape"""
-        raise NotImplementedError()
-
     def _build_types_to_shapes(self) -> Dict[str, URIRef]:
         """Iterates the classes_to_shapes dictionary to create a term to shape dictionary filtering
          the terms available in the context """
diff --git a/kgforge/specializations/models/rdf/rdf_model_service_from_directory.py b/kgforge/specializations/models/rdf/rdf_model_service_from_directory.py
@@ -28,9 +28,14 @@
 class RdfModelServiceFromDirectory(RdfModelService):
 
     def __init__(self, dir_path: Path, context_iri: str) -> None:
-        self._graph = load_rdf_files_into_graph(dir_path, Graph())
-        self._shapes_graph = ShapesGraphWrapper(self._graph)
-        super().__init__(self._graph, context_iri)
+
+        graph, shape_to_source, class_to_shape = self._build_shapes_map(dir_path=dir_path)
+        self._shapes_graph = ShapesGraphWrapper(graph)
+
+        super().__init__(
+            graph=graph, context_iri=context_iri, shape_to_source=shape_to_source,
+            class_to_shape=class_to_shape
+        )
 
     def materialize(self, iri: URIRef) -> NodeProperties:
         sh = self._shapes_graph.lookup_shape_from_node(iri)
@@ -58,34 +63,53 @@ def resolve_context(self, iri: str) -> Dict:
     def generate_context(self) -> Dict:
         return self._generate_context()
 
-    def _build_shapes_map(self) -> Tuple[Dict[URIRef, str], Dict[str, URIRef]]:
-        query = """
-            PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
-            PREFIX sh: <http://www.w3.org/ns/shacl#>
-            SELECT ?type ?shape WHERE {
-                { ?shape sh:targetClass ?type .}
-                UNION {
-                    SELECT (?shape as ?type) ?shape WHERE {
-                        ?shape a sh:NodeShape .
-                        ?shape a rdfs:Class
-                    }
-                }
-            } ORDER BY ?type
-        """
-        res = self._graph.query(query)
-
-        class_to_shape: Dict[str, URIRef] = {
-            row["type"]: URIRef(row["shape"])
-            for row in res
-        }
-
-        # FIXME should return the file path where the schema is in
-        shape_to_file = dict(
-            (e, "")  # TODO file source
-            for e in class_to_shape.values()
-        )
+    def _build_shapes_map(
+            self, dir_path: Path
+    ) -> Tuple[Graph, Dict[URIRef, str], Dict[str, URIRef]]:
 
-        return shape_to_file, class_to_shape
+        query = """
+                    PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+                    PREFIX sh: <http://www.w3.org/ns/shacl#>
+                    SELECT ?type ?shape WHERE {
+                        { ?shape sh:targetClass ?type .}
+                        UNION {
+                            SELECT (?shape as ?type) ?shape WHERE {
+                                ?shape a sh:NodeShape .
+                                ?shape a rdfs:Class
+                            }
+                        }
+                    } ORDER BY ?type
+                """
+
+        class_to_shape: Dict[str, URIRef] = dict()
+        shape_to_file: Dict[URIRef, str] = dict()
+        graph = Graph()
+
+        extensions = [".ttl", ".n3", ".json", ".rdf"]
+        for f in dir_path.rglob(os.path.join("*.*")):
+            graph_i = Graph()
+            if f.suffix in extensions:
+                file_format = guess_format(f.name)
+                if file_format is None:
+                    file_format = "json-ld"
+                graph_i.parse(f.as_posix(), format=file_format)
+
+            res = graph_i.query(query)
+
+            class_to_shape_i = dict(
+                (row["type"], URIRef(row["shape"]))
+                for row in res
+            )
+            class_to_shape.update(class_to_shape_i)
+
+            shape_to_file.update(dict(
+                (e, f.as_posix())
+                for e in class_to_shape_i.values()
+            ))
+
+            graph += graph_i
+
+        return graph, shape_to_file, class_to_shape
 
 
 def load_rdf_files_into_graph(path: Path, memory_graph: Graph) -> Graph:
diff --git a/kgforge/specializations/models/rdf/rdf_model_service_from_store.py b/kgforge/specializations/models/rdf/rdf_model_service_from_store.py
@@ -46,9 +46,13 @@ def __init__(self, default_store: Store, context_iri: Optional[str] = None,
 
         self._imported = []
 
-        g = Graph()
-        self._shapes_graph = ShapesGraphWrapper(g)
-        super().__init__(g, context_iri)
+        graph, shape_to_resource, class_to_shape = self._build_shapes_map()
+        self._shapes_graph = ShapesGraphWrapper(graph)
+
+        super().__init__(
+            graph=graph, context_iri=context_iri, shape_to_source=shape_to_resource,
+            class_to_shape=class_to_shape
+        )
 
     def materialize(self, iri: URIRef) -> NodeProperties:
         shape: ShapeWrapper = self._load_and_get_type_shape(iri)
@@ -75,7 +79,7 @@ def generate_context(self) -> Dict:
 
         return self._generate_context()
 
-    def _build_shapes_map(self) -> Tuple[Dict[URIRef, str], Dict[str, URIRef]]:
+    def _build_shapes_map(self) -> Tuple[Graph, Dict[URIRef, str], Dict[str, URIRef]]:
         query = f"""
             PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
             PREFIX sh: <http://www.w3.org/ns/shacl#>
@@ -95,8 +99,8 @@ def _build_shapes_map(self) -> Tuple[Dict[URIRef, str], Dict[str, URIRef]]:
         limit = 100
         offset = 0
         count = limit
-        class_to_shape = dict()
-        shape_to_resource: Dict[URIRef, URIRef] = dict()
+        class_to_shape: Dict[str, URIRef] = dict()
+        shape_to_resource: Dict[URIRef, str] = dict()
 
         while count == limit:
             resources = self.context_store.sparql(query, debug=False, limit=limit, offset=offset)
@@ -107,7 +111,7 @@ def _build_shapes_map(self) -> Tuple[Dict[URIRef, str], Dict[str, URIRef]]:
             count = len(resources)
             offset += count
 
-        return shape_to_resource, class_to_shape
+        return Graph(), shape_to_resource, class_to_shape
 
     def recursive_resolve(self, context: Union[Dict, List, str]) -> Dict:
         document = {}
diff --git a/kgforge/specializations/models/rdf_model.py b/kgforge/specializations/models/rdf_model.py
@@ -103,9 +103,9 @@ def get_shape_from_type(self, type: str) -> URIRef:
             raise ValueError(f"Type {type} not found")
         return self.service.types_to_shapes[type]
 
-    def schema_id(self, type: str) -> URIRef:
+    def schema_source(self, type: str) -> str:
         shape_iri: URIRef = self.get_shape_from_type(type)
-        return self.service.shape_source(shape_iri)
+        return self.service.get_shape_source(shape_iri)
 
     # Validation.
 
diff --git a/tests/specializations/models/data.py b/tests/specializations/models/data.py
@@ -13,6 +13,7 @@
 # along with Blue Brain Nexus Forge. If not, see <https://choosealicense.com/licenses/lgpl-3.0/>.
 from copy import deepcopy
 
+from utils import full_path_relative_to_root
 
 ORGANIZATION = {
     "id": "",
@@ -134,4 +135,14 @@
     "Organization": "http://www.example.com/OrganizationShape",
     "Person": "http://www.example.com/PersonShape",
     "PostalAddress": "http://schema.org/PostalAddress",
-}
+}
+
+SCHEMA_SOURCE_MAP = {
+    "Activity": full_path_relative_to_root('tests/data/shacl-model/commons/shapes-2.json'),
+    "Association": full_path_relative_to_root('tests/data/shacl-model/commons/shapes-1.json'),
+    "Building": full_path_relative_to_root('tests/data/shacl-model/commons/shapes-3.json'),
+    "Employee": full_path_relative_to_root('tests/data/shacl-model/commons/shapes-1.json'),
+    "Organization": full_path_relative_to_root('tests/data/shacl-model/commons/shapes-1.json'),
+    "Person": full_path_relative_to_root('tests/data/shacl-model/commons/shapes-1.json'),
+    "PostalAddress": full_path_relative_to_root('tests/data/shacl-model/commons/shapes-1.json'),
+}
diff --git a/tests/specializations/models/test_rdf_model.py b/tests/specializations/models/test_rdf_model.py
@@ -99,11 +99,35 @@ def valid_activity_resource(self, activity_json):
         return resource
 
     @pytest.mark.parametrize("type_,", TYPES_SCHEMAS_MAP.keys())
-    @pytest.mark.xfail
     def test_type_to_schema(self, rdf_model: RdfModel, type_):
-        # FIXME TYPES_SCHEMAS_MAP should be a type to file dictionary
-        # see _build_shapes_map from RdfModelServiceFromDirectory
-        assert rdf_model.schema_id(type_) == URIRef(TYPES_SCHEMAS_MAP[type_])
+
+        # The problem:
+        # For DirectoryService,
+        # the best way to track the file from which a schema originates is
+        # - before building the shapes map
+        # - on service initialisation, when graph loading (gets all schemas),
+
+        # For StoreService,
+        # the best way to track the file from which a schema originates is
+        # - when building the shapes map, querying the store to get resource_id
+        # - not on service initialisation, no graph loading
+        #   (empty graph provided, lazily loaded on request)
+
+        # Solution: graph loading should happen in building the shapes map. Shape loading returns:
+        #   the graph with the shapes (empty for Store, full for Directory)
+        #   shape_to_source
+        #   class_to_shape
+
+        # Would mean that in
+        # - RdfModelServiceFromStore g = Graph() would happen in the implementation of
+        # _build_shapes_map, and not in constructor
+        # - RdfModelFromDirectory load_rdf_files_into_graph() would happen in the implementation of
+        # _build_shapes_map, and not in constructor
+
+        # - RdfModelService: self.shape_to_source, self.class_to_shape can be parameters of the
+        # constructor of this abstract class, and they'd be passed to super by the implementations
+
+        assert rdf_model.schema_source(type_) == SCHEMA_SOURCE_MAP[type_]
 
     def test_validate_one(self, rdf_model: RdfModel, valid_activity_resource):
         rdf_model.validate(valid_activity_resource, False, type_="Activity")
