Merge remote-tracking branch 'origin/develop' into five-safes-on-upstream-develop

Author: EttoreM

pyproject.toml

@@ -133,6 +133,9 @@ extend-ignore-re = ["[0-9a-f]{7,40}"] # Ignore long hexadecimal strings, which a
 [tool.typos.files]
 extend-exclude = ["tests/data", "docs/diagrams", "*.json", "*.html", "*__init__.py"]
 
+[tool.typos.default.extend-words]
+TRE = "TRE"
+
 # Pylint configuration.
 # Ruff (see ruff.toml) is the primary linter/formatter; this section keeps
 # pylint consistent with it so the two tools do not disagree on style or on

rocrate_validator/profiles/ro-crate/1.1/must/1_file-descriptor_metadata.ttl

@@ -95,13 +95,12 @@ ro-crate:ROCrateMetadataFileDescriptorRecommendedProperties a sh:NodeShape ;
         sh:class schema_org:Dataset ;
         sh:message "The RO-Crate metadata file descriptor MUST have an `about` property referencing the Root Data Entity" ;
     ] ;
-    sh:property [
-        a sh:PropertyShape ;
-        sh:name "Metadata File Descriptor entity: `conformsTo` property" ;
-        sh:description """Check if the RO-Crate Metadata File Descriptor has a `conformsTo` property which points to the RO-Crate specification version""" ;
-        sh:minCount 1 ;
-        sh:nodeKind sh:IRI ;
-        sh:path dct:conformsTo ;
-        sh:hasValue <https://w3id.org/ro/crate/1.1> ;
-        sh:message "The RO-Crate metadata file descriptor MUST have a `conformsTo` property with the RO-Crate specification version" ;
-    ] .
+    sh:property ro-crate:conformsToROCrateSpec .
+
+ro-crate:conformsToROCrateSpec sh:name "Metadata File Descriptor entity: `conformsTo` property" ;
+    sh:description """Check if the RO-Crate Metadata File Descriptor has a `conformsTo` property which points to the RO-Crate specification version""" ;
+    sh:minCount 1 ;
+    sh:nodeKind sh:IRI ;
+    sh:path dct:conformsTo ;
+    sh:hasValue <https://w3id.org/ro/crate/1.1> ;
+    sh:message "The RO-Crate metadata file descriptor MUST have a `conformsTo` property with the RO-Crate specification version" .

rocrate_validator/profiles/ro-crate/1.1/must/2_root_data_entity_metadata.ttl

@@ -60,6 +60,7 @@ ro-crate:RootDataEntityType
 
 
 ro-crate:FindRootDataEntity a sh:NodeShape, validator:HiddenShape;
+    sh:order 0 ; # load this check before any other check which has sh:order. pySHACL sorts rules by sh:order before execution
     sh:name "Identify the Root Data Entity of the RO-Crate" ;
     sh:description """The Root Data Entity is the top-level Data Entity in the RO-Crate and serves as the starting point for the description of the RO-Crate.
         It is a schema:Dataset and is indirectly identified by the about property of the resource ro-crate-metadata.json in the RO-Crate

rocrate_validator/profiles/ro-crate/1.1/prefixes.ttl

@@ -46,4 +46,8 @@ ro-crate:sparqlPrefixes
     sh:declare [
         sh:prefix "ro" ;
         sh:namespace "./"^^xsd:anyURI ;
-    ] .
+    ] ;
+    sh:declare [
+        sh:prefix "ro-crate" ;
+        sh:namespace "https://github.com/crs4/rocrate-validator/profiles/ro-crate/"^^xsd:anyURI ;
+    ].

tests/integration/profiles/test_metadata_only.py

@@ -21,6 +21,7 @@
 import pytest
 
 from rocrate_validator import models
+from rocrate_validator.constants import DEFAULT_PROFILE_IDENTIFIER
 from tests.ro_crates import ValidROC
 from tests.shared import do_entity_test
 
@@ -55,7 +56,17 @@ def test_valid_ro_crates_from_folder(valid_roc_path):
         temp_path = Path(tmpdirname) / valid_roc_path.name
         shutil.copytree(valid_roc_path, temp_path)
         valid_roc_path = temp_path
-        do_entity_test(valid_roc_path, models.Severity.REQUIRED, True, [], [], metadata_only=True)
+        do_entity_test(
+            valid_roc_path,
+            models.Severity.REQUIRED,
+            True,
+            [],
+            [],
+            profile_identifier=(
+                "five-safes-crate" if "five-safes-crate" in valid_roc_path.name else DEFAULT_PROFILE_IDENTIFIER
+            ),
+            metadata_only=True,
+        )
 
 
 @pytest.mark.parametrize("valid_roc_path", valid_roc_paths())
@@ -69,5 +80,14 @@ def test_valid_ro_crates_from_metadata_dict(valid_roc_path):
         assert metadata_dict is not None, "Failed to load metadata dict"
         assert isinstance(metadata_dict, dict), "Metadata dict is not a dictionary"
         do_entity_test(
-            valid_roc_path, models.Severity.REQUIRED, True, [], [], metadata_dict=metadata_dict, metadata_only=True
+            valid_roc_path,
+            models.Severity.REQUIRED,
+            True,
+            [],
+            [],
+            profile_identifier=(
+                "five-safes-crate" if "five-safes-crate" in valid_roc_path.name else DEFAULT_PROFILE_IDENTIFIER
+            ),
+            metadata_dict=metadata_dict,
+            metadata_only=True,
         )

tests/shared.py

@@ -1,4 +1,5 @@
 # Copyright (c) 2024-2026 CRS4
+# Copyright (c) 2025-2026 eScience Lab, The University of Manchester
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -49,7 +50,6 @@ def first(c: Collection[T]) -> T:
 
 
 def load_graph_and_preserve_relative_ids(json_data, base="http://example.org/"):
-
     rel_ids = set()
 
     def collect_ids(obj):
@@ -78,54 +78,90 @@ def replace_uri_in_graph(graph, old_uri_str, new_uri_str):
         new = rdflib.URIRef(new_uri_str)
         triples_to_add = []
         triples_to_remove = []
+
         for s, p, o in graph.triples((None, None, None)):
             s2 = new if (isinstance(s, rdflib.URIRef) and str(s) == old_uri_str) else s
             o2 = new if (isinstance(o, rdflib.URIRef) and str(o) == old_uri_str) else o
+
             if (s2, p, o2) != (s, p, o):
                 triples_to_remove.append((s, p, o))
                 triples_to_add.append((s2, p, o2))
-        for t in triples_to_remove:
-            graph.remove(t)
-        for t in triples_to_add:
-            graph.add(t)
+
+        for triple in triples_to_remove:
+            graph.remove(triple)
+
+        for triple in triples_to_add:
+            graph.add(triple)
 
     for expanded, rel in mapping.items():
         replace_uri_in_graph(g, expanded, rel)
 
     return g
 
 
+def _uses_https_schema(graph: rdflib.Graph) -> bool:
+    for s, p, o in graph.triples((None, None, None)):
+        for term in (s, p, o):
+            if isinstance(term, rdflib.URIRef) and str(term).startswith("https://schema.org/"):
+                return True
+
+    return False
+
+
 def _prepare_temp_rocrate(
     rocrate_path: Path,
     rocrate_entity_patch: dict | None,
     rocrate_entity_mod_sparql: str | None,
 ) -> Path:
     # `mkdtemp` returns a stable path the test owns; `TemporaryDirectory().name`
     # was deleted on GC before copytree ran. `dirs_exist_ok=True` lets us copy
-    # into the (already-created) mkdtemp directory.
+    # into the already-created directory.
     temp_rocrate_path = Path(tempfile.mkdtemp())
-    shutil.copytree(rocrate_path, temp_rocrate_path, dirs_exist_ok=True)
-    with (temp_rocrate_path / "ro-crate-metadata.json").open(encoding="utf-8") as f:
+    shutil.copytree(
+        rocrate_path,
+        temp_rocrate_path,
+        dirs_exist_ok=True,
+    )
+
+    metadata_path = temp_rocrate_path / "ro-crate-metadata.json"
+
+    with metadata_path.open(encoding="utf-8") as f:
         rocrate = json.load(f)
+
     if rocrate_entity_patch is not None:
         for key, value in rocrate_entity_patch.items():
             for entity in rocrate["@graph"]:
                 if entity["@id"] == key:
                     entity.update(value)
                     break
-        with (temp_rocrate_path / "ro-crate-metadata.json").open("w", encoding="utf-8") as f:
+
+        with metadata_path.open("w", encoding="utf-8") as f:
             json.dump(rocrate, f)
+
     if rocrate_entity_mod_sparql is not None:
         rocrate_graph = load_graph_and_preserve_relative_ids(rocrate)
+
+        if _uses_https_schema(rocrate_graph):
+            rocrate_entity_mod_sparql = rocrate_entity_mod_sparql.replace(
+                "http://schema.org/",
+                "https://schema.org/",
+            )
+
         rocrate_graph.update(rocrate_entity_mod_sparql)
-        context_uri = "https://w3id.org/ro/crate/1.1/context"
+
+        context = rocrate.get(
+            "@context",
+            "https://w3id.org/ro/crate/1.2/context",
+        )
+
         rocrate_graph.serialize(
-            Path(temp_rocrate_path, "ro-crate-metadata.json"),
+            metadata_path,
             format="json-ld",
-            context=context_uri,
+            context=context,
             indent=2,
             use_native_types=True,
         )
+
     return temp_rocrate_path
 
 
@@ -151,30 +187,36 @@ def do_entity_test(  # pylint: disable=too-many-locals
     """
     Shared function to test a RO-Crate entity.
 
-    Additional keyword arguments (kwargs) are passed along to initialise ValidationSettings.
+    Additional keyword arguments (kwargs) are passed along to initialise
+    ValidationSettings.
     """
     assert not (rocrate_entity_patch and rocrate_entity_mod_sparql), (
         "Cannot use rocrate_entity_patch and rocrate_entity_mod_sparql together"
     )
 
-    # declare variables
     failed_requirements = None
     detected_issues = None
 
     if not isinstance(rocrate_path, Path) and not rocrate_path.startswith("http"):
         rocrate_path = Path(rocrate_path)
 
     temp_rocrate_path = None
+
     if (
         any([rocrate_entity_patch, rocrate_entity_mod_sparql])
         and isinstance(rocrate_path, Path)
         and rocrate_path.is_dir()
     ):
-        temp_rocrate_path = _prepare_temp_rocrate(rocrate_path, rocrate_entity_patch, rocrate_entity_mod_sparql)
+        temp_rocrate_path = _prepare_temp_rocrate(
+            rocrate_path,
+            rocrate_entity_patch,
+            rocrate_entity_mod_sparql,
+        )
         rocrate_path = temp_rocrate_path
 
     if expected_triggered_requirements is None:
         expected_triggered_requirements = []
+
     if expected_triggered_issues is None:
         expected_triggered_issues = []
 
@@ -183,8 +225,8 @@ def do_entity_test(  # pylint: disable=too-many-locals
         logger.debug("Requirement severity: %s", requirement_severity)
         logger.debug("Checks to skip: %s", skip_checks)
 
-        # validate RO-Crate
         relative_root_path = Path(rocrate_relative_root_path) if rocrate_relative_root_path is not None else None
+
         result: models.ValidationResult = services.validate(
             models.ValidationSettings(
                 rocrate_uri=models.URI(rocrate_path),
@@ -198,50 +240,82 @@ def do_entity_test(  # pylint: disable=too-many-locals
                 **kwargs,
             )
         )
-        logger.debug("Expected validation result: %s", expected_validation_result)
+
+        logger.debug(
+            "Expected validation result: %s",
+            expected_validation_result,
+        )
 
         assert result.context is not None, "Validation context should not be None"
+
         logger.debug(
             "Expected requirement severity to be %s, got %s",
             requirement_severity,
             result.context.requirement_severity,
         )
+
         assert result.passed() == expected_validation_result, (
             f"RO-Crate should be {'valid' if expected_validation_result else 'invalid'}"
         )
 
-        # check requirement
-        failed_requirements = [_.name for _ in result.failed_requirements]
+        failed_requirements = [requirement.name for requirement in result.failed_requirements]
 
-        # check that the expected requirements are triggered
         for expected_triggered_requirement in expected_triggered_requirements:
             if expected_triggered_requirement not in failed_requirements:
                 raise AssertionError(
-                    f"The expected requirement "
-                    f'"{expected_triggered_requirement}" was not found in the failed requirements'
+                    "The expected requirement "
+                    f'"{expected_triggered_requirement}" was not found in the '
+                    "failed requirements"
                 )
 
-        # check requirement issues
         detected_issues = [
             issue.message for issue in result.get_issues(requirement_severity) if issue.message is not None
         ]
+
         logger.debug("Detected issues: %s", detected_issues)
         logger.debug("Expected issues: %s", expected_triggered_issues)
+
         for expected_issue in expected_triggered_issues:
-            if not any(expected_issue in issue for issue in detected_issues):  # support partial match
+            if not any(expected_issue in issue for issue in detected_issues):
                 raise AssertionError(f'The expected issue "{expected_issue}" was not found in the detected issues')
+
     except Exception:
         if logger.isEnabledFor(logging.DEBUG):
-            logger.exception("Failed to validate RO-Crate @ path: %s", rocrate_path)
-            logger.debug("Requirement severity: %s", requirement_severity)
-            logger.debug("Expected validation result: %s", expected_validation_result)
-            logger.debug("Expected triggered requirements: %s", expected_triggered_requirements)
-            logger.debug("Expected triggered issues: %s", expected_triggered_issues)
-            logger.debug("Failed requirements: %s", failed_requirements)
-            logger.debug("Detected issues: %s", detected_issues)
+            logger.exception(
+                "Failed to validate RO-Crate @ path: %s",
+                rocrate_path,
+            )
+            logger.debug(
+                "Requirement severity: %s",
+                requirement_severity,
+            )
+            logger.debug(
+                "Expected validation result: %s",
+                expected_validation_result,
+            )
+            logger.debug(
+                "Expected triggered requirements: %s",
+                expected_triggered_requirements,
+            )
+            logger.debug(
+                "Expected triggered issues: %s",
+                expected_triggered_issues,
+            )
+            logger.debug(
+                "Failed requirements: %s",
+                failed_requirements,
+            )
+            logger.debug(
+                "Detected issues: %s",
+                detected_issues,
+            )
+
         raise
+
     finally:
-        # cleanup
         if temp_rocrate_path is not None:
-            logger.debug("Cleaning up temporary RO-Crate @ path: %s", temp_rocrate_path)
+            logger.debug(
+                "Cleaning up temporary RO-Crate @ path: %s",
+                temp_rocrate_path,
+            )
             shutil.rmtree(temp_rocrate_path)