Merge pull request crs4#177 from kikkomep/fix/issue-176

fix: support non-HTTP URI schemes for (Web)Data Entities (crs4#176)

Author: kikkomep

rocrate_validator/errors.py

@@ -12,7 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typing import Optional
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING, Optional, Union
+
+if TYPE_CHECKING:
+    # Imported only for type-checking to avoid a circular import:
+    # rocrate_validator.utils.uri imports this module at runtime.
+    from rocrate_validator.utils.uri import URI
 
 
 class ROCValidatorError(Exception):
@@ -243,29 +251,34 @@ def __repr__(self):
 class ROCrateInvalidURIError(ROCValidatorError):
     """Raised when an invalid URI is provided."""
 
-    def __init__(self, uri: str, message: Optional[str] = None):
+    def __init__(self, uri: Union[str, Path, URI], message: Optional[str] = None):
         self._uri = uri
         self._message = message or self.default_error_message(uri)
 
     @property
-    def uri(self) -> Optional[str]:
-        """The invalid URI."""
+    def uri(self) -> Union[str, Path, URI]:
+        """The invalid URI, as originally provided (str, Path, or URI)."""
         return self._uri
 
     @property
-    def message(self) -> Optional[str]:
+    def uri_string(self) -> str:
+        """The invalid URI normalised to its string form."""
+        return str(self._uri)
+
+    @property
+    def message(self) -> str:
         """The error message."""
         return self._message
 
     def __str__(self) -> str:
         return self._message
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         return f"ROCrateInvalidURIError({self._uri!r})"
 
     @classmethod
-    def default_error_message(cls, uri: str) -> str:
-        return f"\"{uri}\" is not a valid RO-Crate URI. "\
+    def default_error_message(cls, uri: Union[str, Path, URI]) -> str:
+        return f"\"{str(uri)}\" is not a valid RO-Crate URI. "\
             "It MUST be either a local path to the RO-Crate root directory or a local/remote RO-Crate ZIP file."
 
 

rocrate_validator/profiles/ro-crate/must/4_data_entity_metadata.py

@@ -38,11 +38,13 @@ def check_availability(self, context: ValidationContext) -> bool:
             return True
         # Perform the check
         result = True
+        # Web-based Data Entities (absolute URIs with any scheme other than `file`,
+        # e.g. http://, https://, ftp://, scp://, s3://, ...) are not required to
+        # be part of the local payload per the RO-Crate specification.
         for entity in context.ro_crate.metadata.get_data_entities(exclude_web_data_entities=True):
             assert entity.id is not None, "Entity has no @id"
             logger.debug("Ensure the presence of the Data Entity '%s' within the RO-Crate", entity.id)
             try:
-                logger.debug("Ensure the presence of the Data Entity '%s' within the RO-Crate", entity.id)
                 if entity.has_local_identifier():
                     logger.debug(
                         "Ignoring the Data Entity '%s' as it is a local entity with a local identifier. "

rocrate_validator/profiles/ro-crate/should/5_web_data_entity_metadata.py

@@ -16,6 +16,7 @@
 from rocrate_validator.models import ValidationContext
 from rocrate_validator.requirements.python import (PyFunctionCheck, check,
                                                    requirement)
+from rocrate_validator.utils.uri import AvailabilityStatus
 
 # set up logging
 logger = logging.getLogger(__name__)
@@ -32,13 +33,37 @@ class WebDataEntityRecommendedChecker(PyFunctionCheck):
     def check_availability(self, context: ValidationContext) -> bool:
         """
         Check if the Web-based Data Entity is directly downloadable
-        by a simple retrieval (e.g. HTTP GET) permitting redirection and HTTP/HTTPS URIs
+        by a simple retrieval (e.g. HTTP GET) permitting redirection and HTTP/HTTPS URIs.
+
+        Resources that cannot be natively retrieved by the validator (e.g.
+        `scp://`, `s3://`, `sftp://`) or that are protected by an authorization
+        mechanism (HTTP 401/403) are reported as recommendation-level issues
+        and logged as warnings, without invalidating the validation.
         """
         result = True
         for entity in context.ro_crate.metadata.get_web_data_entities():
             assert entity.id is not None, "Entity has no @id"
             try:
-                if not entity.is_available():
+                status = entity.check_availability()
+                if status == AvailabilityStatus.AVAILABLE:
+                    continue
+                if status == AvailabilityStatus.UNAUTHORIZED:
+                    msg = (
+                        f"Web-based Data Entity {entity.id} is protected by an "
+                        f"authorization mechanism; availability could not be verified"
+                    )
+                    logger.warning(msg)
+                    context.result.add_issue(msg, self)
+                elif status == AvailabilityStatus.UNCHECKABLE:
+                    scheme = entity.id_as_uri.scheme
+                    msg = (
+                        f"Web-based Data Entity {entity.id} uses scheme "
+                        f"'{scheme}' which is not natively supported by the "
+                        f"validator; availability could not be verified"
+                    )
+                    logger.warning(msg)
+                    context.result.add_issue(msg, self)
+                else:
                     context.result.add_issue(
                         f'Web-based Data Entity {entity.id} is not available', self)
                     result = False
@@ -59,6 +84,12 @@ def check_content_size(self, context: ValidationContext) -> bool:
         result = True
         for entity in context.ro_crate.metadata.get_web_data_entities():
             assert entity.id is not None, "Entity has no @id"
+            # Skip entities whose scheme the validator cannot natively fetch
+            # (e.g. scp://, s3://): without retrieving the content there is
+            # no actual size to compare `contentSize` against. Reachability
+            # is then checked separately via `is_available()` below.
+            if not entity.id_as_uri.is_natively_checkable():
+                continue
             if entity.is_available():
                 content_size = entity.get_property("contentSize")
                 if content_size and int(content_size) != context.ro_crate.get_external_file_size(entity.id):

rocrate_validator/rocrate.py

@@ -30,7 +30,7 @@
 from rocrate_validator.errors import ROCrateInvalidURIError
 from rocrate_validator.utils.uri import validate_rocrate_uri
 from rocrate_validator.utils.http import HttpRequester
-from rocrate_validator.utils.uri import URI
+from rocrate_validator.utils.uri import URI, AvailabilityStatus, is_external_reference
 
 # set up logging
 logger = logging.getLogger(__name__)
@@ -140,8 +140,17 @@ def id_as_path(self) -> Path:
     @classmethod
     def get_id_as_uri(cls, entity_id: str, ro_crate: ROCrate) -> URI:
         assert entity_id, "Entity ID cannot be None"
-        if entity_id.startswith("http"):
+        # Per RO-Crate 1.1 § 4.2.2, an `@id` is either a relative URI path or
+        # an external URI/IRI (RFC 3986/3987). External references are used
+        # as-is (without resolving them against the crate URI) so the entity
+        # is classified as remote/web-based; this covers both authority-based
+        # forms (``http://``, ``scp://``) and scheme-only ones (``urn:``,
+        # ``doi:``, ``arcp:``).
+        if is_external_reference(entity_id):
             return URI(entity_id)
+        # Otherwise the `@id` is a relative path: if the RO-Crate itself is
+        # remote, resolve it against the crate URI so the entity is still
+        # classified as remote/web-based.
         if ro_crate.uri.is_remote_resource():
             if entity_id.startswith("./"):
                 return URI(f"{ro_crate.uri}/{entity_id[2:]}")
@@ -208,58 +217,66 @@ def raw_data(self) -> object:
     def is_local(self) -> bool:
         return not self.is_remote()
 
-    def is_available(self) -> bool:
+    def check_availability(self) -> AvailabilityStatus:
+        """
+        Return a fine-grained availability status for this entity.
+
+        This is the primary check; :meth:`is_available` is the boolean
+        shortcut built on top of it. The status distinguishes definitely
+        unavailable resources, auth-protected ones, and remote URIs whose
+        scheme the validator cannot natively check (scp://, s3://, ...).
+        """
         try:
-            # check if the entity points to an external file
-            if self.id.startswith("http"):
+            entity_uri = self.id_as_uri
+            # Remote entities with a scheme we can natively reach are checked
+            # by inspecting the remote response status.
+            if entity_uri.is_natively_checkable():
                 logger.debug("Checking the availability of a remote entity")
-                return self.ro_crate.get_external_file_size(self.id) > 0
-
-            # check if the entity is part of the local RO-Crate
+                return entity_uri.check_availability()
+
+            # Remote entities with a non-natively-checkable scheme cannot be
+            # verified (scp://, sftp://, s3://, ...): report UNCHECKABLE so
+            # callers can warn without invalidating the validation.
+            if entity_uri.is_remote_resource():
+                logger.debug(
+                    "Cannot natively verify availability for entity '%s' (scheme '%s')",
+                    self.id,
+                    entity_uri.scheme,
+                )
+                return AvailabilityStatus.UNCHECKABLE
+
+            # Local entity: locate it inside the (local or remote) RO-Crate.
             if self.ro_crate.uri.is_local_resource():
-                # check if the file exists in the local file system
                 if isinstance(self.ro_crate, ROCrateLocalFolder):
-                    logger.debug(
-                        "Checking the availability of a local entity in a local folder"
-                    )
-                    return self.ro_crate.has_file(
-                        self.id_as_path
-                    ) or self.ro_crate.has_directory(self.id_as_path)
-                # check if the file exists in the local zip file
+                    found = self.ro_crate.has_file(self.id_as_path) or self.ro_crate.has_directory(self.id_as_path)
+                    return AvailabilityStatus.AVAILABLE if found else AvailabilityStatus.UNAVAILABLE
                 if isinstance(self.ro_crate, ROCrateLocalZip):
-                    logger.debug(
-                        "Checking the availability of a local entity in a local zip file"
-                    )
-                    # Skip the check for the root of a ZIP archive
                     if self.id == "./":
-                        logger.debug(
-                            "Skipping the check for the presence of the Data Entity '%s' within the RO-Crate "
-                            "as it is the root of a ZIP archive",
-                            self.id,
-                        )
-                        return True
-                    return self.ro_crate.has_directory(
+                        return AvailabilityStatus.AVAILABLE
+                    found = self.ro_crate.has_directory(unquote(str(self.id))) or self.ro_crate.has_file(
                         unquote(str(self.id))
-                    ) or self.ro_crate.has_file(unquote(str(self.id)))
+                    )
+                    return AvailabilityStatus.AVAILABLE if found else AvailabilityStatus.UNAVAILABLE
 
-            # check if the entity is part of the remote RO-Crate
-            logger.debug(
-                "Checking the availability of a remote entity in a remote RO-Crate"
-            )
             if self.ro_crate.uri.is_remote_resource():
                 if self.id == "./":
-                    return self.ro_crate.get_file_size(Path(self.id_as_uri())) > 0
-                return self.ro_crate.has_directory(
-                    unquote(str(self.id))
-                ) or self.ro_crate.has_file(unquote(str(self.id)))
+                    found = self.ro_crate.get_file_size(Path(self.id_as_uri())) > 0
+                else:
+                    found = self.ro_crate.has_directory(unquote(str(self.id))) or self.ro_crate.has_file(
+                        unquote(str(self.id))
+                    )
+                return AvailabilityStatus.AVAILABLE if found else AvailabilityStatus.UNAVAILABLE
         except Exception as e:
             if logger.isEnabledFor(logging.DEBUG):
                 logger.exception(e)
-            return False
+            return AvailabilityStatus.UNAVAILABLE
 
-        raise ROCrateInvalidURIError(
-            uri=self.id, message="Could not determine the availability of the entity"
-        )
+        # Fallthrough: the crate URI is neither a recognized local nor a
+        # remote resource — the entity location cannot be determined.
+        raise ROCrateInvalidURIError(uri=self.id, message="Could not determine the availability of the entity")
+
+    def is_available(self) -> bool:
+        return self.check_availability() == AvailabilityStatus.AVAILABLE
 
     def get_size(self) -> int:
         try: