feat: import datasets from scicat

components/renku_data_services/data_connectors/core.py

@@ -50,15 +50,19 @@
 from renku_data_services.data_connectors.config import DepositConfig
 from renku_data_services.data_connectors.constants import ALLOWED_GLOBAL_DATA_CONNECTOR_PROVIDERS
 from renku_data_services.data_connectors.doi import schema_org
-from renku_data_services.data_connectors.doi.metadata import create_envidat_metadata_url, get_dataset_metadata
+from renku_data_services.data_connectors.doi.metadata import (
+    create_envidat_metadata_url,
+    create_scicat_metadata_url,
+    get_dataset_metadata,
+)
 from renku_data_services.data_connectors.doi.models import DOI, SchemaOrgDataset
 from renku_data_services.k8s.client_interfaces import K8sClient
 from renku_data_services.k8s.clients import DepositUploadJobClient
 from renku_data_services.k8s.constants import DEFAULT_K8S_CLUSTER, ClusterId
 from renku_data_services.k8s.models import GVK, K8sObject, K8sObjectMeta
 from renku_data_services.notebooks.data_sources import DataSourceRepository
 from renku_data_services.storage import models as storage_models
-from renku_data_services.storage.constants import ENVIDAT_V1_PROVIDER
+from renku_data_services.storage.constants import ENVIDAT_V1_PROVIDER, SCICAT_V1_PROVDER
 from renku_data_services.storage.rclone import RCloneDOIMetadata, RCloneValidator
 from renku_data_services.utils.core import get_openbis_pat
 
@@ -151,6 +155,11 @@ async def validate_unsaved_storage_doi(
             configuration = converted_storage.configuration
             source_path = converted_storage.source_path or "/"
             storage_type = ENVIDAT_V1_PROVIDER
+        case "doi.psi.ch" | "www.doi.psi.ch":
+            converted_storage = await convert_scicat_v1_data_connector_to_s3(storage)
+            configuration = converted_storage.configuration
+            source_path = converted_storage.source_path or "/"
+            storage_type = SCICAT_V1_PROVDER
         case _:
             # Most likely supported by rclone doi provider, you have to call validator.get_doi_metadata to confirm
             configuration = storage.configuration
@@ -564,6 +573,54 @@ async def convert_envidat_v1_data_connector_to_s3(
     return new_config
 
 
+async def convert_scicat_v1_data_connector_to_s3(
+    payload: apispec.CloudStorageCorePost,
+) -> apispec.CloudStorageCorePost:
+    """Converts a doi-like configuration for Scicat to S3."""
+    config = payload.configuration
+    doi = config.get("doi")
+    if not isinstance(doi, str):
+        if doi is None:
+            raise errors.ValidationError(
+                message="Cannot get configuration for Envidat data connector because "
+                "the doi is missing from the payload."
+            )
+        raise errors.ValidationError(
+            message=f"Cannot get configuration for Envidat data connector because the doi '{doi}' "
+            "in the payload is not a string."
+        )
+    if len(doi) == 0:
+        raise errors.ValidationError(
+            message="Cannot get configuration for Envidat data connector because the doi is a string with zero length."
+        )
+    doi = DOI(doi)
+
+    new_config = payload.model_copy(deep=True)
+    new_config.configuration = {}
+
+    envidat_url = create_scicat_metadata_url(doi)
+    headers = {"accept": "application/ld+json"}
+
+    clnt = httpx.AsyncClient(follow_redirects=True, timeout=5)
+    async with clnt:
+        res = await clnt.get(envidat_url, headers=headers)
+        if res.status_code != 200:
+            raise errors.ValidationError(
+                message="Cannot get configuration for Envidat data connector because Envidat responded "
+                f"with an unexpected {res.status_code} status code at {res.url}.",
+                detail=f"Response from envidat: {res.text}",
+            )
+    dataset = SchemaOrgDataset.model_validate_json(res.text)
+    s3_config = schema_org.get_rclone_config(
+        dataset,
+        schema_org.DatasetProvider.scicat,
+    )
+    new_config.configuration = dict(s3_config.rclone_config)
+    new_config.source_path = s3_config.path
+    new_config.storage_type = "s3"
+    return new_config
+
+
 def validate_deposit(body: apispec.DepositPost, original_id: str) -> models.UnsavedDepositJob:
     """Validate the payload to creation of a deposit."""
     dc_id = ULID.from_str(body.data_connector_id)

components/renku_data_services/data_connectors/doi/metadata.py

@@ -124,6 +124,11 @@ def create_envidat_metadata_url(doi: models.DOI) -> str:
     return f"{url}?{params}"
 
 
+def create_scicat_metadata_url(doi: models.DOI) -> str:
+    """Create the metadata url for envidat from a DOI."""
+    return f"https://doi.psi.ch/detail/{doi}"
+
+
 async def _get_envidat_metadata(metadata_url: str) -> models.DOIMetadata | None:
     """Get metadata about the envidat dataset."""
     clnt = httpx.AsyncClient(follow_redirects=True, timeout=5)

components/renku_data_services/data_connectors/doi/models.py

@@ -4,6 +4,7 @@
 
 import re
 from dataclasses import dataclass
+from datetime import datetime
 from typing import Any, Self
 from urllib.parse import urlparse
 
@@ -149,6 +150,8 @@ class SchemaOrgDistribution(BaseModel):
     model_config = ConfigDict(extra="ignore")
     type: str = Field(alias="@type")
     content_url: str = Field(alias="contentUrl")
+    name: str | None = None
+    expires: datetime | None = None
 
 
 class SchemaOrgDataset(BaseModel):

components/renku_data_services/data_connectors/doi/schema_org.py

@@ -13,6 +13,7 @@ class DatasetProvider(StrEnum):
     """The provider for the dataset."""
 
     envidat = "envidat"
+    scicat = "scicat"
 
 
 @dataclass
@@ -35,7 +36,8 @@ def get_rclone_config(dataset: SchemaOrgDataset, provider: DatasetProvider) -> S
     match provider:
         case DatasetProvider.envidat:
             return __get_rclone_s3_config_envidat(dataset)
-        # TODO: Add scicat here
+        case DatasetProvider.scicat:
+            return __get_rclone_s3_config_scicat(dataset)
         case _:
             raise errors.ValidationError(message=f"Got an unknown dataset provider {provider}")
 
@@ -75,3 +77,34 @@ def __get_rclone_s3_config_envidat(dataset: SchemaOrgDataset) -> S3Config:
         bucket,
         prefix,
     )
+
+
+def __get_rclone_s3_config_scicat(dataset: SchemaOrgDataset) -> S3Config:
+    """Get the S3 rclone configuration and source path from a dataset returned by scicat.
+
+    A single dataset may contain more than one S3 url.
+    The S3 information is encoded in the distribution, in fields where the name is 'S3 URI'.
+    """
+    output: list[S3Config] = []
+    for dist in dataset.distribution:
+        if dist.name and dist.name == "S3 URI":
+            parsed = urlparse(dist.content_url)
+            query_parsed = parse_qs(parsed.query)
+            bucket = parsed.path
+            prefix = query_parsed.get("prefix", [None])[0]
+            if not bucket:
+                raise errors.ValidationError(message="The S3 bucket from scicat metadata cannot be found")
+            if not prefix:
+                raise errors.ValidationError(message="The S3 prefix from scicat metadata cannot be found")
+            output.append(
+                S3Config(
+                    rclone_config={
+                        "type": "s3",
+                        "provider": "Other",
+                        "endpoint": f"{parsed.scheme}://{parsed.hostname}",
+                    },
+                    bucket=bucket,
+                    prefix=prefix,
+                )
+            )
+    return output[0]

components/renku_data_services/storage/constants.py

@@ -4,8 +4,7 @@
 from typing import Final
 
 ENVIDAT_V1_PROVIDER: Final[str] = "envidat_v1"
-# TODO: where is this constants used?
-# SCICAT_V1_PROVIDER: Final[str] = "scicat_v1"
+SCICAT_V1_PROVDER: Final[str] = "scicat_v1"
 
 
 @dataclass(frozen=True, eq=True, kw_only=True)