From c6b7b50c200db6bcd0aa3ada0cfe4ed187e7cb62 Mon Sep 17 00:00:00 2001 From: Tasko Olevski <16360283+olevski@users.noreply.github.com> Date: Wed, 22 Jul 2026 14:40:25 +0200 Subject: [PATCH] feat: parse scicat metadata to get s3 configs --- .../data_connectors/core.py | 61 ++++++++++++++++++- .../data_connectors/doi/metadata.py | 5 ++ .../data_connectors/doi/models.py | 3 + .../data_connectors/doi/schema_org.py | 35 ++++++++++- .../renku_data_services/storage/constants.py | 3 +- 5 files changed, 102 insertions(+), 5 deletions(-) diff --git a/components/renku_data_services/data_connectors/core.py b/components/renku_data_services/data_connectors/core.py index a5515ac35b..f153367e41 100644 --- a/components/renku_data_services/data_connectors/core.py +++ b/components/renku_data_services/data_connectors/core.py @@ -50,7 +50,11 @@ from renku_data_services.data_connectors.config import DepositConfig from renku_data_services.data_connectors.constants import ALLOWED_GLOBAL_DATA_CONNECTOR_PROVIDERS from renku_data_services.data_connectors.doi import schema_org -from renku_data_services.data_connectors.doi.metadata import create_envidat_metadata_url, get_dataset_metadata +from renku_data_services.data_connectors.doi.metadata import ( + create_envidat_metadata_url, + create_scicat_metadata_url, + get_dataset_metadata, +) from renku_data_services.data_connectors.doi.models import DOI, SchemaOrgDataset from renku_data_services.k8s.client_interfaces import K8sClient from renku_data_services.k8s.clients import DepositUploadJobClient @@ -58,7 +62,7 @@ from renku_data_services.k8s.models import GVK, K8sObject, K8sObjectMeta from renku_data_services.notebooks.data_sources import DataSourceRepository from renku_data_services.storage import models as storage_models -from renku_data_services.storage.constants import ENVIDAT_V1_PROVIDER +from renku_data_services.storage.constants import ENVIDAT_V1_PROVIDER, SCICAT_V1_PROVDER from renku_data_services.storage.rclone import RCloneDOIMetadata, RCloneValidator from renku_data_services.utils.core import get_openbis_pat @@ -151,6 +155,11 @@ async def validate_unsaved_storage_doi( configuration = converted_storage.configuration source_path = converted_storage.source_path or "/" storage_type = ENVIDAT_V1_PROVIDER + case "doi.psi.ch" | "www.doi.psi.ch": + converted_storage = await convert_scicat_v1_data_connector_to_s3(storage) + configuration = converted_storage.configuration + source_path = converted_storage.source_path or "/" + storage_type = SCICAT_V1_PROVDER case _: # Most likely supported by rclone doi provider, you have to call validator.get_doi_metadata to confirm configuration = storage.configuration @@ -564,6 +573,54 @@ async def convert_envidat_v1_data_connector_to_s3( return new_config +async def convert_scicat_v1_data_connector_to_s3( + payload: apispec.CloudStorageCorePost, +) -> apispec.CloudStorageCorePost: + """Converts a doi-like configuration for Scicat to S3.""" + config = payload.configuration + doi = config.get("doi") + if not isinstance(doi, str): + if doi is None: + raise errors.ValidationError( + message="Cannot get configuration for Envidat data connector because " + "the doi is missing from the payload." + ) + raise errors.ValidationError( + message=f"Cannot get configuration for Envidat data connector because the doi '{doi}' " + "in the payload is not a string." + ) + if len(doi) == 0: + raise errors.ValidationError( + message="Cannot get configuration for Envidat data connector because the doi is a string with zero length." + ) + doi = DOI(doi) + + new_config = payload.model_copy(deep=True) + new_config.configuration = {} + + envidat_url = create_scicat_metadata_url(doi) + headers = {"accept": "application/ld+json"} + + clnt = httpx.AsyncClient(follow_redirects=True, timeout=5) + async with clnt: + res = await clnt.get(envidat_url, headers=headers) + if res.status_code != 200: + raise errors.ValidationError( + message="Cannot get configuration for Envidat data connector because Envidat responded " + f"with an unexpected {res.status_code} status code at {res.url}.", + detail=f"Response from envidat: {res.text}", + ) + dataset = SchemaOrgDataset.model_validate_json(res.text) + s3_config = schema_org.get_rclone_config( + dataset, + schema_org.DatasetProvider.scicat, + ) + new_config.configuration = dict(s3_config.rclone_config) + new_config.source_path = s3_config.path + new_config.storage_type = "s3" + return new_config + + def validate_deposit(body: apispec.DepositPost, original_id: str) -> models.UnsavedDepositJob: """Validate the payload to creation of a deposit.""" dc_id = ULID.from_str(body.data_connector_id) diff --git a/components/renku_data_services/data_connectors/doi/metadata.py b/components/renku_data_services/data_connectors/doi/metadata.py index 7811f5ffe1..d0ccd019f8 100644 --- a/components/renku_data_services/data_connectors/doi/metadata.py +++ b/components/renku_data_services/data_connectors/doi/metadata.py @@ -124,6 +124,11 @@ def create_envidat_metadata_url(doi: models.DOI) -> str: return f"{url}?{params}" +def create_scicat_metadata_url(doi: models.DOI) -> str: + """Create the metadata url for envidat from a DOI.""" + return f"https://doi.psi.ch/detail/{doi}" + + async def _get_envidat_metadata(metadata_url: str) -> models.DOIMetadata | None: """Get metadata about the envidat dataset.""" clnt = httpx.AsyncClient(follow_redirects=True, timeout=5) diff --git a/components/renku_data_services/data_connectors/doi/models.py b/components/renku_data_services/data_connectors/doi/models.py index d5a4d8fd25..0f3cc11fd3 100644 --- a/components/renku_data_services/data_connectors/doi/models.py +++ b/components/renku_data_services/data_connectors/doi/models.py @@ -4,6 +4,7 @@ import re from dataclasses import dataclass +from datetime import datetime from typing import Any, Self from urllib.parse import urlparse @@ -149,6 +150,8 @@ class SchemaOrgDistribution(BaseModel): model_config = ConfigDict(extra="ignore") type: str = Field(alias="@type") content_url: str = Field(alias="contentUrl") + name: str | None = None + expires: datetime | None = None class SchemaOrgDataset(BaseModel): diff --git a/components/renku_data_services/data_connectors/doi/schema_org.py b/components/renku_data_services/data_connectors/doi/schema_org.py index 19b98aa9c5..1b2671225d 100644 --- a/components/renku_data_services/data_connectors/doi/schema_org.py +++ b/components/renku_data_services/data_connectors/doi/schema_org.py @@ -13,6 +13,7 @@ class DatasetProvider(StrEnum): """The provider for the dataset.""" envidat = "envidat" + scicat = "scicat" @dataclass @@ -35,7 +36,8 @@ def get_rclone_config(dataset: SchemaOrgDataset, provider: DatasetProvider) -> S match provider: case DatasetProvider.envidat: return __get_rclone_s3_config_envidat(dataset) - # TODO: Add scicat here + case DatasetProvider.scicat: + return __get_rclone_s3_config_scicat(dataset) case _: raise errors.ValidationError(message=f"Got an unknown dataset provider {provider}") @@ -75,3 +77,34 @@ def __get_rclone_s3_config_envidat(dataset: SchemaOrgDataset) -> S3Config: bucket, prefix, ) + + +def __get_rclone_s3_config_scicat(dataset: SchemaOrgDataset) -> S3Config: + """Get the S3 rclone configuration and source path from a dataset returned by scicat. + + A single dataset may contain more than one S3 url. + The S3 information is encoded in the distribution, in fields where the name is 'S3 URI'. + """ + output: list[S3Config] = [] + for dist in dataset.distribution: + if dist.name and dist.name == "S3 URI": + parsed = urlparse(dist.content_url) + query_parsed = parse_qs(parsed.query) + bucket = parsed.path + prefix = query_parsed.get("prefix", [None])[0] + if not bucket: + raise errors.ValidationError(message="The S3 bucket from scicat metadata cannot be found") + if not prefix: + raise errors.ValidationError(message="The S3 prefix from scicat metadata cannot be found") + output.append( + S3Config( + rclone_config={ + "type": "s3", + "provider": "Other", + "endpoint": f"{parsed.scheme}://{parsed.hostname}", + }, + bucket=bucket, + prefix=prefix, + ) + ) + return output[0] diff --git a/components/renku_data_services/storage/constants.py b/components/renku_data_services/storage/constants.py index c2152171e4..c671a65b65 100644 --- a/components/renku_data_services/storage/constants.py +++ b/components/renku_data_services/storage/constants.py @@ -4,8 +4,7 @@ from typing import Final ENVIDAT_V1_PROVIDER: Final[str] = "envidat_v1" -# TODO: where is this constants used? -# SCICAT_V1_PROVIDER: Final[str] = "scicat_v1" +SCICAT_V1_PROVDER: Final[str] = "scicat_v1" @dataclass(frozen=True, eq=True, kw_only=True)