From 84112895fea008ad455541f78a7c3edf0e00d564 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Thu, 30 Jul 2026 19:10:33 +0200 Subject: [PATCH 1/7] WIP: starting move --- .../data_connectors/api.spec.yaml | 240 ---------- .../data_connectors/apispec.py | 6 +- .../data_connectors/blueprints.py | 211 --------- .../data_connectors/config.py | 27 -- .../data_connectors/core.py | 79 ---- .../renku_data_services/data_connectors/db.py | 388 +--------------- .../data_connectors/models.py | 107 ----- .../data_connectors/orm.py | 89 +--- .../renku_data_services/storage/api.spec.yaml | 381 +++++++++++++++- .../renku_data_services/storage/apispec.py | 91 +++- .../renku_data_services/storage/blueprints.py | 239 +++++++++- .../renku_data_services/storage/config.py | 38 ++ .../renku_data_services/storage/core.py | 97 ++++ components/renku_data_services/storage/db.py | 422 ++++++++++++++++++ .../renku_data_services/storage/models.py | 130 +++++- components/renku_data_services/storage/orm.py | 114 +++++ 16 files changed, 1502 insertions(+), 1157 deletions(-) create mode 100644 components/renku_data_services/storage/config.py create mode 100644 components/renku_data_services/storage/core.py create mode 100644 components/renku_data_services/storage/db.py create mode 100644 components/renku_data_services/storage/orm.py diff --git a/components/renku_data_services/data_connectors/api.spec.yaml b/components/renku_data_services/data_connectors/api.spec.yaml index 215bfce7a9..e37c9532d0 100644 --- a/components/renku_data_services/data_connectors/api.spec.yaml +++ b/components/renku_data_services/data_connectors/api.spec.yaml @@ -70,226 +70,6 @@ paths: $ref: "#/components/responses/Error" tags: - data_connectors - /data_connectors/storage: - post: - summary: Create a new project storage - requestBody: - required: true - content: - application/json: - schema: - $ref: "#/components/schemas/ProjectStoragePost" - responses: - "201": - description: The data connector was created - content: - application/json: - schema: - $ref: "#/components/schemas/ProjectStorage" - default: - $ref: "#/components/responses/Error" - tags: - - data_connectors - /data_connectors/storage/config: - get: - summary: Get the current configuration for project storage - responses: - "200": - description: The configuration data - content: - application/json: - schema: - $ref: "#/components/schemas/ProjectStorageConfig" - default: - $ref: "#/components/responses/Error" - tags: - - data_connectors - /data_connectors/storage/allow: - get: - summary: List all projects in the storage allow list - parameters: - - in: query - description: query parameters - name: params - style: form - explode: true - schema: - $ref: "#/components/schemas/ProjectStorageAllowListQuery" - responses: - "200": - description: List of storage allow entries - content: - application/json: - schema: - $ref: "#/components/schemas/ProjectStorageAllowList" - headers: - page: - description: The index of the current page (starting at 1). - required: true - schema: - type: integer - per-page: - description: The number of items per page. - required: true - schema: - type: integer - total: - description: The total number of items. - required: true - schema: - type: integer - total-pages: - description: The total number of pages. - required: true - schema: - type: integer - default: - $ref: "#/components/responses/Error" - tags: - - data_connectors - post: - summary: Add a project to the storage allow list - requestBody: - required: true - content: - application/json: - schema: - $ref: "#/components/schemas/ProjectStorageAllowPost" - responses: - "201": - description: The project was added to the allow list - content: - application/json: - schema: - $ref: "#/components/schemas/ProjectStorageAllowPost" - default: - $ref: "#/components/responses/Error" - tags: - - data_connectors - /data_connectors/storage/allow/{project_id}: - parameters: - - in: path - name: project_id - required: true - schema: - $ref: "#/components/schemas/Ulid" - get: - summary: Get the storage allow entry for a project - responses: - "200": - description: The project storage allow entry - content: - application/json: - schema: - $ref: "#/components/schemas/ProjectStorageAllow" - "404": - description: The project is not in the storage allow list - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - default: - $ref: "#/components/responses/Error" - tags: - - data_connectors - patch: - summary: Change the maximum size for a project storage - parameters: - - $ref: "#/components/parameters/If-Match" - requestBody: - required: true - content: - application/json: - schema: - $ref: "#/components/schemas/ProjectStorageAllowPatch" - responses: - "200": - description: The patched project storage allow entry - content: - application/json: - schema: - $ref: "#/components/schemas/ProjectStorageAllow" - "404": - description: The project storage allow entry doesn't exist - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - default: - $ref: "#/components/responses/Error" - tags: - - data_connectors - delete: - summary: Remove a project from the storage allow list - responses: - "204": - description: The project was removed from the allow list - default: - $ref: "#/components/responses/Error" - tags: - - data_connectors - /data_connectors/storage/{storage_id}: - parameters: - - in: path - name: storage_id - required: true - schema: - $ref: "#/components/schemas/Ulid" - get: - summary: Get a project storage for a project id. - responses: - "200": - description: The project storage information - content: - application/json: - schema: - $ref: "#/components/schemas/ProjectStorage" - "404": - description: The project storage does not exist - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - default: - $ref: "#/components/responses/Error" - tags: - - data_connectors - patch: - summary: Change the size or mount path of a project storage - parameters: - - $ref: "#/components/parameters/If-Match" - requestBody: - required: true - content: - application/json: - schema: - $ref: "#/components/schemas/ProjectStoragePatch" - responses: - "200": - description: The patched project storage entry - content: - application/json: - schema: - $ref: "#/components/schemas/ProjectStorage" - "404": - description: The project storage does not exist - content: - application/json: - schema: - $ref: "#/components/schemas/ErrorResponse" - default: - $ref: "#/components/responses/Error" - tags: - - data_connectors - delete: - summary: Delete a specific project storage - responses: - "204": - description: The project storage was deleted or did not exist in the first place - default: - $ref: "#/components/responses/Error" - tags: - - data_connectors /data_connectors/global: post: summary: Create a new data connector @@ -702,26 +482,6 @@ paths: $ref: "#/components/responses/Error" tags: - data_connectors - /projects/{project_id}/storage: - parameters: - - in: path - name: project_id - required: true - schema: - $ref: "#/components/schemas/Ulid" - get: - summary: Get the project storage for a given project - responses: - "200": - description: The list of project storages (currently either one or empty). - content: - application/json: - schema: - $ref: "#/components/schemas/ProjectStorageList" - default: - $ref: "#/components/responses/Error" - tags: - - projects /projects/{project_id}/data_connector_links: parameters: - in: path diff --git a/components/renku_data_services/data_connectors/apispec.py b/components/renku_data_services/data_connectors/apispec.py index 54bf106238..73463e5815 100644 --- a/components/renku_data_services/data_connectors/apispec.py +++ b/components/renku_data_services/data_connectors/apispec.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: api.spec.yaml -# timestamp: 2026-07-30T10:50:42+00:00 +# timestamp: 2026-07-30T15:37:24+00:00 from __future__ import annotations @@ -563,10 +563,6 @@ class DataConnectorsGetParametersQuery(BaseAPISpec): params: DataConnectorsGetQuery | None = None -class DataConnectorsStorageAllowGetParametersQuery(BaseAPISpec): - params: ProjectStorageAllowListQuery | None = None - - class DataConnectorLinksGetParametersQuery(BaseAPISpec): params: DataConnectorLinksGetQuery | None = None diff --git a/components/renku_data_services/data_connectors/blueprints.py b/components/renku_data_services/data_connectors/blueprints.py index 2cb2d50773..3f418866b2 100644 --- a/components/renku_data_services/data_connectors/blueprints.py +++ b/components/renku_data_services/data_connectors/blueprints.py @@ -45,11 +45,7 @@ validate_deposit, validate_deposit_patch, validate_deposit_status_change, - validate_project_storage_allow_patch, - validate_project_storage_allow_post, - validate_project_storage_patch, validate_unsaved_data_connector, - validate_unsaved_project_storage, ) from renku_data_services.data_connectors.db import ( DOI, @@ -147,183 +143,6 @@ async def _post( return "/data_connectors", ["POST"], _post - def get_one_storage(self) -> BlueprintFactoryResponse: - """Get a specific project storage connector.""" - - @authenticate(self.authenticator) - @extract_if_none_match - async def _get_one(_: Request, user: base_models.APIUser, storage_id: ULID, etag: str | None) -> HTTPResponse: - project_storage = await self.data_connector_repo.get_project_storage(user=user, storage_id=storage_id) - if project_storage is None: - raise errors.MissingResourceError(message=f"No project storage found for storage: {storage_id}") - - if project_storage.etag == etag: - return HTTPResponse(status=304) - - headers = {"ETag": project_storage.etag} - return validated_json( - apispec.ProjectStorage, - self._dump_project_storage(project_storage), - headers=headers, - ) - - return "/data_connectors/storage/", ["GET"], _get_one - - def post_storage(self) -> BlueprintFactoryResponse: - """Create a new shared project storage.""" - - @authenticate(self.authenticator) - @only_authenticated - @validate(json=apispec.ProjectStoragePost) - async def _post_storage( - _: Request, user: base_models.APIUser, body: apispec.ProjectStoragePost - ) -> JSONResponse: - dc = validate_unsaved_project_storage(body) - result = await self.data_connector_repo.insert_project_storage(user, dc) - headers = {"ETag": result.etag} - return validated_json( - apispec.ProjectStorage, self._dump_project_storage(result), headers=headers, status=201 - ) - - return "/data_connectors/storage", ["POST"], _post_storage - - def patch_storage(self) -> BlueprintFactoryResponse: - """Partially update a project storage entry.""" - - @authenticate(self.authenticator) - @only_authenticated - @if_match_required - @validate(json=apispec.ProjectStoragePatch) - async def _patch_storage( - _: Request, - user: base_models.APIUser, - storage_id: ULID, - body: apispec.ProjectStoragePatch, - etag: str, - ) -> JSONResponse: - existing_storage = await self.data_connector_repo.get_project_storage(user=user, storage_id=storage_id) - if existing_storage is None: - raise errors.MissingResourceError(message=f"No project storage found for storage: {storage_id}") - - storage_patch = validate_project_storage_patch(existing_storage, body) - updated_storage = await self.data_connector_repo.update_project_storage( - user=user, storage_id=storage_id, patch=storage_patch, etag=etag - ) - headers = {"ETag": updated_storage.etag} - return validated_json( - apispec.ProjectStorage, - self._dump_project_storage(updated_storage), - headers=headers, - ) - - return "/data_connectors/storage/", ["PATCH"], _patch_storage - - def get_all_storage_allows(self) -> BlueprintFactoryResponse: - """List all projects in the storage allow list.""" - - @authenticate(self.authenticator) - @only_admins - @validate_query(query=apispec.ProjectStorageAllowListQuery) - @paginate - async def _get_all_storage_allows( - _: Request, - user: base_models.APIUser, - pagination: PaginationRequest, - query: apispec.ProjectStorageAllowListQuery, - ) -> tuple[list[dict[str, Any]], int]: - project_name = query.project_name if query.project_name else None - allows, total = await self.data_connector_repo.get_project_storage_allows( - user, pagination, project_name=project_name - ) - return [ - validate_and_dump( - apispec.ProjectStorageAllow, - self._dump_project_storage_allow_detail(a), - ) - for a in allows - ], total - - return "/data_connectors/storage/allow", ["GET"], _get_all_storage_allows - - def post_storage_allow(self) -> BlueprintFactoryResponse: - """Add a project to the storage allow list.""" - - @authenticate(self.authenticator) - @only_admins - @validate(json=apispec.ProjectStorageAllowPost) - async def _post_storage_allow( - _: Request, user: base_models.APIUser, body: apispec.ProjectStorageAllowPost - ) -> JSONResponse: - allow = validate_project_storage_allow_post(body) - inserted = await self.data_connector_repo.insert_project_storage_allow(user, allow) - return validated_json( - apispec.ProjectStorageAllowPost, - self._dump_project_storage_allow_post(inserted), - status=201, - ) - - return "/data_connectors/storage/allow", ["POST"], _post_storage_allow - - def patch_storage_allow(self) -> BlueprintFactoryResponse: - """Partially update a project storage allow entry.""" - - @authenticate(self.authenticator) - @only_admins - @if_match_required - @validate(json=apispec.ProjectStorageAllowPatch) - async def _patch( - _: Request, - user: base_models.APIUser, - project_id: ULID, - body: apispec.ProjectStorageAllowPatch, - etag: str, - ) -> JSONResponse: - existing_entry = await self.data_connector_repo.get_project_storage_allow(user=user, project_id=project_id) - if not existing_entry: - raise errors.MissingResourceError(message=f"No project storage allow entry for project {project_id}") - - pse_patch = validate_project_storage_allow_patch(existing_entry, body) - pse_update = await self.data_connector_repo.update_project_storage_allow( - user=user, project_id=project_id, patch=pse_patch, etag=etag - ) - - headers = {"ETag": pse_update.new.etag} - return validated_json( - apispec.ProjectStorageAllow, self._dump_project_storage_allow_detail(pse_update.new), headers=headers - ) - - return "/data_connectors/storage/allow/", ["PATCH"], _patch - - def get_storage_allow(self) -> BlueprintFactoryResponse: - """Get the storage allow entry for a project.""" - - @authenticate(self.authenticator) - @only_authenticated - async def _get_storage_allow(_: Request, user: base_models.APIUser, project_id: ULID) -> HTTPResponse: - allow = await self.data_connector_repo.get_project_storage_allow(user, project_id) - if allow is None: - raise errors.MissingResourceError(message=f"Project {project_id} is not in the storage allow list.") - - headers = {"ETag": allow.etag} - return validated_json( - apispec.ProjectStorageAllow, self._dump_project_storage_allow_detail(allow), headers=headers - ) - - return "/data_connectors/storage/allow/", ["GET"], _get_storage_allow - - def delete_storage_allow(self) -> BlueprintFactoryResponse: - """Remove a project from the storage allow list.""" - - @authenticate(self.authenticator) - @only_admins - async def _delete_storage_allow(_: Request, user: base_models.APIUser, project_id: ULID) -> HTTPResponse: - deleted = await self.data_connector_repo.delete_project_storage_allow(user, project_id) - if deleted: - await self.project_storage_k8s.delete_volume(deleted) - return HTTPResponse(status=204) - - return "/data_connectors/storage/allow/", ["DELETE"], _delete_storage_allow - def post_global(self) -> BlueprintFactoryResponse: """Create a new global data connector.""" @@ -823,36 +642,6 @@ def _dump_data_connector_secret(secret: models.DataConnectorSecret) -> dict[str, secret_id=str(secret.secret_id), ) - @staticmethod - def _dump_project_storage(ps: models.ProjectStorage) -> apispec.ProjectStorage: - return apispec.ProjectStorage( - id=str(ps.id), - project_id=str(ps.project_id), - size=int(ps.size.to_gibi()), - mount_path=ps.mount_path.as_posix(), - created_by=ps.created_by, - creation_date=ps.creation_date, - updated_at=ps.updated_at, - etag=ps.etag, - ) - - @staticmethod - def _dump_project_storage_allow_detail(ps: models.ProjectStorageAllowDetail) -> apispec.ProjectStorageAllow: - return apispec.ProjectStorageAllow( - project_id=str(ps.project_id), - max_size=int(ps.max_size.to_gibi()), - name=ps.name, - namespace=ps.namespace_path.serialize(), - etag=ps.etag, - ) - - @staticmethod - def _dump_project_storage_allow_post(ps: models.ProjectStorageAllow) -> apispec.ProjectStorageAllowPost: - return apispec.ProjectStorageAllowPost( - project_id=str(ps.project_id), - max_size=int(ps.max_size.to_gibi()), - ) - async def __get_zenodo_access_token(self, user: base_models.APIUser) -> str: provider = await self.connected_services_repo.get_provider_for_kind(user, ProviderKind.zenodo) if not provider: diff --git a/components/renku_data_services/data_connectors/config.py b/components/renku_data_services/data_connectors/config.py index ae1acac09d..f429405c0c 100644 --- a/components/renku_data_services/data_connectors/config.py +++ b/components/renku_data_services/data_connectors/config.py @@ -11,39 +11,12 @@ from kubernetes.client import ApiClient, V1Toleration from renku_data_services.app_config import logging -from renku_data_services.base_models.bytesize import ByteSize from renku_data_services.errors import errors from renku_data_services.k8s.constants import DEFAULT_K8S_CLUSTER, ClusterId logger = logging.getLogger(__name__) -@dataclass -class ProjectStorageConfig: - """The configuration for project storage.""" - - enabled: bool - storage_class: str - maximum_size: ByteSize - - @classmethod - def from_env(cls) -> ProjectStorageConfig: - """Create a configuration from environment variables.""" - - enabled = os.environ.get("PROJECT_STORAGE_ENABLED", "").lower() == "true" - storage_class = os.environ.get("PROJECT_STORAGE_STORAGE_CLASS") - maximum_size = os.environ.get("PROJECT_STORAGE_MAX_SIZE_GB") or "10" - maximum_size = ByteSize.from_gibi(int(maximum_size)) - - # TODO: set defaults for easier PR/CI deployments for now - enabled = True - storage_class = "azurefile" - if enabled and not storage_class: - raise errors.ConfigurationError(message="A storage_class is required for enabled project storage") - - return ProjectStorageConfig(enabled, storage_class=storage_class, maximum_size=maximum_size) - - @dataclass class DepositConfig: """The configuration for running data deposit uploads.""" diff --git a/components/renku_data_services/data_connectors/core.py b/components/renku_data_services/data_connectors/core.py index 9d3ec0a6d0..05b675bdd5 100644 --- a/components/renku_data_services/data_connectors/core.py +++ b/components/renku_data_services/data_connectors/core.py @@ -43,7 +43,6 @@ from renku_data_services import base_models, errors from renku_data_services.authz.models import Visibility -from renku_data_services.base_models.bytesize import ByteSize from renku_data_services.base_models.core import ( NamespacePath, ProjectPath, @@ -212,84 +211,6 @@ async def validate_unsaved_data_connector( ) -def _validate_mount_path(path: str | None) -> None: - # - invalid_prefixes = [ - "/", - "/bin", - "/sbin", - "/usr", - "/lib", - "/lib64", - "/boot", - "/etc", - "/proc", - "/sys", - "/dev", - "/run", - "/sys", - "/var", - "/tmp", # nosec B108 - "/home", - "/root", - ] - if not path or path == "": - raise errors.ValidationError(message="The mount path must not be empty") - - for prefix in invalid_prefixes: - if path == prefix or path.startswith(f"{prefix}/"): - raise errors.ValidationError(message=f"The mount path is invalid: '{path}'") - - -def validate_unsaved_project_storage(body: apispec.ProjectStoragePost) -> models.UnsavedProjectStorage: - """Validate the user input for a new project storage definition. - - The namespace must be a project namespace. The project must be - enabled for project storages and the user must be an owner. - """ - - _validate_mount_path(body.mount_path) - - namespace_path = ProjectPath.parse(body.namespace) - return models.UnsavedProjectStorage( - namespace_path=namespace_path, size=ByteSize.from_gibi(body.size), mount_path=PurePosixPath(body.mount_path) - ) - - -def validate_project_storage_patch( - existing: models.ProjectStorage, body: apispec.ProjectStoragePatch -) -> models.ProjectStoragePatch: - """Validate a patch of a project storage entry.""" - size = ByteSize.from_gibi(body.size) if body.size else None - if size and size < ByteSize.from_gibi(1): - raise errors.ValidationError(message="The size must be at least 1GB") - mount_path = PurePosixPath(body.mount_path) if body.mount_path else None - if mount_path: - _validate_mount_path(body.mount_path) - return models.ProjectStoragePatch(size=size, mount_path=mount_path) - - -def validate_project_storage_allow_post(body: apispec.ProjectStorageAllowPost) -> models.ProjectStorageAllow: - """Validate.""" - allow = models.ProjectStorageAllow( - project_id=ULID.from_str(body.project_id), max_size=ByteSize.from_gibi(body.max_size), updated_at=datetime.now() - ) - if allow.max_size < ByteSize.from_gibi(1): - raise errors.ValidationError(message=f"The maximum size must be at least 1GB, but {allow.max_size} was given.") - return allow - - -def validate_project_storage_allow_patch( - existing: models.ProjectStorageAllowDetail, body: apispec.ProjectStorageAllowPatch -) -> models.ProjectStorageAllowPatch: - """Validate a patch of a project storage allow entry.""" - - size = ByteSize.from_gibi(body.max_size) if body.max_size is not None else None - if size and size < ByteSize.from_gibi(1): - raise errors.ValidationError(message="The maximum size must be at least 1GB") - return models.ProjectStorageAllowPatch(max_size=size) - - async def prevalidate_unsaved_global_data_connector( body: apispec.GlobalDataConnectorPost, validator: RCloneValidator ) -> models.PrevalidatedGlobalDataConnector: diff --git a/components/renku_data_services/data_connectors/db.py b/components/renku_data_services/data_connectors/db.py index 4007627481..6b4e993632 100644 --- a/components/renku_data_services/data_connectors/db.py +++ b/components/renku_data_services/data_connectors/db.py @@ -8,7 +8,7 @@ from typing import TypeVar from cryptography.hazmat.primitives.asymmetric import rsa -from sqlalchemy import ColumnExpressionArgument, Select, and_, delete, exists, func, or_, select +from sqlalchemy import ColumnExpressionArgument, Select, delete, func, or_, select from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import joinedload from ulid import ULID @@ -45,7 +45,7 @@ from renku_data_services.secrets.models import SecretKind from renku_data_services.storage.rclone import RCloneValidator from renku_data_services.users.db import UserRepo -from renku_data_services.utils.core import with_db_session, with_db_transaction +from renku_data_services.utils.core import with_db_transaction class DataConnectorRepository: @@ -375,390 +375,6 @@ async def insert_namespaced_data_connector( raise errors.ProgrammingError(message=f"Expected to get a namespaced data connector ('{dc.id}')") return dc - async def get_project_storage(self, user: base_models.APIUser, storage_id: ULID) -> models.ProjectStorage | None: - """Get a project storage by its id.""" - - if user.id is None: - raise errors.UnauthorizedError(message="You do not have the required permissions for this operation.") - - async with self.session_maker() as session: - result_orm = await session.scalars( - select(schemas.ProjectStorageORM).where(schemas.ProjectStorageORM.id == storage_id) - ) - result_orm = result_orm.one_or_none() - if not result_orm: - return None - - result = result_orm.dump() - authorized = await self.authz.has_permission(user, ResourceType.project, result.project_id, Scope.READ) - if not authorized: - return None - - return result - - async def get_storage_to(self, user: base_models.APIUser, project_id: ULID) -> models.ProjectStorage | None: - """Get a project storage to a project if it exists and the feature is enabled.""" - - if not self.project_storage_config.enabled: - return None - else: - return await self._get_storage_to_project(user, project_id) - - async def _get_storage_to_project( - self, user: base_models.APIUser, project_id: ULID - ) -> models.ProjectStorage | None: - """Get a project storage to a project if it exists.""" - - if user.id is None: - raise errors.UnauthorizedError(message="You do not have the required permissions for this operation.") - - async with self.session_maker() as session: - result_orm = await session.scalars( - select(schemas.ProjectStorageORM).where(schemas.ProjectStorageORM.project_id == project_id) - ) - result_orm = result_orm.one_or_none() - if not result_orm: - return None - - result = result_orm.dump() - authorized = await self.authz.has_permission(user, ResourceType.project, result.project_id, Scope.READ) - if not authorized: - return None - - return result - - def get_project_storage_config(self) -> ProjectStorageConfig: - """Return the current config for project storage.""" - - return self.project_storage_config - - @with_db_transaction - async def insert_project_storage( - self, user: base_models.APIUser, input: models.UnsavedProjectStorage, *, session: AsyncSession | None = None - ) -> models.ProjectStorage: - """Insert a new project storage.""" - - # When the feature is disabled, we disallow insertion, but still allow managing existing data - if not self.project_storage_config.enabled: - raise errors.MissingResourceError(message="The project storage api is not enabled.") - - if not session: - raise errors.ProgrammingError(message="A database session is required.") - if user.id is None: - raise errors.UnauthorizedError(message="You do not have the required permissions for this operation.") - - # there is only one such storage possible for a project - project = await self.project_repo.get_project_by_namespace_slug( - user, input.namespace_path.first.value, input.namespace_path.second, with_documentation=False - ) - - authorized = await self.authz.has_permission(user, ResourceType.project, project.id, Scope.DELETE) - if not authorized: - raise errors.MissingResourceError( - message=f"Project with id '{project.id}' does not exist or you do not have access to it." - ) - - allowed = await session.execute( - select(schemas.ProjectStorageAllowORM).where(schemas.ProjectStorageAllowORM.project_id == project.id) - ) - allowed = allowed.scalar() - if not allowed: - raise errors.ForbiddenError(message=f"Project storage is not enabled for project {project.id}.") - - existing_storage = await session.execute( - select(exists().where(schemas.ProjectStorageORM.project_id == project.id)) - ) - existing_storage = existing_storage.scalar() - if existing_storage: - raise errors.ValidationError(message=f"There is already a project storage for project {project.id}") - - if input.size > allowed.max_size: - raise errors.ValidationError( - message=( - f"The project storage size ({input.size}) for project {project.id} " - f"exceeds the maximum size of {allowed.max_size}" - ) - ) - - new_storage = schemas.ProjectStorageORM( - project_id=project.id, - storage_class=self.project_storage_config.storage_class, - size_limit=input.size, - mount_path=input.mount_path, - created_by_id=user.id, - ) - session.add(new_storage) - await session.flush() - return new_storage.dump() - - @with_db_transaction - async def update_project_storage( - self, - user: base_models.APIUser, - storage_id: ULID, - patch: models.ProjectStoragePatch, - etag: str, - *, - session: AsyncSession | None = None, - ) -> models.ProjectStorage: - """Update some properties of a project storage entry.""" - if not session: - raise errors.ProgrammingError(message="A database session is required.") - - result = await session.scalars( - select(schemas.ProjectStorageORM).where(schemas.ProjectStorageORM.id == storage_id) - ) - storage_orm = result.one_or_none() - if storage_orm is None: - raise errors.MissingResourceError(message=f"Project storage with id '{storage_id}' does not exist.") - - # Check authorization - user must be "owner", meaning allowed to delete the project - authorized = await self.authz.has_permission(user, ResourceType.project, storage_orm.project_id, Scope.DELETE) - if not authorized: - raise errors.MissingResourceError( - message=f"Project storage with id '{storage_id}' does not exist or you do not have access to it." - ) - - current_storage = storage_orm.dump() - current_etag = current_storage.etag - if current_etag != etag: - raise errors.ConflictError(message=f"Current ETag is {current_etag}, not {etag}.") - - # Check if size would exceed the allowed maximum - new_size = patch.size if patch.size else current_storage.size - allowed = await session.execute( - select(schemas.ProjectStorageAllowORM).where( - schemas.ProjectStorageAllowORM.project_id == storage_orm.project_id - ) - ) - allowed = allowed.scalar() - if allowed and new_size > allowed.max_size: - raise errors.ValidationError( - message=( - f"The project storage size ({new_size}) for project {storage_orm.project_id} " - f"exceeds the maximum size of {allowed.max_size}" - ) - ) - - if patch.size is not None: - storage_orm.size_limit = patch.size - if patch.mount_path is not None: - storage_orm.mount_path = patch.mount_path - - await session.flush() - await session.refresh(storage_orm) - return storage_orm.dump() - - @with_db_transaction - async def insert_project_storage_allow( - self, user: base_models.APIUser, input: models.ProjectStorageAllow, *, session: AsyncSession | None = None - ) -> models.ProjectStorageAllow: - """Insert a new project storage allow entry.""" - if not session: - raise errors.ProgrammingError(message="A database session is required.") - if user.id is None or not user.is_admin: - raise errors.UnauthorizedError(message="You do not have the required permissions for this operation.") - - existing = await session.execute( - select(exists().where(schemas.ProjectStorageAllowORM.project_id == input.project_id)) - ) - if existing.scalar(): - raise errors.ValidationError(message=f"Project {input.project_id} is already in the allow list.") - - if input.max_size > self.project_storage_config.maximum_size: - raise errors.ValidationError( - message=( - f"The maximum size {input.max_size} exceeds the configured " - f"one of {self.project_storage_config.maximum_size}." - ) - ) - - new_allow = schemas.ProjectStorageAllowORM( - project_id=input.project_id, - max_size=input.max_size, - ) - session.add(new_allow) - await session.flush() - return new_allow.dump() - - @with_db_transaction - async def delete_project_storage( - self, user: base_models.APIUser, storage_id: ULID, *, session: AsyncSession | None = None - ) -> models.DeletedProjectStorage | None: - """Delete a specific project storage.""" - if not session: - raise errors.ProgrammingError(message="A database session is required.") - - result = await session.scalars( - select(schemas.ProjectStorageORM).where(schemas.ProjectStorageORM.id == storage_id) - ) - storage_orm = result.one_or_none() - if storage_orm is None: - return None - - authorized = await self.authz.has_permission(user, ResourceType.project, storage_orm.project_id, Scope.DELETE) - if not authorized: - raise errors.MissingResourceError( - message=f"Project storage with id '{storage_id}' does not exist or you do not have access to it." - ) - - await session.delete(storage_orm) - ps = storage_orm.dump() - return models.DeletedProjectStorage(project_id=ps.project_id) - - @with_db_session - async def get_project_storage_allow( - self, - user: base_models.APIUser, - project_id: ULID, - *, - session: AsyncSession | None = None, - ) -> models.ProjectStorageAllowDetail | None: - """Get the storage allow entry for a project if it exists.""" - if not session: - raise errors.ProgrammingError(message="A database session is required.") - - authorized = await self.authz.has_permission(user, ResourceType.project, project_id, Scope.READ) - if not authorized: - raise errors.MissingResourceError( - message=f"Project with id '{project_id}' does not exist or you do not have access to it." - ) - stmt = ( - select( - schemas.ProjectStorageAllowORM.project_id, - schemas.ProjectStorageAllowORM.max_size, - ProjectORM.name, - ns_schemas.NamespaceORM.slug.label("namespace_slug"), - ns_schemas.EntitySlugORM.slug.label("project_slug"), - schemas.ProjectStorageAllowORM.updated_at, - ) - .join(ProjectORM, ProjectORM.id == schemas.ProjectStorageAllowORM.project_id) - .join( - ns_schemas.EntitySlugORM, - and_( - ns_schemas.EntitySlugORM.project_id == schemas.ProjectStorageAllowORM.project_id, - ns_schemas.EntitySlugORM.data_connector_id.is_(None), - ), - ) - .join(ns_schemas.NamespaceORM, ns_schemas.NamespaceORM.id == ns_schemas.EntitySlugORM.namespace_id) - .where(schemas.ProjectStorageAllowORM.project_id == project_id) - ) - result = (await session.execute(stmt)).one_or_none() - if result: - return models.ProjectStorageAllowDetail.create(**result._mapping) - return None - - async def get_project_storage_allows( - self, user: base_models.APIUser, pagination: PaginationRequest, project_name: str | None = None - ) -> tuple[list[models.ProjectStorageAllowDetail], int]: - """Get all project storage allow entries, optionally filtered by project name.""" - if user.id is None or not user.is_admin: - raise errors.ForbiddenError(message="You do not have the required permissions for this operation.") - - async with self.session_maker() as session: - stmt = ( - select( - schemas.ProjectStorageAllowORM.project_id, - schemas.ProjectStorageAllowORM.max_size, - ProjectORM.name, - ns_schemas.NamespaceORM.slug.label("namespace_slug"), - ns_schemas.EntitySlugORM.slug.label("project_slug"), - schemas.ProjectStorageAllowORM.updated_at, - ) - .join(ProjectORM, ProjectORM.id == schemas.ProjectStorageAllowORM.project_id) - .join( - ns_schemas.EntitySlugORM, - and_( - ns_schemas.EntitySlugORM.project_id == schemas.ProjectStorageAllowORM.project_id, - ns_schemas.EntitySlugORM.data_connector_id.is_(None), - ), - ) - .join(ns_schemas.NamespaceORM, ns_schemas.NamespaceORM.id == ns_schemas.EntitySlugORM.namespace_id) - ) - - stmt_count = select(func.count()).select_from(schemas.ProjectStorageAllowORM) - if project_name: - stmt = stmt.where(ProjectORM.name.ilike(f"%{project_name}%")) - stmt_count = stmt_count.where(ProjectORM.name.ilike(f"%{project_name}%")) - stmt = ( - stmt.order_by(schemas.ProjectStorageAllowORM.project_id) - .limit(pagination.per_page) - .offset(pagination.offset) - ) - rows = await session.execute(stmt) - results = [models.ProjectStorageAllowDetail.create(**row._mapping) for row in rows] - total = await session.scalar(stmt_count) or 0 - return results, total - - @with_db_transaction - async def update_project_storage_allow( - self, - user: base_models.APIUser, - project_id: ULID, - patch: models.ProjectStorageAllowPatch, - etag: str, - *, - session: AsyncSession | None = None, - ) -> models.ProjectStorageAllowUpdate: - """Update some properties of a project storage allow entry.""" - if not session: - raise errors.ProgrammingError(message="A database session is required.") - - old = await self.get_project_storage_allow(user, project_id, session=session) - ps_orm = await session.scalars( - select(schemas.ProjectStorageAllowORM).where(schemas.ProjectStorageAllowORM.project_id == project_id) - ) - ps_orm = ps_orm.one_or_none() - if not old or not ps_orm or not isinstance(user, base_models.AuthenticatedAPIUser) or not user.is_admin: - raise errors.MissingResourceError( - message=( - f"Project storage allow entry for project '{project_id}' " - "does not exist or you do not have access to it." - ) - ) - - current_etag = old.etag - if current_etag != etag: - raise errors.ConflictError(message=f"Current ETag is {current_etag}, not {etag}.") - - if patch.max_size: - ps_orm.max_size = patch.max_size - - await session.flush() - await session.refresh(ps_orm) - - new = models.ProjectStorageAllowDetail( - project_id=project_id, - max_size=ps_orm.max_size, - name=old.name, - namespace_path=old.namespace_path, - updated_at=ps_orm.updated_at, - ) - return models.ProjectStorageAllowUpdate(old=old, new=new) - - @with_db_transaction - async def delete_project_storage_allow( - self, user: base_models.APIUser, project_id: ULID, *, session: AsyncSession | None = None - ) -> models.DeletedProjectStorage | None: - """Delete a project storage allow entry.""" - if not session: - raise errors.ProgrammingError(message="A database session is required.") - - if user.id is None or not user.is_admin: - raise errors.UnauthorizedError(message="You do not have the required permissions for this operation.") - - storage = await self._get_storage_to_project(user, project_id) - result = await session.scalars( - select(schemas.ProjectStorageAllowORM).where(schemas.ProjectStorageAllowORM.project_id == project_id) - ) - allow_orm = result.one_or_none() - if allow_orm: - await session.delete(allow_orm) - - if storage: - return models.DeletedProjectStorage(project_id=storage.project_id) - return None - @with_db_transaction async def insert_global_data_connector( self, diff --git a/components/renku_data_services/data_connectors/models.py b/components/renku_data_services/data_connectors/models.py index 0f2aa2907a..23870fc2b4 100644 --- a/components/renku_data_services/data_connectors/models.py +++ b/components/renku_data_services/data_connectors/models.py @@ -11,7 +11,6 @@ from ulid import ULID from renku_data_services.authz.models import Visibility -from renku_data_services.base_models.bytesize import ByteSize from renku_data_services.base_models.core import ( DataConnectorInProjectPath, DataConnectorPath, @@ -91,112 +90,6 @@ def path(self) -> DataConnectorPath | DataConnectorInProjectPath: return self.namespace / DataConnectorSlug(self.slug) -@dataclass(frozen=True, eq=True, kw_only=True) -class UnsavedProjectStorage: - """Project storage definition.""" - - namespace_path: ProjectPath - size: ByteSize - mount_path: PurePosixPath - - -@dataclass(frozen=True, eq=True, kw_only=True) -class ProjectStoragePatch: - """Model for changes requested on a project storage.""" - - size: ByteSize | None - mount_path: PurePosixPath | None - - -@dataclass(frozen=True, eq=True, kw_only=True) -class ProjectStorage: - """Stored project storage information.""" - - id: ULID - project_id: ULID - storage_class: str - size: ByteSize - mount_path: PurePosixPath - created_by: str - creation_date: datetime - updated_at: datetime - - @property - def etag(self) -> str: - """Entity tag value for this project storage object.""" - return compute_etag_from_fields( - self.updated_at, self.project_id, self.storage_class, self.size.to_bytes(), self.mount_path.as_posix() - ) - - -@dataclass(frozen=True, eq=True, kw_only=True) -class DeletedProjectStorage: - """A project storage that has been deleted.""" - - project_id: ULID - - -@dataclass(frozen=True, eq=True, kw_only=True) -class ProjectStorageAllow: - """Allowed project storage with max size.""" - - project_id: ULID - max_size: ByteSize - updated_at: datetime - - @property - def etag(self) -> str: - """Entity tag value for this project storage allow object.""" - return compute_etag_from_fields(self.updated_at, self.project_id, self.max_size.to_bytes()) - - -@dataclass(frozen=True, eq=True, kw_only=True) -class ProjectStorageAllowDetail: - """Allowed project storage with max size.""" - - project_id: ULID - max_size: ByteSize - name: str - namespace_path: ProjectPath - updated_at: datetime - - @classmethod - def create( - cls, - project_id: ULID, - max_size: ByteSize, - name: str, - namespace_slug: str, - project_slug: str, - updated_at: datetime, - ) -> ProjectStorageAllowDetail: - """Create an instance with the project path given as two strings.""" - np = ProjectPath.from_strings(namespace_slug, project_slug) - return ProjectStorageAllowDetail( - project_id=project_id, max_size=max_size, name=name, namespace_path=np, updated_at=updated_at - ) - - @property - def etag(self) -> str: - """Entity tag value for this project storage allow object.""" - return compute_etag_from_fields(self.updated_at, self.project_id, self.max_size.to_bytes()) - - -@dataclass(frozen=True, eq=True, kw_only=True) -class ProjectStorageAllowPatch: - """Model for changes requested on a project storage allow entry.""" - - max_size: ByteSize | None - - -@dataclass(frozen=True, eq=True, kw_only=True) -class ProjectStorageAllowUpdate: - """Return data when updating an allow entry.""" - - old: ProjectStorageAllowDetail - new: ProjectStorageAllowDetail - - @dataclass(frozen=True, eq=True, kw_only=True) class GlobalDataConnector(BaseDataConnector): """Global data connector model.""" diff --git a/components/renku_data_services/data_connectors/orm.py b/components/renku_data_services/data_connectors/orm.py index 94be7d0077..5a8cd31966 100644 --- a/components/renku_data_services/data_connectors/orm.py +++ b/components/renku_data_services/data_connectors/orm.py @@ -13,7 +13,6 @@ from ulid import ULID from renku_data_services.authz import models as authz_models -from renku_data_services.base_models.bytesize import ByteSize from renku_data_services.base_orm.registry import COMMON_ORM_REGISTRY from renku_data_services.crc.orm import ClusterORM from renku_data_services.data_connectors import models @@ -23,7 +22,7 @@ from renku_data_services.project.orm import ProjectORM from renku_data_services.secrets.orm import SecretORM from renku_data_services.users.orm import UserORM -from renku_data_services.utils.sqlalchemy import ByteSizeType, PurePosixPathType, ULIDType +from renku_data_services.utils.sqlalchemy import ULIDType if TYPE_CHECKING: from renku_data_services.namespace.orm import EntitySlugOldORM, EntitySlugORM @@ -333,89 +332,3 @@ def dump(self) -> models.DepositJob: updated_at=self.updated_at, ), ) - - -class ProjectStorageAllowORM(BaseORM): - """ORM model for project storage allow list with size limits.""" - - __tablename__ = "project_storage_allow" - - project_id: Mapped[ULID] = mapped_column( - "project_id", - ForeignKey(ProjectORM.id, ondelete="CASCADE"), - primary_key=True, - unique=True, - index=True, - ) - """ID of the project.""" - - max_size: Mapped[ByteSize] = mapped_column("max_size", ByteSizeType()) - """Maximum allowed size in bytes.""" - - updated_at: Mapped[datetime] = mapped_column( - "updated_at", - DateTime(timezone=True), - default=None, - server_default=func.now(), - onupdate=func.now(), - nullable=False, - ) - - def dump(self) -> models.ProjectStorageAllow: - """Convert the ORM row to a ProjectStorageAllow model.""" - return models.ProjectStorageAllow( - project_id=self.project_id, - max_size=self.max_size, - updated_at=self.updated_at, - ) - - -class ProjectStorageORM(BaseORM): - """ORM model for project storage configuration.""" - - __tablename__ = "project_storage" - - id: Mapped[ULID] = mapped_column( - "id", ULIDType, primary_key=True, server_default=text("generate_ulid()"), init=False - ) - project_id: Mapped[ULID] = mapped_column( - ForeignKey(ProjectStorageAllowORM.project_id, ondelete="CASCADE"), index=True, nullable=False, unique=True - ) - """ID of the project (must exist in project_storage_allow).""" - - storage_class: Mapped[str] = mapped_column("storage_class", String(20)) - """The storage class (e.g. azurefile).""" - - size_limit: Mapped[ByteSize] = mapped_column("size_limit", ByteSizeType()) - """The storage limit in bytes.""" - - mount_path: Mapped[PurePosixPath] = mapped_column("target_path", PurePosixPathType()) - """Folder to mount to.""" - - created_by_id: Mapped[str] = mapped_column(ForeignKey(UserORM.keycloak_id), index=True, nullable=False) - """User ID of the creator of the project storage.""" - - creation_date: Mapped[datetime] = mapped_column( - "creation_date", DateTime(timezone=True), default=func.now(), nullable=False - ) - updated_at: Mapped[datetime] = mapped_column( - "updated_at", - DateTime(timezone=True), - default=None, - server_default=func.now(), - onupdate=func.now(), - nullable=False, - ) - - def dump(self) -> models.ProjectStorage: - """Convert the ORM row to a ProjectStorage model.""" - return models.ProjectStorage( - id=self.id, - project_id=self.project_id, - storage_class=self.storage_class, - size=self.size_limit, - mount_path=self.mount_path, - created_by=self.created_by_id, - creation_date=self.creation_date, - updated_at=self.updated_at, - ) diff --git a/components/renku_data_services/storage/api.spec.yaml b/components/renku_data_services/storage/api.spec.yaml index 83170b888e..0c296970d7 100644 --- a/components/renku_data_services/storage/api.spec.yaml +++ b/components/renku_data_services/storage/api.spec.yaml @@ -89,6 +89,248 @@ paths: $ref: "#/components/responses/Error" tags: - storage + + /data_connectors/storage: + post: + summary: Create a new project storage + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/ProjectStoragePost" + responses: + "201": + description: The data connector was created + content: + application/json: + schema: + $ref: "#/components/schemas/ProjectStorage" + default: + $ref: "#/components/responses/Error" + tags: + - data_connectors + /data_connectors/storage/config: + get: + summary: Get the current configuration for project storage + responses: + "200": + description: The configuration data + content: + application/json: + schema: + $ref: "#/components/schemas/ProjectStorageConfig" + default: + $ref: "#/components/responses/Error" + tags: + - data_connectors + /data_connectors/storage/allow: + get: + summary: List all projects in the storage allow list + parameters: + - in: query + description: query parameters + name: params + style: form + explode: true + schema: + $ref: "#/components/schemas/ProjectStorageAllowListQuery" + responses: + "200": + description: List of storage allow entries + content: + application/json: + schema: + $ref: "#/components/schemas/ProjectStorageAllowList" + headers: + page: + description: The index of the current page (starting at 1). + required: true + schema: + type: integer + per-page: + description: The number of items per page. + required: true + schema: + type: integer + total: + description: The total number of items. + required: true + schema: + type: integer + total-pages: + description: The total number of pages. + required: true + schema: + type: integer + default: + $ref: "#/components/responses/Error" + tags: + - data_connectors + post: + summary: Add a project to the storage allow list + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/ProjectStorageAllowPost" + responses: + "201": + description: The project was added to the allow list + content: + application/json: + schema: + $ref: "#/components/schemas/ProjectStorageAllowPost" + default: + $ref: "#/components/responses/Error" + tags: + - data_connectors + /data_connectors/storage/allow/{project_id}: + parameters: + - in: path + name: project_id + required: true + schema: + $ref: "#/components/schemas/Ulid" + get: + summary: Get the storage allow entry for a project + responses: + "200": + description: The project storage allow entry + content: + application/json: + schema: + $ref: "#/components/schemas/ProjectStorageAllow" + "404": + description: The project is not in the storage allow list + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + default: + $ref: "#/components/responses/Error" + tags: + - data_connectors + patch: + summary: Change the maximum size for a project storage + parameters: + - $ref: "#/components/parameters/If-Match" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/ProjectStorageAllowPatch" + responses: + "200": + description: The patched project storage allow entry + content: + application/json: + schema: + $ref: "#/components/schemas/ProjectStorageAllow" + "404": + description: The project storage allow entry doesn't exist + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + default: + $ref: "#/components/responses/Error" + tags: + - data_connectors + delete: + summary: Remove a project from the storage allow list + responses: + "204": + description: The project was removed from the allow list + default: + $ref: "#/components/responses/Error" + tags: + - data_connectors + /data_connectors/storage/{storage_id}: + parameters: + - in: path + name: storage_id + required: true + schema: + $ref: "#/components/schemas/Ulid" + get: + summary: Get a project storage for a project id. + responses: + "200": + description: The project storage information + content: + application/json: + schema: + $ref: "#/components/schemas/ProjectStorage" + "404": + description: The project storage does not exist + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + default: + $ref: "#/components/responses/Error" + tags: + - data_connectors + patch: + summary: Change the size or mount path of a project storage + parameters: + - $ref: "#/components/parameters/If-Match" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/ProjectStoragePatch" + responses: + "200": + description: The patched project storage entry + content: + application/json: + schema: + $ref: "#/components/schemas/ProjectStorage" + "404": + description: The project storage does not exist + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + default: + $ref: "#/components/responses/Error" + tags: + - data_connectors + delete: + summary: Delete a specific project storage + responses: + "204": + description: The project storage was deleted or did not exist in the first place + default: + $ref: "#/components/responses/Error" + tags: + - data_connectors + /projects/{project_id}/storage: + parameters: + - in: path + name: project_id + required: true + schema: + $ref: "#/components/schemas/Ulid" + get: + summary: Get the project storage for a given project + responses: + "200": + description: The list of project storages (currently either one or empty). + content: + application/json: + schema: + $ref: "#/components/schemas/ProjectStorageList" + default: + $ref: "#/components/responses/Error" + tags: + - projects + components: schemas: RCloneConfig: @@ -239,7 +481,144 @@ components: description: Sentry trace ID for linking to corresponding log entries required: ["code", "message"] required: ["error"] - + ProjectStoragePost: + description: | + A special data connector for associating a shared read+write + volume. There can be exactly one such storage per project. + additionalProperties: false + properties: + namespace: + $ref: "#/components/schemas/ProjectSlug" + size: + type: integer + format: int32 + mount_path: + type: string + required: + - namespace + - size + - mount_path + ProjectStorage: + description: | + A stored shared storage configuration for a project. + additionalProperties: false + properties: + id: + $ref: "#/components/schemas/Ulid" + project_id: + $ref: "#/components/schemas/Ulid" + size: + type: integer + format: int32 + mount_path: + type: string + creation_date: + $ref: "#/components/schemas/CreationDate" + created_by: + $ref: "#/components/schemas/UserId" + updated_at: + $ref: "#/components/schemas/CreationDate" + etag: + $ref: "#/components/schemas/ETag" + required: + - id + - project_id + - size + - mount_path + - creation_date + - created_by + - updated_at + - etag + ProjectStoragePatch: + description: | + Data that can be updated on a project storage entry + type: object + additionalProperties: false + properties: + size: + type: integer + format: int64 + description: The maximum size in GB + mount_path: + type: string + description: The mount path for the storage + ProjectStorageAllowPatch: + description: | + Data that can be updated on a project storage allow entry + type: object + additionalProperties: false + properties: + max_size: + type: integer + format: int64 + description: The maximum size in GB + ProjectStorageAllowPost: + description: | + A project to be added to the storage allow list. + additionalProperties: false + properties: + project_id: + $ref: "#/components/schemas/Ulid" + max_size: + type: integer + format: int64 + description: Maximum size in GB + required: + - project_id + - max_size + ProjectStorageAllow: + description: | + A project in the storage allow list. + additionalProperties: false + properties: + project_id: + $ref: "#/components/schemas/Ulid" + name: + type: string + namespace: + type: string + max_size: + type: integer + format: int64 + description: Maximum size in GB + etag: + $ref: "#/components/schemas/ETag" + required: + - project_id + - max_size + - name + - namespace + - etag + ProjectStorageAllowList: + description: | + A list of project storage allow entries. + type: array + items: + $ref: "#/components/schemas/ProjectStorageAllow" + ProjectStorageAllowListQuery: + description: Query params for listing storage allow entries + allOf: + - $ref: "#/components/schemas/PaginationRequest" + - properties: + project_name: + description: Filter by project name (partial match). + type: string + default: "" + ProjectStorageConfig: + description: | + The current configuration for project storage as defined by admins. + type: object + additionalProperties: false + properties: + enabled: + type: boolean + max_size: + type: integer + format: int64 + description: The maximum size in GB + required: + - enabled + - max_size responses: Error: description: The schema for all 4xx and 5xx responses diff --git a/components/renku_data_services/storage/apispec.py b/components/renku_data_services/storage/apispec.py index f00a6e0603..0f56697fee 100644 --- a/components/renku_data_services/storage/apispec.py +++ b/components/renku_data_services/storage/apispec.py @@ -1,13 +1,13 @@ # generated by datamodel-codegen: # filename: api.spec.yaml -# timestamp: 2026-07-30T05:48:01+00:00 +# timestamp: 2026-07-30T15:37:23+00:00 from __future__ import annotations from enum import StrEnum from typing import Any -from pydantic import Field, RootModel +from pydantic import ConfigDict, Field, RootModel from renku_data_services.storage.apispec_base import BaseAPISpec @@ -100,6 +100,29 @@ class ErrorResponse(BaseAPISpec): error: Error +class ProjectStoragePatch(BaseAPISpec): + model_config = ConfigDict( + extra="forbid", + ) + size: int | None = Field(None, description="The maximum size in GB") + mount_path: str | None = Field(None, description="The mount path for the storage") + + +class ProjectStorageAllowPatch(BaseAPISpec): + model_config = ConfigDict( + extra="forbid", + ) + max_size: int | None = Field(None, description="The maximum size in GB") + + +class ProjectStorageConfig(BaseAPISpec): + model_config = ConfigDict( + extra="forbid", + ) + enabled: bool + max_size: int = Field(..., description="The maximum size in GB") + + class StorageSchemaTestConnectionPostRequest(BaseAPISpec): configuration: dict[str, int | str | bool | dict[str, Any] | None] source_path: str = Field( @@ -113,6 +136,14 @@ class StorageSchemaObscurePostRequest(BaseAPISpec): configuration: dict[str, int | str | bool | dict[str, Any] | None] +class PaginationRequest(RootModel[Any]): + root: Any + + +class ProjectStorageList(RootModel[Any]): + root: Any + + class RCloneEntry(BaseAPISpec): name: str = Field(..., description="Human readable name of the provider") description: str = Field(..., description="description of the provider") @@ -122,6 +153,62 @@ class RCloneEntry(BaseAPISpec): ) +class ProjectStoragePost(BaseAPISpec): + model_config = ConfigDict( + extra="forbid", + ) + namespace: Any + size: int + mount_path: str + + +class ProjectStorage(BaseAPISpec): + model_config = ConfigDict( + extra="forbid", + ) + id: Any + project_id: Any + size: int + mount_path: str + creation_date: Any + created_by: Any + updated_at: Any + etag: Any + + +class ProjectStorageAllowPost(BaseAPISpec): + model_config = ConfigDict( + extra="forbid", + ) + project_id: Any + max_size: int = Field(..., description="Maximum size in GB") + + +class ProjectStorageAllow(BaseAPISpec): + model_config = ConfigDict( + extra="forbid", + ) + project_id: Any + name: str + namespace: str + max_size: int = Field(..., description="Maximum size in GB") + etag: Any + + +class ProjectStorageAllowList(RootModel[list[ProjectStorageAllow]]): + root: list[ProjectStorageAllow] = Field( + ..., description="A list of project storage allow entries.\n" + ) + + +class ProjectStorageAllowListQuery(BaseAPISpec): + project_name: str = Field("", description="Filter by project name (partial match).") + + +class DataConnectorsStorageAllowGetParametersQuery(BaseAPISpec): + params: ProjectStorageAllowListQuery | None = None + + class RCloneSchema(RootModel[list[RCloneEntry]]): root: list[RCloneEntry] = Field( ..., description="List of RClone schemas for different storage types" diff --git a/components/renku_data_services/storage/blueprints.py b/components/renku_data_services/storage/blueprints.py index 56d3263b69..4cfc684326 100644 --- a/components/renku_data_services/storage/blueprints.py +++ b/components/renku_data_services/storage/blueprints.py @@ -1,18 +1,35 @@ """Cloud storage app.""" from dataclasses import dataclass +from typing import Any from sanic import HTTPResponse, Request, empty -from sanic.response import JSONResponse +from sanic.response import HTTPResponse, JSONResponse from sanic_ext import validate +from ulid import ULID import renku_data_services.base_models as base_models from renku_data_services import errors -from renku_data_services.base_api.auth import authenticate +from renku_data_services.base_api.auth import ( + authenticate, + only_admins, + only_authenticated, +) from renku_data_services.base_api.blueprint import BlueprintFactoryResponse, CustomBlueprint -from renku_data_services.base_models.validation import validated_json +from renku_data_services.base_api.etag import extract_if_none_match, if_match_required +from renku_data_services.base_api.misc import validate_query +from renku_data_services.base_api.pagination import PaginationRequest, paginate +from renku_data_services.base_models.validation import validate_and_dump, validated_json +from renku_data_services.data_connectors.project_storage_k8s import ProjectStorageK8s from renku_data_services.notebooks.data_sources import DataSourceRepository -from renku_data_services.storage import apispec +from renku_data_services.storage import apispec, models +from renku_data_services.storage.core import ( + validate_project_storage_allow_patch, + validate_project_storage_allow_post, + validate_project_storage_patch, + validate_unsaved_project_storage, +) +from renku_data_services.storage.db import ProjectStorageRepository from renku_data_services.storage.rclone import RCloneValidator @@ -77,3 +94,217 @@ async def _obscure( return validated_json(apispec.RCloneConfigValidate, config) return "/storage_schema/obscure", ["POST"], _obscure + + +@dataclass(kw_only=True) +class ProjectStorageBP(CustomBlueprint): + project_storage_k8s: ProjectStorageK8s + project_storage_repo: ProjectStorageRepository + authenticator: base_models.Authenticator + + def get_one_storage(self) -> BlueprintFactoryResponse: + """Get a specific project storage connector.""" + + @authenticate(self.authenticator) + @extract_if_none_match + async def _get_one(_: Request, user: base_models.APIUser, storage_id: ULID, etag: str | None) -> HTTPResponse: + project_storage = await self.project_storage_repo.get_project_storage(user=user, storage_id=storage_id) + if project_storage is None: + raise errors.MissingResourceError(message=f"No project storage found for storage: {storage_id}") + + if project_storage.etag == etag: + return HTTPResponse(status=304) + + headers = {"ETag": project_storage.etag} + return validated_json( + apispec.ProjectStorage, + self._dump_project_storage(project_storage), + headers=headers, + ) + + return "/data_connectors/storage/", ["GET"], _get_one + + def post_storage(self) -> BlueprintFactoryResponse: + """Create a new shared project storage.""" + + @authenticate(self.authenticator) + @only_authenticated + @validate(json=apispec.ProjectStoragePost) + async def _post_storage( + _: Request, user: base_models.APIUser, body: apispec.ProjectStoragePost + ) -> JSONResponse: + dc = validate_unsaved_project_storage(body) + result = await self.project_storage_repo.insert_project_storage(user, dc) + headers = {"ETag": result.etag} + return validated_json( + apispec.ProjectStorage, self._dump_project_storage(result), headers=headers, status=201 + ) + + return "/data_connectors/storage", ["POST"], _post_storage + + def patch_storage(self) -> BlueprintFactoryResponse: + """Partially update a project storage entry.""" + + @authenticate(self.authenticator) + @only_authenticated + @if_match_required + @validate(json=apispec.ProjectStoragePatch) + async def _patch_storage( + _: Request, + user: base_models.APIUser, + storage_id: ULID, + body: apispec.ProjectStoragePatch, + etag: str, + ) -> JSONResponse: + existing_storage = await self.project_storage_repo.get_project_storage(user=user, storage_id=storage_id) + if existing_storage is None: + raise errors.MissingResourceError(message=f"No project storage found for storage: {storage_id}") + + storage_patch = validate_project_storage_patch(existing_storage, body) + updated_storage = await self.project_storage_repo.update_project_storage( + user=user, storage_id=storage_id, patch=storage_patch, etag=etag + ) + headers = {"ETag": updated_storage.etag} + return validated_json( + apispec.ProjectStorage, + self._dump_project_storage(updated_storage), + headers=headers, + ) + + return "/data_connectors/storage/", ["PATCH"], _patch_storage + + def get_all_storage_allows(self) -> BlueprintFactoryResponse: + """List all projects in the storage allow list.""" + + @authenticate(self.authenticator) + @only_admins + @validate_query(query=apispec.ProjectStorageAllowListQuery) + @paginate + async def _get_all_storage_allows( + _: Request, + user: base_models.APIUser, + pagination: PaginationRequest, + query: apispec.ProjectStorageAllowListQuery, + ) -> tuple[list[dict[str, Any]], int]: + project_name = query.project_name if query.project_name else None + allows, total = await self.project_storage_repo.get_project_storage_allows( + user, pagination, project_name=project_name + ) + return [ + validate_and_dump( + apispec.ProjectStorageAllow, + self._dump_project_storage_allow_detail(a), + ) + for a in allows + ], total + + return "/data_connectors/storage/allow", ["GET"], _get_all_storage_allows + + def post_storage_allow(self) -> BlueprintFactoryResponse: + """Add a project to the storage allow list.""" + + @authenticate(self.authenticator) + @only_admins + @validate(json=apispec.ProjectStorageAllowPost) + async def _post_storage_allow( + _: Request, user: base_models.APIUser, body: apispec.ProjectStorageAllowPost + ) -> JSONResponse: + allow = validate_project_storage_allow_post(body) + inserted = await self.project_storage_repo.insert_project_storage_allow(user, allow) + return validated_json( + apispec.ProjectStorageAllowPost, + self._dump_project_storage_allow_post(inserted), + status=201, + ) + + return "/data_connectors/storage/allow", ["POST"], _post_storage_allow + + def patch_storage_allow(self) -> BlueprintFactoryResponse: + """Partially update a project storage allow entry.""" + + @authenticate(self.authenticator) + @only_admins + @if_match_required + @validate(json=apispec.ProjectStorageAllowPatch) + async def _patch( + _: Request, + user: base_models.APIUser, + project_id: ULID, + body: apispec.ProjectStorageAllowPatch, + etag: str, + ) -> JSONResponse: + existing_entry = await self.project_storage_repo.get_project_storage_allow(user=user, project_id=project_id) + if not existing_entry: + raise errors.MissingResourceError(message=f"No project storage allow entry for project {project_id}") + + pse_patch = validate_project_storage_allow_patch(existing_entry, body) + pse_update = await self.project_storage_repo.update_project_storage_allow( + user=user, project_id=project_id, patch=pse_patch, etag=etag + ) + + headers = {"ETag": pse_update.new.etag} + return validated_json( + apispec.ProjectStorageAllow, self._dump_project_storage_allow_detail(pse_update.new), headers=headers + ) + + return "/data_connectors/storage/allow/", ["PATCH"], _patch + + def get_storage_allow(self) -> BlueprintFactoryResponse: + """Get the storage allow entry for a project.""" + + @authenticate(self.authenticator) + @only_authenticated + async def _get_storage_allow(_: Request, user: base_models.APIUser, project_id: ULID) -> HTTPResponse: + allow = await self.project_storage_repo.get_project_storage_allow(user, project_id) + if allow is None: + raise errors.MissingResourceError(message=f"Project {project_id} is not in the storage allow list.") + + headers = {"ETag": allow.etag} + return validated_json( + apispec.ProjectStorageAllow, self._dump_project_storage_allow_detail(allow), headers=headers + ) + + return "/data_connectors/storage/allow/", ["GET"], _get_storage_allow + + def delete_storage_allow(self) -> BlueprintFactoryResponse: + """Remove a project from the storage allow list.""" + + @authenticate(self.authenticator) + @only_admins + async def _delete_storage_allow(_: Request, user: base_models.APIUser, project_id: ULID) -> HTTPResponse: + deleted = await self.project_storage_repo.delete_project_storage_allow(user, project_id) + if deleted: + await self.project_storage_k8s.delete_volume(deleted) + return HTTPResponse(status=204) + + return "/data_connectors/storage/allow/", ["DELETE"], _delete_storage_allow + + @staticmethod + def _dump_project_storage(ps: models.ProjectStorage) -> apispec.ProjectStorage: + return apispec.ProjectStorage( + id=str(ps.id), + project_id=str(ps.project_id), + size=int(ps.size.to_gibi()), + mount_path=ps.mount_path.as_posix(), + created_by=ps.created_by, + creation_date=ps.creation_date, + updated_at=ps.updated_at, + etag=ps.etag, + ) + + @staticmethod + def _dump_project_storage_allow_detail(ps: models.ProjectStorageAllowDetail) -> apispec.ProjectStorageAllow: + return apispec.ProjectStorageAllow( + project_id=str(ps.project_id), + max_size=int(ps.max_size.to_gibi()), + name=ps.name, + namespace=ps.namespace_path.serialize(), + etag=ps.etag, + ) + + @staticmethod + def _dump_project_storage_allow_post(ps: models.ProjectStorageAllow) -> apispec.ProjectStorageAllowPost: + return apispec.ProjectStorageAllowPost( + project_id=str(ps.project_id), + max_size=int(ps.max_size.to_gibi()), + ) diff --git a/components/renku_data_services/storage/config.py b/components/renku_data_services/storage/config.py new file mode 100644 index 0000000000..e7c7142e3f --- /dev/null +++ b/components/renku_data_services/storage/config.py @@ -0,0 +1,38 @@ +"""Configuration for project storage.""" + +from __future__ import annotations + +import os +from dataclasses import dataclass + +from renku_data_services.app_config import logging +from renku_data_services.base_models.bytesize import ByteSize +from renku_data_services.errors import errors + +logger = logging.getLogger(__name__) + + +@dataclass +class ProjectStorageConfig: + """The configuration for project storage.""" + + enabled: bool + storage_class: str + maximum_size: ByteSize + + @classmethod + def from_env(cls) -> ProjectStorageConfig: + """Create a configuration from environment variables.""" + + enabled = os.environ.get("PROJECT_STORAGE_ENABLED", "").lower() == "true" + storage_class = os.environ.get("PROJECT_STORAGE_STORAGE_CLASS") + maximum_size = os.environ.get("PROJECT_STORAGE_MAX_SIZE_GB") or "10" + maximum_size = ByteSize.from_gibi(int(maximum_size)) + + # TODO: set defaults for easier PR/CI deployments for now + enabled = True + storage_class = "azurefile" + if enabled and not storage_class: + raise errors.ConfigurationError(message="A storage_class is required for enabled project storage") + + return ProjectStorageConfig(enabled, storage_class=storage_class, maximum_size=maximum_size) diff --git a/components/renku_data_services/storage/core.py b/components/renku_data_services/storage/core.py new file mode 100644 index 0000000000..ecc391c6cb --- /dev/null +++ b/components/renku_data_services/storage/core.py @@ -0,0 +1,97 @@ +"""Business logic for project storage.""" + +from __future__ import annotations + +from datetime import datetime +from pathlib import PurePosixPath +from typing import TYPE_CHECKING + +from ulid import ULID + +from renku_data_services import errors +from renku_data_services.base_models.bytesize import ByteSize +from renku_data_services.base_models.core import ( + ProjectPath, +) +from renku_data_services.storage import apispec, models + +if TYPE_CHECKING: + pass + + +def _validate_mount_path(path: str | None) -> None: + # + invalid_prefixes = [ + "/", + "/bin", + "/sbin", + "/usr", + "/lib", + "/lib64", + "/boot", + "/etc", + "/proc", + "/sys", + "/dev", + "/run", + "/sys", + "/var", + "/tmp", # nosec B108 + "/home", + "/root", + ] + if not path or path == "": + raise errors.ValidationError(message="The mount path must not be empty") + + for prefix in invalid_prefixes: + if path == prefix or path.startswith(f"{prefix}/"): + raise errors.ValidationError(message=f"The mount path is invalid: '{path}'") + + +def validate_unsaved_project_storage(body: apispec.ProjectStoragePost) -> models.UnsavedProjectStorage: + """Validate the user input for a new project storage definition. + + The namespace must be a project namespace. The project must be + enabled for project storages and the user must be an owner. + """ + + _validate_mount_path(body.mount_path) + + namespace_path = ProjectPath.parse(body.namespace) + return models.UnsavedProjectStorage( + namespace_path=namespace_path, size=ByteSize.from_gibi(body.size), mount_path=PurePosixPath(body.mount_path) + ) + + +def validate_project_storage_patch( + existing: models.ProjectStorage, body: apispec.ProjectStoragePatch +) -> models.ProjectStoragePatch: + """Validate a patch of a project storage entry.""" + size = ByteSize.from_gibi(body.size) if body.size else None + if size and size < ByteSize.from_gibi(1): + raise errors.ValidationError(message="The size must be at least 1GB") + mount_path = PurePosixPath(body.mount_path) if body.mount_path else None + if mount_path: + _validate_mount_path(body.mount_path) + return models.ProjectStoragePatch(size=size, mount_path=mount_path) + + +def validate_project_storage_allow_post(body: apispec.ProjectStorageAllowPost) -> models.ProjectStorageAllow: + """Validate.""" + allow = models.ProjectStorageAllow( + project_id=ULID.from_str(body.project_id), max_size=ByteSize.from_gibi(body.max_size), updated_at=datetime.now() + ) + if allow.max_size < ByteSize.from_gibi(1): + raise errors.ValidationError(message=f"The maximum size must be at least 1GB, but {allow.max_size} was given.") + return allow + + +def validate_project_storage_allow_patch( + existing: models.ProjectStorageAllowDetail, body: apispec.ProjectStorageAllowPatch +) -> models.ProjectStorageAllowPatch: + """Validate a patch of a project storage allow entry.""" + + size = ByteSize.from_gibi(body.max_size) if body.max_size is not None else None + if size and size < ByteSize.from_gibi(1): + raise errors.ValidationError(message="The maximum size must be at least 1GB") + return models.ProjectStorageAllowPatch(max_size=size) diff --git a/components/renku_data_services/storage/db.py b/components/renku_data_services/storage/db.py new file mode 100644 index 0000000000..ee033808ef --- /dev/null +++ b/components/renku_data_services/storage/db.py @@ -0,0 +1,422 @@ +"""Database access for project storage classes.""" + +from collections.abc import Callable + +from sqlalchemy import and_, exists, func, select +from sqlalchemy.ext.asyncio import AsyncSession +from ulid import ULID + +from renku_data_services import base_models, errors +from renku_data_services.authz.authz import Authz, ResourceType +from renku_data_services.authz.models import Scope +from renku_data_services.base_api.pagination import PaginationRequest +from renku_data_services.namespace import orm as ns_schemas +from renku_data_services.namespace.db import GroupRepository +from renku_data_services.project.db import ProjectRepository +from renku_data_services.project.orm import ProjectORM +from renku_data_services.storage import models +from renku_data_services.storage import orm as schemas +from renku_data_services.storage.config import ProjectStorageConfig +from renku_data_services.utils.core import with_db_session, with_db_transaction + + +class ProjectStorageRepository: + """Repository for project storage.""" + + def __init__( + self, + session_maker: Callable[..., AsyncSession], + authz: Authz, + project_repo: ProjectRepository, + group_repo: GroupRepository, + project_storage_config: ProjectStorageConfig, + ) -> None: + self.session_maker = session_maker + self.authz = authz + self.project_repo = project_repo + self.group_repo = group_repo + self.project_storage_config = project_storage_config + + async def get_storage_to(self, user: base_models.APIUser, project_id: ULID) -> models.ProjectStorage | None: + """Get a project storage to a project if it exists and the feature is enabled.""" + + if not self.project_storage_config.enabled: + return None + else: + return await self._get_storage_to_project(user, project_id) + + async def _get_storage_to_project( + self, user: base_models.APIUser, project_id: ULID + ) -> models.ProjectStorage | None: + """Get a project storage to a project if it exists.""" + + if user.id is None: + raise errors.UnauthorizedError(message="You do not have the required permissions for this operation.") + + async with self.session_maker() as session: + result_orm = await session.scalars( + select(schemas.ProjectStorageORM).where(schemas.ProjectStorageORM.project_id == project_id) + ) + result_orm = result_orm.one_or_none() + if not result_orm: + return None + + result = result_orm.dump() + authorized = await self.authz.has_permission(user, ResourceType.project, result.project_id, Scope.READ) + if not authorized: + return None + + return result + + def get_project_storage_config(self) -> ProjectStorageConfig: + """Return the current config for project storage.""" + + return self.project_storage_config + + async def get_project_storage(self, user: base_models.APIUser, storage_id: ULID) -> models.ProjectStorage | None: + """Get a project storage by its id.""" + + if user.id is None: + raise errors.UnauthorizedError(message="You do not have the required permissions for this operation.") + + async with self.session_maker() as session: + result_orm = await session.scalars( + select(schemas.ProjectStorageORM).where(schemas.ProjectStorageORM.id == storage_id) + ) + result_orm = result_orm.one_or_none() + if not result_orm: + return None + + result = result_orm.dump() + authorized = await self.authz.has_permission(user, ResourceType.project, result.project_id, Scope.READ) + if not authorized: + return None + + return result + + @with_db_transaction + async def insert_project_storage( + self, user: base_models.APIUser, input: models.UnsavedProjectStorage, *, session: AsyncSession | None = None + ) -> models.ProjectStorage: + """Insert a new project storage.""" + + # When the feature is disabled, we disallow insertion, but still allow managing existing data + if not self.project_storage_config.enabled: + raise errors.MissingResourceError(message="The project storage api is not enabled.") + + if not session: + raise errors.ProgrammingError(message="A database session is required.") + if user.id is None: + raise errors.UnauthorizedError(message="You do not have the required permissions for this operation.") + + # there is only one such storage possible for a project + project = await self.project_repo.get_project_by_namespace_slug( + user, input.namespace_path.first.value, input.namespace_path.second, with_documentation=False + ) + + authorized = await self.authz.has_permission(user, ResourceType.project, project.id, Scope.DELETE) + if not authorized: + raise errors.MissingResourceError( + message=f"Project with id '{project.id}' does not exist or you do not have access to it." + ) + + allowed = await session.execute( + select(schemas.ProjectStorageAllowORM).where(schemas.ProjectStorageAllowORM.project_id == project.id) + ) + allowed = allowed.scalar() + if not allowed: + raise errors.ForbiddenError(message=f"Project storage is not enabled for project {project.id}.") + + existing_storage = await session.execute( + select(exists().where(schemas.ProjectStorageORM.project_id == project.id)) + ) + existing_storage = existing_storage.scalar() + if existing_storage: + raise errors.ValidationError(message=f"There is already a project storage for project {project.id}") + + if input.size > allowed.max_size: + raise errors.ValidationError( + message=( + f"The project storage size ({input.size}) for project {project.id} " + f"exceeds the maximum size of {allowed.max_size}" + ) + ) + + new_storage = schemas.ProjectStorageORM( + project_id=project.id, + storage_class=self.project_storage_config.storage_class, + size_limit=input.size, + mount_path=input.mount_path, + created_by_id=user.id, + ) + session.add(new_storage) + await session.flush() + return new_storage.dump() + + @with_db_transaction + async def update_project_storage( + self, + user: base_models.APIUser, + storage_id: ULID, + patch: models.ProjectStoragePatch, + etag: str, + *, + session: AsyncSession | None = None, + ) -> models.ProjectStorage: + """Update some properties of a project storage entry.""" + if not session: + raise errors.ProgrammingError(message="A database session is required.") + + result = await session.scalars( + select(schemas.ProjectStorageORM).where(schemas.ProjectStorageORM.id == storage_id) + ) + storage_orm = result.one_or_none() + if storage_orm is None: + raise errors.MissingResourceError(message=f"Project storage with id '{storage_id}' does not exist.") + + # Check authorization - user must be "owner", meaning allowed to delete the project + authorized = await self.authz.has_permission(user, ResourceType.project, storage_orm.project_id, Scope.DELETE) + if not authorized: + raise errors.MissingResourceError( + message=f"Project storage with id '{storage_id}' does not exist or you do not have access to it." + ) + + current_storage = storage_orm.dump() + current_etag = current_storage.etag + if current_etag != etag: + raise errors.ConflictError(message=f"Current ETag is {current_etag}, not {etag}.") + + # Check if size would exceed the allowed maximum + new_size = patch.size if patch.size else current_storage.size + allowed = await session.execute( + select(schemas.ProjectStorageAllowORM).where( + schemas.ProjectStorageAllowORM.project_id == storage_orm.project_id + ) + ) + allowed = allowed.scalar() + if allowed and new_size > allowed.max_size: + raise errors.ValidationError( + message=( + f"The project storage size ({new_size}) for project {storage_orm.project_id} " + f"exceeds the maximum size of {allowed.max_size}" + ) + ) + + if patch.size is not None: + storage_orm.size_limit = patch.size + if patch.mount_path is not None: + storage_orm.mount_path = patch.mount_path + + await session.flush() + await session.refresh(storage_orm) + return storage_orm.dump() + + @with_db_transaction + async def insert_project_storage_allow( + self, user: base_models.APIUser, input: models.ProjectStorageAllow, *, session: AsyncSession | None = None + ) -> models.ProjectStorageAllow: + """Insert a new project storage allow entry.""" + if not session: + raise errors.ProgrammingError(message="A database session is required.") + if user.id is None or not user.is_admin: + raise errors.UnauthorizedError(message="You do not have the required permissions for this operation.") + + existing = await session.execute( + select(exists().where(schemas.ProjectStorageAllowORM.project_id == input.project_id)) + ) + if existing.scalar(): + raise errors.ValidationError(message=f"Project {input.project_id} is already in the allow list.") + + if input.max_size > self.project_storage_config.maximum_size: + raise errors.ValidationError( + message=( + f"The maximum size {input.max_size} exceeds the configured " + f"one of {self.project_storage_config.maximum_size}." + ) + ) + + new_allow = schemas.ProjectStorageAllowORM( + project_id=input.project_id, + max_size=input.max_size, + ) + session.add(new_allow) + await session.flush() + return new_allow.dump() + + @with_db_transaction + async def delete_project_storage( + self, user: base_models.APIUser, storage_id: ULID, *, session: AsyncSession | None = None + ) -> models.DeletedProjectStorage | None: + """Delete a specific project storage.""" + if not session: + raise errors.ProgrammingError(message="A database session is required.") + + result = await session.scalars( + select(schemas.ProjectStorageORM).where(schemas.ProjectStorageORM.id == storage_id) + ) + storage_orm = result.one_or_none() + if storage_orm is None: + return None + + authorized = await self.authz.has_permission(user, ResourceType.project, storage_orm.project_id, Scope.DELETE) + if not authorized: + raise errors.MissingResourceError( + message=f"Project storage with id '{storage_id}' does not exist or you do not have access to it." + ) + + await session.delete(storage_orm) + ps = storage_orm.dump() + return models.DeletedProjectStorage(project_id=ps.project_id) + + @with_db_session + async def get_project_storage_allow( + self, + user: base_models.APIUser, + project_id: ULID, + *, + session: AsyncSession | None = None, + ) -> models.ProjectStorageAllowDetail | None: + """Get the storage allow entry for a project if it exists.""" + if not session: + raise errors.ProgrammingError(message="A database session is required.") + + authorized = await self.authz.has_permission(user, ResourceType.project, project_id, Scope.READ) + if not authorized: + raise errors.MissingResourceError( + message=f"Project with id '{project_id}' does not exist or you do not have access to it." + ) + stmt = ( + select( + schemas.ProjectStorageAllowORM.project_id, + schemas.ProjectStorageAllowORM.max_size, + ProjectORM.name, + ns_schemas.NamespaceORM.slug.label("namespace_slug"), + ns_schemas.EntitySlugORM.slug.label("project_slug"), + schemas.ProjectStorageAllowORM.updated_at, + ) + .join(ProjectORM, ProjectORM.id == schemas.ProjectStorageAllowORM.project_id) + .join( + ns_schemas.EntitySlugORM, + and_( + ns_schemas.EntitySlugORM.project_id == schemas.ProjectStorageAllowORM.project_id, + ns_schemas.EntitySlugORM.data_connector_id.is_(None), + ), + ) + .join(ns_schemas.NamespaceORM, ns_schemas.NamespaceORM.id == ns_schemas.EntitySlugORM.namespace_id) + .where(schemas.ProjectStorageAllowORM.project_id == project_id) + ) + result = (await session.execute(stmt)).one_or_none() + if result: + return models.ProjectStorageAllowDetail.create(**result._mapping) + return None + + async def get_project_storage_allows( + self, user: base_models.APIUser, pagination: PaginationRequest, project_name: str | None = None + ) -> tuple[list[models.ProjectStorageAllowDetail], int]: + """Get all project storage allow entries, optionally filtered by project name.""" + if user.id is None or not user.is_admin: + raise errors.ForbiddenError(message="You do not have the required permissions for this operation.") + + async with self.session_maker() as session: + stmt = ( + select( + schemas.ProjectStorageAllowORM.project_id, + schemas.ProjectStorageAllowORM.max_size, + ProjectORM.name, + ns_schemas.NamespaceORM.slug.label("namespace_slug"), + ns_schemas.EntitySlugORM.slug.label("project_slug"), + schemas.ProjectStorageAllowORM.updated_at, + ) + .join(ProjectORM, ProjectORM.id == schemas.ProjectStorageAllowORM.project_id) + .join( + ns_schemas.EntitySlugORM, + and_( + ns_schemas.EntitySlugORM.project_id == schemas.ProjectStorageAllowORM.project_id, + ns_schemas.EntitySlugORM.data_connector_id.is_(None), + ), + ) + .join(ns_schemas.NamespaceORM, ns_schemas.NamespaceORM.id == ns_schemas.EntitySlugORM.namespace_id) + ) + + stmt_count = select(func.count()).select_from(schemas.ProjectStorageAllowORM) + if project_name: + stmt = stmt.where(ProjectORM.name.ilike(f"%{project_name}%")) + stmt_count = stmt_count.where(ProjectORM.name.ilike(f"%{project_name}%")) + stmt = ( + stmt.order_by(schemas.ProjectStorageAllowORM.project_id) + .limit(pagination.per_page) + .offset(pagination.offset) + ) + rows = await session.execute(stmt) + results = [models.ProjectStorageAllowDetail.create(**row._mapping) for row in rows] + total = await session.scalar(stmt_count) or 0 + return results, total + + @with_db_transaction + async def update_project_storage_allow( + self, + user: base_models.APIUser, + project_id: ULID, + patch: models.ProjectStorageAllowPatch, + etag: str, + *, + session: AsyncSession | None = None, + ) -> models.ProjectStorageAllowUpdate: + """Update some properties of a project storage allow entry.""" + if not session: + raise errors.ProgrammingError(message="A database session is required.") + + old = await self.get_project_storage_allow(user, project_id, session=session) + ps_orm = await session.scalars( + select(schemas.ProjectStorageAllowORM).where(schemas.ProjectStorageAllowORM.project_id == project_id) + ) + ps_orm = ps_orm.one_or_none() + if not old or not ps_orm or not isinstance(user, base_models.AuthenticatedAPIUser) or not user.is_admin: + raise errors.MissingResourceError( + message=( + f"Project storage allow entry for project '{project_id}' " + "does not exist or you do not have access to it." + ) + ) + + current_etag = old.etag + if current_etag != etag: + raise errors.ConflictError(message=f"Current ETag is {current_etag}, not {etag}.") + + if patch.max_size: + ps_orm.max_size = patch.max_size + + await session.flush() + await session.refresh(ps_orm) + + new = models.ProjectStorageAllowDetail( + project_id=project_id, + max_size=ps_orm.max_size, + name=old.name, + namespace_path=old.namespace_path, + updated_at=ps_orm.updated_at, + ) + return models.ProjectStorageAllowUpdate(old=old, new=new) + + @with_db_transaction + async def delete_project_storage_allow( + self, user: base_models.APIUser, project_id: ULID, *, session: AsyncSession | None = None + ) -> models.DeletedProjectStorage | None: + """Delete a project storage allow entry.""" + if not session: + raise errors.ProgrammingError(message="A database session is required.") + + if user.id is None or not user.is_admin: + raise errors.UnauthorizedError(message="You do not have the required permissions for this operation.") + + storage = await self._get_storage_to_project(user, project_id) + result = await session.scalars( + select(schemas.ProjectStorageAllowORM).where(schemas.ProjectStorageAllowORM.project_id == project_id) + ) + allow_orm = result.one_or_none() + if allow_orm: + await session.delete(allow_orm) + + if storage: + return models.DeletedProjectStorage(project_id=storage.project_id) + return None diff --git a/components/renku_data_services/storage/models.py b/components/renku_data_services/storage/models.py index 63fcfc83f8..01d384dec2 100644 --- a/components/renku_data_services/storage/models.py +++ b/components/renku_data_services/storage/models.py @@ -1,6 +1,11 @@ -"""Models for cloud storage.""" +"""Models for storage.""" + +from __future__ import annotations from collections.abc import Generator, MutableMapping +from dataclasses import dataclass +from datetime import datetime +from pathlib import PurePosixPath from typing import Any from urllib.parse import ParseResult, urlparse @@ -8,7 +13,12 @@ from ulid import ULID from renku_data_services import errors +from renku_data_services.base_models.bytesize import ByteSize +from renku_data_services.base_models.core import ( + ProjectPath, +) from renku_data_services.storage.rclone import RCloneValidator +from renku_data_services.utils.etag import compute_etag_from_fields class RCloneConfig(BaseModel, MutableMapping): @@ -19,7 +29,7 @@ class RCloneConfig(BaseModel, MutableMapping): _validator: RCloneValidator = PrivateAttr(default=RCloneValidator()) @model_validator(mode="after") - def check_rclone_schema(self) -> "RCloneConfig": + def check_rclone_schema(self) -> RCloneConfig: """Validate that the reclone config is valid.""" self._validator.validate(self.config) return self @@ -71,7 +81,7 @@ class UnsavedCloudStorage(BaseModel): """Path inside the target repository to mount/clone data to.""" @classmethod - def from_dict(cls, data: dict) -> "UnsavedCloudStorage": + def from_dict(cls, data: dict) -> UnsavedCloudStorage: """Create the model from a plain dictionary.""" if "project_id" not in data: @@ -101,7 +111,7 @@ def from_dict(cls, data: dict) -> "UnsavedCloudStorage": @classmethod def from_url( cls, storage_url: str, name: str, readonly: bool, project_id: str, target_path: str - ) -> "UnsavedCloudStorage": + ) -> UnsavedCloudStorage: """Get Cloud Storage/rclone config from a storage URL. Example: @@ -132,7 +142,7 @@ def from_url( @classmethod def from_s3_url( cls, storage_url: ParseResult, project_id: str, name: str, readonly: bool, target_path: str - ) -> "UnsavedCloudStorage": + ) -> UnsavedCloudStorage: """Get Cloud storage from an S3 URL. Example: @@ -176,7 +186,7 @@ def from_s3_url( @classmethod def from_azure_url( cls, storage_url: ParseResult, project_id: str, name: str, readonly: bool, target_path: str - ) -> "UnsavedCloudStorage": + ) -> UnsavedCloudStorage: """Get Cloud storage from an Azure URL. Example: @@ -212,7 +222,7 @@ def from_azure_url( @classmethod def _from_ambiguous_url( cls, storage_url: ParseResult, project_id: str, name: str, readonly: bool, target_path: str - ) -> "UnsavedCloudStorage": + ) -> UnsavedCloudStorage: """Get cloud storage from an ambiguous storage url.""" if storage_url.hostname is None: raise errors.ValidationError(message="Storage URL must contain a host") @@ -228,3 +238,109 @@ class CloudStorage(UnsavedCloudStorage): """Cloudstorage saved in the database.""" storage_id: ULID = Field() + + +@dataclass(frozen=True, eq=True, kw_only=True) +class UnsavedProjectStorage: + """Project storage definition.""" + + namespace_path: ProjectPath + size: ByteSize + mount_path: PurePosixPath + + +@dataclass(frozen=True, eq=True, kw_only=True) +class ProjectStoragePatch: + """Model for changes requested on a project storage.""" + + size: ByteSize | None + mount_path: PurePosixPath | None + + +@dataclass(frozen=True, eq=True, kw_only=True) +class ProjectStorage: + """Stored project storage information.""" + + id: ULID + project_id: ULID + storage_class: str + size: ByteSize + mount_path: PurePosixPath + created_by: str + creation_date: datetime + updated_at: datetime + + @property + def etag(self) -> str: + """Entity tag value for this project storage object.""" + return compute_etag_from_fields( + self.updated_at, self.project_id, self.storage_class, self.size.to_bytes(), self.mount_path.as_posix() + ) + + +@dataclass(frozen=True, eq=True, kw_only=True) +class DeletedProjectStorage: + """A project storage that has been deleted.""" + + project_id: ULID + + +@dataclass(frozen=True, eq=True, kw_only=True) +class ProjectStorageAllow: + """Allowed project storage with max size.""" + + project_id: ULID + max_size: ByteSize + updated_at: datetime + + @property + def etag(self) -> str: + """Entity tag value for this project storage allow object.""" + return compute_etag_from_fields(self.updated_at, self.project_id, self.max_size.to_bytes()) + + +@dataclass(frozen=True, eq=True, kw_only=True) +class ProjectStorageAllowDetail: + """Allowed project storage with max size.""" + + project_id: ULID + max_size: ByteSize + name: str + namespace_path: ProjectPath + updated_at: datetime + + @classmethod + def create( + cls, + project_id: ULID, + max_size: ByteSize, + name: str, + namespace_slug: str, + project_slug: str, + updated_at: datetime, + ) -> ProjectStorageAllowDetail: + """Create an instance with the project path given as two strings.""" + np = ProjectPath.from_strings(namespace_slug, project_slug) + return ProjectStorageAllowDetail( + project_id=project_id, max_size=max_size, name=name, namespace_path=np, updated_at=updated_at + ) + + @property + def etag(self) -> str: + """Entity tag value for this project storage allow object.""" + return compute_etag_from_fields(self.updated_at, self.project_id, self.max_size.to_bytes()) + + +@dataclass(frozen=True, eq=True, kw_only=True) +class ProjectStorageAllowPatch: + """Model for changes requested on a project storage allow entry.""" + + max_size: ByteSize | None + + +@dataclass(frozen=True, eq=True, kw_only=True) +class ProjectStorageAllowUpdate: + """Return data when updating an allow entry.""" + + old: ProjectStorageAllowDetail + new: ProjectStorageAllowDetail diff --git a/components/renku_data_services/storage/orm.py b/components/renku_data_services/storage/orm.py new file mode 100644 index 0000000000..cdfbd01812 --- /dev/null +++ b/components/renku_data_services/storage/orm.py @@ -0,0 +1,114 @@ +"""SQLAlchemy schemas for the data project storage database.""" + +from __future__ import annotations + +from datetime import datetime +from pathlib import PurePosixPath +from typing import TYPE_CHECKING + +from sqlalchemy import DateTime, ForeignKey, MetaData, String, func, text +from sqlalchemy.orm import DeclarativeBase, Mapped, MappedAsDataclass, mapped_column +from ulid import ULID + +from renku_data_services.base_models.bytesize import ByteSize +from renku_data_services.base_orm.registry import COMMON_ORM_REGISTRY +from renku_data_services.project.orm import ProjectORM +from renku_data_services.storage import models +from renku_data_services.users.orm import UserORM +from renku_data_services.utils.sqlalchemy import ByteSizeType, PurePosixPathType, ULIDType + +if TYPE_CHECKING: + pass + + +class BaseORM(MappedAsDataclass, DeclarativeBase): + """Base class for all ORM classes.""" + + metadata = MetaData(schema="storage") + registry = COMMON_ORM_REGISTRY + + +class ProjectStorageAllowORM(BaseORM): + """ORM model for project storage allow list with size limits.""" + + __tablename__ = "project_storage_allow" + + project_id: Mapped[ULID] = mapped_column( + "project_id", + ForeignKey(ProjectORM.id, ondelete="CASCADE"), + primary_key=True, + unique=True, + index=True, + ) + """ID of the project.""" + + max_size: Mapped[ByteSize] = mapped_column("max_size", ByteSizeType()) + """Maximum allowed size in bytes.""" + + updated_at: Mapped[datetime] = mapped_column( + "updated_at", + DateTime(timezone=True), + default=None, + server_default=func.now(), + onupdate=func.now(), + nullable=False, + ) + + def dump(self) -> models.ProjectStorageAllow: + """Convert the ORM row to a ProjectStorageAllow model.""" + return models.ProjectStorageAllow( + project_id=self.project_id, + max_size=self.max_size, + updated_at=self.updated_at, + ) + + +class ProjectStorageORM(BaseORM): + """ORM model for project storage configuration.""" + + __tablename__ = "project_storage" + + id: Mapped[ULID] = mapped_column( + "id", ULIDType, primary_key=True, server_default=text("generate_ulid()"), init=False + ) + project_id: Mapped[ULID] = mapped_column( + ForeignKey(ProjectStorageAllowORM.project_id, ondelete="CASCADE"), index=True, nullable=False, unique=True + ) + """ID of the project (must exist in project_storage_allow).""" + + storage_class: Mapped[str] = mapped_column("storage_class", String(20)) + """The storage class (e.g. azurefile).""" + + size_limit: Mapped[ByteSize] = mapped_column("size_limit", ByteSizeType()) + """The storage limit in bytes.""" + + mount_path: Mapped[PurePosixPath] = mapped_column("target_path", PurePosixPathType()) + """Folder to mount to.""" + + created_by_id: Mapped[str] = mapped_column(ForeignKey(UserORM.keycloak_id), index=True, nullable=False) + """User ID of the creator of the project storage.""" + + creation_date: Mapped[datetime] = mapped_column( + "creation_date", DateTime(timezone=True), default=func.now(), nullable=False + ) + updated_at: Mapped[datetime] = mapped_column( + "updated_at", + DateTime(timezone=True), + default=None, + server_default=func.now(), + onupdate=func.now(), + nullable=False, + ) + + def dump(self) -> models.ProjectStorage: + """Convert the ORM row to a ProjectStorage model.""" + return models.ProjectStorage( + id=self.id, + project_id=self.project_id, + storage_class=self.storage_class, + size=self.size_limit, + mount_path=self.mount_path, + created_by=self.created_by_id, + creation_date=self.creation_date, + updated_at=self.updated_at, + ) From 6ca5bf0c04320fe14460fcc66910e64c6d5f122a Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Fri, 31 Jul 2026 09:53:05 +0200 Subject: [PATCH 2/7] Moved more code --- bases/renku_data_services/data_api/app.py | 12 +- bases/renku_data_services/data_api/config.py | 3 +- .../data_api/dependencies.py | 13 +- .../data_connectors/blueprints.py | 49 - .../renku_data_services/data_connectors/db.py | 3 - .../notebooks/blueprints.py | 3 + .../notebooks/core_sessions.py | 10 +- .../renku_data_services/project/blueprints.py | 2 +- .../renku_data_services/storage/blueprints.py | 52 +- .../project_storage_k8s.py | 2 +- .../data_api/test_data_connectors.py | 891 ----------------- .../data_api/test_storage.py | 896 ++++++++++++++++++ .../search/test_reprovision.py | 2 +- test/utils.py | 12 +- 14 files changed, 991 insertions(+), 959 deletions(-) rename components/renku_data_services/{data_connectors => storage}/project_storage_k8s.py (95%) diff --git a/bases/renku_data_services/data_api/app.py b/bases/renku_data_services/data_api/app.py index c91a67f221..53ce2f1b11 100644 --- a/bases/renku_data_services/data_api/app.py +++ b/bases/renku_data_services/data_api/app.py @@ -38,7 +38,7 @@ from renku_data_services.search.reprovision import SearchReprovision from renku_data_services.search.solr_user_query import UsernameResolve from renku_data_services.session.blueprints import BuildsBP, EnvironmentsBP, SessionLaunchersBP -from renku_data_services.storage.blueprints import StorageSchemaBP +from renku_data_services.storage.blueprints import ProjectStorageBP, StorageSchemaBP from renku_data_services.users.blueprints import KCUsersBP, UserPreferencesBP, UserSecretsBP @@ -222,6 +222,7 @@ def register_all_handlers(app: Sanic, dm: DependencyManager) -> Sanic: oauth_client_factory=dm.oauth_http_client_factory, project_repo=dm.project_repo, project_session_secret_repo=dm.project_session_secret_repo, + project_storage_repo=dm.project_storage_repo, rp_repo=dm.rp_repo, session_repo=dm.session_repo, user_repo=dm.kc_user_repo, @@ -280,7 +281,6 @@ def register_all_handlers(app: Sanic, dm: DependencyManager) -> Sanic: data_service_base_url=dm.config.nb_config.data_service_url, k8s_client=dm.k8s_client, deposit_config=dm.config.deposit_config, - project_storage_k8s=dm.project_storage_k8s, ) notifications = NotificationsBP( name="notifications", @@ -311,6 +311,13 @@ def register_all_handlers(app: Sanic, dm: DependencyManager) -> Sanic: internal_token_mint=dm.internal_token_mint, internal_scope_verifier=dm.internal_scope_verifier, ) + project_storage = ProjectStorageBP( + name="project_storage", + url_prefix=url_prefix, + project_storage_k8s=dm.project_storage_k8s, + project_storage_repo=dm.project_storage_repo, + authenticator=dm.authenticator, + ) app.blueprint( [ resource_pools.blueprint(), @@ -342,6 +349,7 @@ def register_all_handlers(app: Sanic, dm: DependencyManager) -> Sanic: capacity_reservation.blueprint(), resource_usage.blueprint(), internal_authentication.blueprint(), + project_storage.blueprint(), ] ) if builds is not None: diff --git a/bases/renku_data_services/data_api/config.py b/bases/renku_data_services/data_api/config.py index 1994faeaf3..6b2868ee4c 100644 --- a/bases/renku_data_services/data_api/config.py +++ b/bases/renku_data_services/data_api/config.py @@ -14,12 +14,13 @@ ) from renku_data_services.app_config.logging import Config as LoggingConfig from renku_data_services.authz.config import AuthzConfig -from renku_data_services.data_connectors.config import DepositConfig, ProjectStorageConfig +from renku_data_services.data_connectors.config import DepositConfig from renku_data_services.db_config.config import DBConfig from renku_data_services.notebooks.config import NotebooksConfig from renku_data_services.secrets.config import PublicSecretsConfig from renku_data_services.session.config import BuildsConfig from renku_data_services.solr.solr_client import SolrClientConfig +from renku_data_services.storage.config import ProjectStorageConfig from renku_data_services.users.config import UserPreferencesConfig diff --git a/bases/renku_data_services/data_api/dependencies.py b/bases/renku_data_services/data_api/dependencies.py index aeda0d9995..f286ee3e78 100644 --- a/bases/renku_data_services/data_api/dependencies.py +++ b/bases/renku_data_services/data_api/dependencies.py @@ -40,7 +40,6 @@ ) from renku_data_services.data_connectors.deposits.envidat import EnvidatClient from renku_data_services.data_connectors.deposits.zenodo import ZenodoAPIClient -from renku_data_services.data_connectors.project_storage_k8s import ProjectStorageK8s from renku_data_services.git.gitlab import DummyGitlabAPI, EmptyGitlabAPI, GitlabAPI from renku_data_services.k8s.client_interfaces import K8sClient from renku_data_services.k8s.clients import ( @@ -79,6 +78,8 @@ from renku_data_services.session.constants import BUILD_RUN_GVK, TASK_RUN_GVK from renku_data_services.session.db import SessionRepository from renku_data_services.session.k8s_client import ShipwrightClient +from renku_data_services.storage.db import ProjectStorageRepository +from renku_data_services.storage.project_storage_k8s import ProjectStorageK8s from renku_data_services.users.db import UserPreferencesRepository from renku_data_services.users.db import UserRepo as KcUserRepo from renku_data_services.users.dummy_kc_api import DummyKeycloakAPI @@ -177,6 +178,7 @@ class DependencyManager: internal_token_mint: RenkuSelfTokenMint internal_scope_verifier: ScopeVerifier project_storage_k8s: ProjectStorageK8s + project_storage_repo: ProjectStorageRepository spec: dict[str, Any] = field(init=False, repr=False, default_factory=dict) app_name: str = "renku_data_services" @@ -367,6 +369,13 @@ def from_env(cls) -> DependencyManager: member_repo=member_repo, ) project_storage_k8s = ProjectStorageK8s(config.nb_config.k8s_v2_client) + project_storage_repo = ProjectStorageRepository( + session_maker=config.db.async_session_maker, + authz=authz, + project_repo=project_repo, + group_repo=group_repo, + project_storage_config=config.project_storage_config, + ) reprovisioning_repo = ReprovisioningRepository(session_maker=config.db.async_session_maker) git_repositories_repo = GitRepositoriesRepository( @@ -423,7 +432,6 @@ def from_env(cls) -> DependencyManager: project_repo=project_repo, group_repo=group_repo, search_updates_repo=search_updates_repo, - project_storage_config=config.project_storage_config, ) data_connector_secret_repo = DataConnectorSecretRepository( session_maker=config.db.async_session_maker, @@ -518,4 +526,5 @@ def from_env(cls) -> DependencyManager: internal_token_mint=internal_token_mint, internal_scope_verifier=internal_scope_verifier, project_storage_k8s=project_storage_k8s, + project_storage_repo=project_storage_repo, ) diff --git a/components/renku_data_services/data_connectors/blueprints.py b/components/renku_data_services/data_connectors/blueprints.py index 3f418866b2..a5107f818e 100644 --- a/components/renku_data_services/data_connectors/blueprints.py +++ b/components/renku_data_services/data_connectors/blueprints.py @@ -11,7 +11,6 @@ from renku_data_services import base_models, errors from renku_data_services.base_api.auth import ( authenticate, - only_admins, only_authenticated, ) from renku_data_services.base_api.blueprint import BlueprintFactoryResponse, CustomBlueprint @@ -54,7 +53,6 @@ ) from renku_data_services.data_connectors.deposits.envidat import EnvidatClient from renku_data_services.data_connectors.deposits.zenodo import ZenodoAPIClient -from renku_data_services.data_connectors.project_storage_k8s import ProjectStorageK8s from renku_data_services.k8s.client_interfaces import K8sClient, SecretClient from renku_data_services.k8s.clients import DepositUploadJobClient from renku_data_services.notebooks.data_sources import DataSourceRepository @@ -79,7 +77,6 @@ class DataConnectorsBP(CustomBlueprint): data_service_base_url: str k8s_client: K8sClient deposit_config: DepositConfig - project_storage_k8s: ProjectStorageK8s def get_all(self) -> BlueprintFactoryResponse: """List data connectors.""" @@ -399,52 +396,6 @@ async def _delete_project_link( _delete_project_link, ) - def get_storage_to_project(self) -> BlueprintFactoryResponse: - """List all project storage to a given project.""" - - @authenticate(self.authenticator) - async def _get_all_storage_to_project( - _: Request, - user: base_models.APIUser, - project_id: ULID, - ) -> JSONResponse: - project_storage = await self.data_connector_repo.get_storage_to(user=user, project_id=project_id) - result = [self._dump_project_storage(project_storage)] if project_storage else [] - return validated_json(apispec.ProjectStorageList, result) - - return "/projects//storage", ["GET"], _get_all_storage_to_project - - def get_storage_config(self) -> BlueprintFactoryResponse: - """Get the current config used for project storage.""" - - @authenticate(self.authenticator) - @only_admins - async def _get_project_config(_: Request, user: base_models.APIUser) -> JSONResponse: - storage_config = self.data_connector_repo.get_project_storage_config() - result = apispec.ProjectStorageConfig( - enabled=storage_config.enabled, max_size=int(storage_config.maximum_size.to_gibi()) - ) - return validated_json(apispec.ProjectStorageConfig, result) - - return "/data_connectors/storage/config", ["GET"], _get_project_config - - def delete_storage(self) -> BlueprintFactoryResponse: - """Delete a specific project storage.""" - - @authenticate(self.authenticator) - @only_authenticated - async def _delete_storage( - _: Request, - user: base_models.APIUser, - storage_id: ULID, - ) -> HTTPResponse: - deleted = await self.data_connector_repo.delete_project_storage(user=user, storage_id=storage_id) - if deleted: - await self.project_storage_k8s.delete_volume(deleted) - return HTTPResponse(status=204) - - return "/data_connectors/storage/", ["DELETE"], _delete_storage - def get_all_data_connectors_links_to_project(self) -> BlueprintFactoryResponse: """List all links from data connectors to a given project.""" diff --git a/components/renku_data_services/data_connectors/db.py b/components/renku_data_services/data_connectors/db.py index 6b4e993632..2e2524453e 100644 --- a/components/renku_data_services/data_connectors/db.py +++ b/components/renku_data_services/data_connectors/db.py @@ -29,7 +29,6 @@ ) from renku_data_services.data_connectors import apispec, models from renku_data_services.data_connectors import orm as schemas -from renku_data_services.data_connectors.config import ProjectStorageConfig from renku_data_services.data_connectors.core import validate_unsaved_global_data_connector from renku_data_services.data_connectors.doi.models import DOI from renku_data_services.k8s.constants import DEFAULT_K8S_CLUSTER @@ -58,14 +57,12 @@ def __init__( project_repo: ProjectRepository, group_repo: GroupRepository, search_updates_repo: SearchUpdatesRepo, - project_storage_config: ProjectStorageConfig, ) -> None: self.session_maker = session_maker self.authz = authz self.project_repo = project_repo self.group_repo = group_repo self.search_updates_repo = search_updates_repo - self.project_storage_config = project_storage_config async def get_data_connectors( self, diff --git a/components/renku_data_services/notebooks/blueprints.py b/components/renku_data_services/notebooks/blueprints.py index c9dccefcae..5cb2517b8a 100644 --- a/components/renku_data_services/notebooks/blueprints.py +++ b/components/renku_data_services/notebooks/blueprints.py @@ -40,6 +40,7 @@ from renku_data_services.resource_usage.db import ResourceRequestsRepo from renku_data_services.session.config import BuildsConfig from renku_data_services.session.db import SessionRepository +from renku_data_services.storage.db import ProjectStorageRepository from renku_data_services.users.db import UserRepo logger = logging.getLogger(__name__) @@ -62,6 +63,7 @@ class NotebooksNewBP(CustomBlueprint): image_check_repo: ImageCheckRepository project_repo: ProjectRepository project_session_secret_repo: ProjectSessionSecretRepository + project_storage_repo: ProjectStorageRepository rp_repo: ResourcePoolRepository session_repo: SessionRepository user_repo: UserRepo @@ -96,6 +98,7 @@ async def _handler( data_connector_repo=self.data_connector_repo, project_repo=self.project_repo, project_session_secret_repo=self.project_session_secret_repo, + project_storage_repo=self.project_storage_repo, rp_repo=self.rp_repo, session_repo=self.session_repo, user_repo=self.user_repo, diff --git a/components/renku_data_services/notebooks/core_sessions.py b/components/renku_data_services/notebooks/core_sessions.py index 804db7c268..9ee858cbd7 100644 --- a/components/renku_data_services/notebooks/core_sessions.py +++ b/components/renku_data_services/notebooks/core_sessions.py @@ -44,7 +44,6 @@ DataConnectorSecretRepository, ) from renku_data_services.data_connectors.models import DataConnectorSecret, DataConnectorWithSecrets -from renku_data_services.data_connectors.project_storage_k8s import ProjectStorageK8s from renku_data_services.errors import ValidationError, errors from renku_data_services.k8s.models import ClusterConnection, K8sSecret, sanitizer from renku_data_services.notebooks import apispec @@ -125,6 +124,8 @@ from renku_data_services.session.config import BuildsConfig from renku_data_services.session.db import SessionRepository from renku_data_services.session.models import Environment, SessionLauncher +from renku_data_services.storage.db import ProjectStorageRepository +from renku_data_services.storage.project_storage_k8s import ProjectStorageK8s from renku_data_services.users.db import UserRepo from renku_data_services.utils.core import get_effective_quota @@ -191,14 +192,14 @@ async def get_extra_containers( async def get_project_storage( user: APIUser, project_storage_k8s: ProjectStorageK8s, - data_connector_repo: DataConnectorRepository, + project_storage_repo: ProjectStorageRepository, project_id: ULID, storage_mount: PurePosixPath, cluster: ClusterConnection, authz: Authz, ) -> SessionExtraResources: """If applicable, fetch the project storage and return it as SessionExtras.""" - project_storage = await data_connector_repo.get_storage_to(user, project_id) + project_storage = await project_storage_repo.get_storage_to(user, project_id) if not project_storage: logger.debug(f"Project {project_id} has no project storage.") return SessionExtraResources() @@ -929,6 +930,7 @@ async def start_session( data_connector_secret_repo: DataConnectorSecretRepository, project_repo: ProjectRepository, project_session_secret_repo: ProjectSessionSecretRepository, + project_storage_repo: ProjectStorageRepository, rp_repo: ResourcePoolRepository, session_repo: SessionRepository, user_repo: UserRepo, @@ -1069,7 +1071,7 @@ async def start_session( project_storage_k8s = ProjectStorageK8s(nb_config.k8s_v2_client) session_extras = session_extras.concat( await get_project_storage( - user, project_storage_k8s, data_connector_repo, project.id, storage_mount, cluster, authz + user, project_storage_k8s, project_storage_repo, project.id, storage_mount, cluster, authz ) ) diff --git a/components/renku_data_services/project/blueprints.py b/components/renku_data_services/project/blueprints.py index 101fc3f4a2..df99f5c9b1 100644 --- a/components/renku_data_services/project/blueprints.py +++ b/components/renku_data_services/project/blueprints.py @@ -23,7 +23,6 @@ from renku_data_services.base_models.metrics import MetricsService, ProjectCreationType from renku_data_services.base_models.validation import validate_and_dump, validated_json from renku_data_services.data_connectors.db import DataConnectorRepository -from renku_data_services.data_connectors.project_storage_k8s import ProjectStorageK8s from renku_data_services.errors import errors from renku_data_services.project import apispec from renku_data_services.project import models as project_models @@ -42,6 +41,7 @@ ProjectSessionSecretRepository, ) from renku_data_services.session.db import SessionRepository +from renku_data_services.storage.project_storage_k8s import ProjectStorageK8s from renku_data_services.users.db import UserRepo diff --git a/components/renku_data_services/storage/blueprints.py b/components/renku_data_services/storage/blueprints.py index 4cfc684326..d5f6e102c8 100644 --- a/components/renku_data_services/storage/blueprints.py +++ b/components/renku_data_services/storage/blueprints.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from typing import Any -from sanic import HTTPResponse, Request, empty +from sanic import Request, empty from sanic.response import HTTPResponse, JSONResponse from sanic_ext import validate from ulid import ULID @@ -20,7 +20,6 @@ from renku_data_services.base_api.misc import validate_query from renku_data_services.base_api.pagination import PaginationRequest, paginate from renku_data_services.base_models.validation import validate_and_dump, validated_json -from renku_data_services.data_connectors.project_storage_k8s import ProjectStorageK8s from renku_data_services.notebooks.data_sources import DataSourceRepository from renku_data_services.storage import apispec, models from renku_data_services.storage.core import ( @@ -30,6 +29,7 @@ validate_unsaved_project_storage, ) from renku_data_services.storage.db import ProjectStorageRepository +from renku_data_services.storage.project_storage_k8s import ProjectStorageK8s from renku_data_services.storage.rclone import RCloneValidator @@ -98,6 +98,8 @@ async def _obscure( @dataclass(kw_only=True) class ProjectStorageBP(CustomBlueprint): + """Handler for project storage.""" + project_storage_k8s: ProjectStorageK8s project_storage_repo: ProjectStorageRepository authenticator: base_models.Authenticator @@ -124,6 +126,52 @@ async def _get_one(_: Request, user: base_models.APIUser, storage_id: ULID, etag return "/data_connectors/storage/", ["GET"], _get_one + def get_storage_to_project(self) -> BlueprintFactoryResponse: + """List all project storage to a given project.""" + + @authenticate(self.authenticator) + async def _get_all_storage_to_project( + _: Request, + user: base_models.APIUser, + project_id: ULID, + ) -> JSONResponse: + project_storage = await self.project_storage_repo.get_storage_to(user=user, project_id=project_id) + result = [self._dump_project_storage(project_storage)] if project_storage else [] + return validated_json(apispec.ProjectStorageList, result) + + return "/projects//storage", ["GET"], _get_all_storage_to_project + + def get_storage_config(self) -> BlueprintFactoryResponse: + """Get the current config used for project storage.""" + + @authenticate(self.authenticator) + @only_admins + async def _get_project_config(_: Request, user: base_models.APIUser) -> JSONResponse: + storage_config = self.project_storage_repo.get_project_storage_config() + result = apispec.ProjectStorageConfig( + enabled=storage_config.enabled, max_size=int(storage_config.maximum_size.to_gibi()) + ) + return validated_json(apispec.ProjectStorageConfig, result) + + return "/data_connectors/storage/config", ["GET"], _get_project_config + + def delete_storage(self) -> BlueprintFactoryResponse: + """Delete a specific project storage.""" + + @authenticate(self.authenticator) + @only_authenticated + async def _delete_storage( + _: Request, + user: base_models.APIUser, + storage_id: ULID, + ) -> HTTPResponse: + deleted = await self.project_storage_repo.delete_project_storage(user=user, storage_id=storage_id) + if deleted: + await self.project_storage_k8s.delete_volume(deleted) + return HTTPResponse(status=204) + + return "/data_connectors/storage/", ["DELETE"], _delete_storage + def post_storage(self) -> BlueprintFactoryResponse: """Create a new shared project storage.""" diff --git a/components/renku_data_services/data_connectors/project_storage_k8s.py b/components/renku_data_services/storage/project_storage_k8s.py similarity index 95% rename from components/renku_data_services/data_connectors/project_storage_k8s.py rename to components/renku_data_services/storage/project_storage_k8s.py index 82fd01692d..6c8b4eabc3 100644 --- a/components/renku_data_services/data_connectors/project_storage_k8s.py +++ b/components/renku_data_services/storage/project_storage_k8s.py @@ -3,9 +3,9 @@ from ulid import ULID from renku_data_services.app_config import logging -from renku_data_services.data_connectors.models import DeletedProjectStorage, ProjectStorage from renku_data_services.k8s.models import ClusterConnection, K8sPersistentVolumeClaim from renku_data_services.notebooks.api.classes.k8s_client import NotebookK8sClient +from renku_data_services.storage.models import DeletedProjectStorage, ProjectStorage logger = logging.getLogger(__name__) diff --git a/test/bases/renku_data_services/data_api/test_data_connectors.py b/test/bases/renku_data_services/data_api/test_data_connectors.py index 7e9479b30b..54b9f6804a 100644 --- a/test/bases/renku_data_services/data_api/test_data_connectors.py +++ b/test/bases/renku_data_services/data_api/test_data_connectors.py @@ -2665,685 +2665,6 @@ async def link_dc_project( return response -@pytest.mark.asyncio -async def test_post_storage_success( - sanic_client: SanicASGITestClient, - create_project, - admin_headers: dict[str, str], - user_headers: dict[str, str], -) -> None: - project = await create_project(sanic_client, "Test Project") - namespace = f"{project['namespace']}/{project['slug']}" - project_id = project["id"] - - payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) - assert response.status_code == 201, response.text - - payload = {"namespace": namespace, "size": 10, "mount_path": "/data"} - _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) - - assert response.status_code == 201, response.text - assert response.json is not None - storage = response.json - assert storage.get("project_id") == project["id"] - assert storage.get("size") == 10 - assert storage.get("mount_path") == "/data" - assert storage.get("created_by") == "user" - assert "ETag" in response.headers - - -@pytest.mark.asyncio -async def test_post_storage_unauthenticated_fails( - sanic_client: SanicASGITestClient, create_project, admin_headers: dict[str, str] -) -> None: - project = await create_project(sanic_client, "Test Project") - namespace = f"{project['namespace']}/{project['slug']}" - - project_id = project["id"] - payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) - assert response.status_code == 201, response.text - - payload = {"namespace": namespace, "size": 10, "mount_path": "/data"} - _, response = await sanic_client.post("/api/data/data_connectors/storage", json=payload) - - assert response.status_code == 401, response.text - - -@pytest.mark.asyncio -async def test_post_storage_not_allowed_fails( - sanic_client: SanicASGITestClient, create_project, user_headers: dict[str, str] -) -> None: - project = await create_project(sanic_client, "Test Project") - namespace = f"{project['namespace']}/{project['slug']}" - - payload = {"namespace": namespace, "size": 10, "mount_path": "/data"} - _, response = await sanic_client.post("/api/data/data_connectors/storage", json=payload, headers=user_headers) - - assert response.status_code == 403, response.text - - -@pytest.mark.asyncio -async def test_post_storage_duplicate_fails( - sanic_client: SanicASGITestClient, create_project, user_headers: dict[str, str], admin_headers: dict[str, str] -) -> None: - project = await create_project(sanic_client, "Test Project") - namespace = f"{project['namespace']}/{project['slug']}" - - project_id = project["id"] - payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) - assert response.status == 201 - - payload = {"namespace": namespace, "size": 10, "mount_path": "/data"} - _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) - assert response.status_code == 201, response.text - - _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) - - assert response.status_code == 422, response.text - - -@pytest.mark.asyncio -async def test_get_one_storage_success( - sanic_client: SanicASGITestClient, create_project, user_headers: dict[str, str], admin_headers: dict[str, str] -) -> None: - project = await create_project(sanic_client, "Test Project") - namespace = f"{project['namespace']}/{project['slug']}" - - project_id = project["id"] - payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) - assert response.status_code == 201, response.text - - payload = {"namespace": namespace, "size": 10, "mount_path": "/data"} - _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) - assert response.status_code == 201, response.text - storage_id = response.json["id"] - - _, response = await sanic_client.get(f"/api/data/data_connectors/storage/{storage_id}", headers=user_headers) - - assert response.status_code == 200, response.text - assert response.json is not None - storage = response.json - assert storage.get("id") == storage_id - assert storage.get("project_id") == project["id"] - assert storage.get("size") == 10 - assert storage.get("mount_path") == "/data" - - -@pytest.mark.asyncio -async def test_get_one_storage_not_found(sanic_client: SanicASGITestClient, user_headers: dict[str, str]) -> None: - from ulid import ULID - - non_existent_id = str(ULID()) - _, response = await sanic_client.get(f"/api/data/data_connectors/storage/{non_existent_id}", headers=user_headers) - - assert response.status_code == 404, response.text - - -@pytest.mark.asyncio -async def test_get_one_storage_etag( - sanic_client: SanicASGITestClient, create_project, user_headers: dict[str, str], admin_headers: dict[str, str] -) -> None: - project = await create_project(sanic_client, "Test Project") - namespace = f"{project['namespace']}/{project['slug']}" - - project_id = project["id"] - payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) - assert response.status_code == 201, response.text - - payload = {"namespace": namespace, "size": 10, "mount_path": "/data"} - _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) - assert response.status_code == 201, response.text - storage_id = response.json["id"] - etag = response.headers["ETag"] - - headers = merge_headers(user_headers, {"If-None-Match": etag}) - _, response = await sanic_client.get(f"/api/data/data_connectors/storage/{storage_id}", headers=headers) - - assert response.status_code == 304, response.text - - -@pytest.mark.asyncio -async def test_get_storage_to_project_for_no_storage( - sanic_client: SanicASGITestClient, create_project, user_headers: dict[str, str] -) -> None: - project = await create_project(sanic_client, "Test Project") - - _, response = await sanic_client.get(f"/api/data/projects/{project['id']}/storage", headers=user_headers) - - assert response.status_code == 200, response.text - assert response.json == [] - - -@pytest.mark.asyncio -async def test_get_storage_to_project_success( - sanic_client: SanicASGITestClient, create_project, user_headers: dict[str, str], admin_headers: dict[str, str] -) -> None: - project = await create_project(sanic_client, "Test Project") - namespace = f"{project['namespace']}/{project['slug']}" - - project_id = project["id"] - payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) - assert response.status == 201 - - payload = {"namespace": namespace, "size": 10, "mount_path": "/data"} - _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) - assert response.status_code == 201, response.text - - _, response = await sanic_client.get(f"/api/data/projects/{project['id']}/storage", headers=user_headers) - - assert response.status_code == 200, response.text - assert len(response.json) == 1 - storage = response.json[0] - assert storage.get("project_id") == project["id"] - assert storage.get("size") == 10 - assert storage.get("mount_path") == "/data" - - -@pytest.mark.asyncio -async def test_delete_storage_success( - sanic_client: SanicASGITestClient, - create_project, - user_headers: dict[str, str], - admin_headers: dict[str, str], - cluster, -) -> None: - project = await create_project(sanic_client, "Test Project") - namespace = f"{project['namespace']}/{project['slug']}" - - project_id = project["id"] - payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) - assert response.status_code == 201, response.text - - payload = {"namespace": namespace, "size": 10, "mount_path": "/data"} - _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) - assert response.status_code == 201, response.text - storage_id = response.json["id"] - - _, response = await sanic_client.delete(f"/api/data/data_connectors/storage/{storage_id}", headers=user_headers) - assert response.status_code == 204, response.text - - _, response = await sanic_client.get(f"/api/data/projects/{project['id']}/storage", headers=user_headers) - assert response.status_code == 200, response.text - assert response.json == [] - - -@pytest.mark.asyncio -async def test_delete_storage_unauthenticated_fails( - sanic_client: SanicASGITestClient, create_project, user_headers: dict[str, str], admin_headers: dict[str, str] -) -> None: - project = await create_project(sanic_client, "Test Project") - namespace = f"{project['namespace']}/{project['slug']}" - - project_id = project["id"] - payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) - assert response.status_code == 201, response.text - - payload = {"namespace": namespace, "size": 10, "mount_path": "/data"} - _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) - assert response.status_code == 201, response.text - storage_id = response.json["id"] - - _, response = await sanic_client.delete(f"/api/data/data_connectors/storage/{storage_id}") - - assert response.status_code == 401, response.text - - -@pytest.mark.asyncio -async def test_post_storage_allow_success( - sanic_client: SanicASGITestClient, create_project, admin_headers: dict[str, str] -) -> None: - project = await create_project(sanic_client, "Test Project") - project_id = project["id"] - - payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) - - assert response.status_code == 201, response.text - assert response.json is not None - allow = response.json - assert allow.get("project_id") == project_id - assert allow.get("max_size") == 10 - - -@pytest.mark.asyncio -async def test_post_storage_allow_requires_admin( - sanic_client: SanicASGITestClient, create_project, user_headers: dict[str, str] -) -> None: - project = await create_project(sanic_client, "Test Project") - - payload = {"project_id": project["id"], "max_size": 10} - _, response = await sanic_client.post("/api/data/data_connectors/storage/allow", headers=user_headers, json=payload) - - assert response.status_code == 403, response.text - - -@pytest.mark.asyncio -async def test_post_storage_allow_unauthenticated_fails( - sanic_client: SanicASGITestClient, create_project, user_headers: dict[str, str] -) -> None: - project = await create_project(sanic_client, "Test Project") - - payload = {"project_id": project["id"], "max_size": 10} - _, response = await sanic_client.post("/api/data/data_connectors/storage/allow", json=payload) - - assert response.status_code == 401, response.text - - -@pytest.mark.asyncio -async def test_post_storage_allow_duplicate_fails( - sanic_client: SanicASGITestClient, create_project, admin_headers: dict[str, str] -) -> None: - project = await create_project(sanic_client, "Test Project") - project_id = project["id"] - - payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) - assert response.status_code == 201, response.text - - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) - - assert response.status_code == 422, response.text - - -@pytest.mark.asyncio -async def test_delete_storage_allow_success( - sanic_client: SanicASGITestClient, create_project, admin_headers: dict[str, str] -) -> None: - project = await create_project(sanic_client, "Test Project") - project_id = project["id"] - - payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) - assert response.status_code == 201, response.text - - _, response = await sanic_client.delete( - f"/api/data/data_connectors/storage/allow/{project_id}", headers=admin_headers - ) - - assert response.status_code == 204, response.text - - # Re-adding after deletion should succeed - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) - assert response.status_code == 201, response.text - - -@pytest.mark.asyncio -async def test_delete_storage_allow_requires_admin( - sanic_client: SanicASGITestClient, create_project, admin_headers: dict[str, str], user_headers: dict[str, str] -) -> None: - project = await create_project(sanic_client, "Test Project") - project_id = project["id"] - - payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) - assert response.status_code == 201, response.text - - _, response = await sanic_client.delete( - f"/api/data/data_connectors/storage/allow/{project_id}", headers=user_headers - ) - - assert response.status_code == 403, response.text - - -@pytest.mark.asyncio -async def test_get_storage_allow_success( - sanic_client: SanicASGITestClient, create_project, admin_headers: dict[str, str], user_headers: dict[str, str] -) -> None: - project = await create_project(sanic_client, "Test Project") - project_id = project["id"] - - payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) - assert response.status_code == 201, response.text - - _, response = await sanic_client.get(f"/api/data/data_connectors/storage/allow/{project_id}", headers=user_headers) - - assert response.status_code == 200, response.text - assert response.json is not None - assert response.json.get("project_id") == project_id - assert response.json.get("max_size") == 10 - - -@pytest.mark.asyncio -async def test_get_storage_allow_not_in_list( - sanic_client: SanicASGITestClient, create_project, user_headers: dict[str, str] -) -> None: - project = await create_project(sanic_client, "Test Project") - project_id = project["id"] - - _, response = await sanic_client.get(f"/api/data/data_connectors/storage/allow/{project_id}", headers=user_headers) - - assert response.status_code == 404, response.text - - -@pytest.mark.asyncio -async def test_get_storage_allow_unauthenticated( - sanic_client: SanicASGITestClient, create_project, admin_headers: dict[str, str] -) -> None: - project = await create_project(sanic_client, "Test Project") - project_id = project["id"] - - payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) - assert response.status_code == 201, response.text - - _, response = await sanic_client.get(f"/api/data/data_connectors/storage/allow/{project_id}") - - assert response.status_code == 401, response.text - - -@pytest.mark.asyncio -async def test_patch_storage_success( - sanic_client: SanicASGITestClient, create_project, user_headers: dict[str, str], admin_headers: dict[str, str] -) -> None: - project = await create_project(sanic_client, "Test Project") - namespace = f"{project['namespace']}/{project['slug']}" - - project_id = project["id"] - payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) - assert response.status_code == 201, response.text - - payload = {"namespace": namespace, "size": 5, "mount_path": "/data"} - _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) - assert response.status_code == 201, response.text - storage = response.json - storage_id = storage["id"] - original_etag = response.headers["ETag"] - - # Patch the size - headers = merge_headers(user_headers, {"If-Match": original_etag}) - patch = {"size": 8} - _, response = await sanic_client.patch( - f"/api/data/data_connectors/storage/{storage_id}", headers=headers, json=patch - ) - - assert response.status_code == 200, response.text - assert response.json is not None - updated_storage = response.json - assert updated_storage.get("id") == storage_id - assert updated_storage.get("size") == 8 - assert updated_storage.get("mount_path") == "/data" - - -@pytest.mark.asyncio -async def test_patch_storage_mount_path( - sanic_client: SanicASGITestClient, create_project, user_headers: dict[str, str], admin_headers: dict[str, str] -) -> None: - project = await create_project(sanic_client, "Test Project") - namespace = f"{project['namespace']}/{project['slug']}" - - project_id = project["id"] - payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) - assert response.status_code == 201, response.text - - payload = {"namespace": namespace, "size": 10, "mount_path": "/data"} - _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) - assert response.status_code == 201, response.text - storage = response.json - storage_id = storage["id"] - - # Patch the mount path - headers = merge_headers(user_headers, {"If-Match": response.headers["ETag"]}) - patch = {"mount_path": "/new/mount"} - _, response = await sanic_client.patch( - f"/api/data/data_connectors/storage/{storage_id}", headers=headers, json=patch - ) - - assert response.status_code == 200, response.text - assert response.json is not None - updated_storage = response.json - assert updated_storage.get("id") == storage_id - assert updated_storage.get("size") == 10 - assert updated_storage.get("mount_path") == "/new/mount" - - -@pytest.mark.asyncio -async def test_patch_storage_both_fields( - sanic_client: SanicASGITestClient, create_project, user_headers: dict[str, str], admin_headers: dict[str, str] -) -> None: - project = await create_project(sanic_client, "Test Project") - namespace = f"{project['namespace']}/{project['slug']}" - - project_id = project["id"] - payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) - assert response.status_code == 201, response.text - - payload = {"namespace": namespace, "size": 5, "mount_path": "/data"} - _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) - assert response.status_code == 201, response.text - storage = response.json - storage_id = storage["id"] - - # Patch both fields at once - headers = merge_headers(user_headers, {"If-Match": response.headers["ETag"]}) - patch = {"size": 8, "mount_path": "/new/mount"} - _, response = await sanic_client.patch( - f"/api/data/data_connectors/storage/{storage_id}", headers=headers, json=patch - ) - - assert response.status_code == 200, response.text - assert response.json is not None - updated_storage = response.json - assert updated_storage.get("id") == storage_id - assert updated_storage.get("size") == 8 - assert updated_storage.get("mount_path") == "/new/mount" - - -@pytest.mark.asyncio -async def test_patch_storage_without_if_match_header( - sanic_client: SanicASGITestClient, create_project, user_headers: dict[str, str], admin_headers: dict[str, str] -) -> None: - project = await create_project(sanic_client, "Test Project") - namespace = f"{project['namespace']}/{project['slug']}" - - project_id = project["id"] - payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) - assert response.status_code == 201, response.text - - payload = {"namespace": namespace, "size": 10, "mount_path": "/data"} - _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) - assert response.status_code == 201, response.text - storage_id = response.json["id"] - - # Patch without If-Match header - patch = {"size": 20} - _, response = await sanic_client.patch( - f"/api/data/data_connectors/storage/{storage_id}", headers=user_headers, json=patch - ) - - assert response.status_code == 428, response.text - assert "If-Match header not provided" in response.text - - -@pytest.mark.asyncio -async def test_patch_storage_with_invalid_etag( - sanic_client: SanicASGITestClient, create_project, user_headers: dict[str, str], admin_headers: dict[str, str] -) -> None: - project = await create_project(sanic_client, "Test Project") - namespace = f"{project['namespace']}/{project['slug']}" - - project_id = project["id"] - payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) - assert response.status_code == 201, response.text - - payload = {"namespace": namespace, "size": 5, "mount_path": "/data"} - _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) - assert response.status_code == 201, response.text - storage_id = response.json["id"] - correct_etag = response.headers["ETag"] - - # Patch with wrong ETag - headers = merge_headers(user_headers, {"If-Match": "wrong-etag"}) - patch = {"size": 8} - _, response = await sanic_client.patch( - f"/api/data/data_connectors/storage/{storage_id}", headers=headers, json=patch - ) - - assert response.status_code == 409, response.text - - # Verify the etag changed after a successful patch - headers = merge_headers(user_headers, {"If-Match": correct_etag}) - patch = {"size": 6} - _, response = await sanic_client.patch( - f"/api/data/data_connectors/storage/{storage_id}", headers=headers, json=patch - ) - assert response.status_code == 200, response.text - new_etag = response.headers["ETag"] - assert new_etag != correct_etag - - -@pytest.mark.asyncio -async def test_patch_storage_not_found(sanic_client: SanicASGITestClient, user_headers: dict[str, str]) -> None: - from ulid import ULID - - non_existent_id = str(ULID()) - headers = merge_headers(user_headers, {"If-Match": "some-etag"}) - patch = {"size": 20} - _, response = await sanic_client.patch( - f"/api/data/data_connectors/storage/{non_existent_id}", headers=headers, json=patch - ) - - assert response.status_code == 404, response.text - - -@pytest.mark.asyncio -async def test_patch_storage_unauthenticated_fails( - sanic_client: SanicASGITestClient, create_project, admin_headers: dict[str, str], user_headers: dict[str, str] -) -> None: - project = await create_project(sanic_client, "Test Project") - namespace = f"{project['namespace']}/{project['slug']}" - - project_id = project["id"] - payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) - assert response.status_code == 201, response.text - - payload = {"namespace": namespace, "size": 5, "mount_path": "/data"} - _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) - assert response.status_code == 201, response.text - storage_id = response.json["id"] - - # Patch without authentication - _, response = await sanic_client.patch(f"/api/data/data_connectors/storage/{storage_id}", json={"size": 8}) - - assert response.status_code == 401, response.text - - -@pytest.mark.asyncio -async def test_patch_storage_exceeds_max_size( - sanic_client: SanicASGITestClient, create_project, user_headers: dict[str, str], admin_headers: dict[str, str] -) -> None: - project = await create_project(sanic_client, "Test Project") - namespace = f"{project['namespace']}/{project['slug']}" - - project_id = project["id"] - payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) - assert response.status_code == 201, response.text - - payload = {"namespace": namespace, "size": 5, "mount_path": "/data"} - _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) - assert response.status_code == 201, response.text - storage_id = response.json["id"] - - # Try to patch size beyond the allowed max (10GB) - headers = merge_headers(user_headers, {"If-Match": response.headers["ETag"]}) - patch = {"size": 11} - _, response = await sanic_client.patch( - f"/api/data/data_connectors/storage/{storage_id}", headers=headers, json=patch - ) - - assert response.status_code == 422, response.text - - -@pytest.mark.asyncio -async def test_patch_storage_invalid_mount_path( - sanic_client: SanicASGITestClient, create_project, user_headers: dict[str, str], admin_headers: dict[str, str] -) -> None: - project = await create_project(sanic_client, "Test Project") - namespace = f"{project['namespace']}/{project['slug']}" - - project_id = project["id"] - payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) - assert response.status_code == 201, response.text - - payload = {"namespace": namespace, "size": 10, "mount_path": "/data"} - _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) - assert response.status_code == 201, response.text - storage_id = response.json["id"] - - # Try to patch with invalid mount path - headers = merge_headers(user_headers, {"If-Match": response.headers["ETag"]}) - patch = {"mount_path": "/etc/passwd"} - _, response = await sanic_client.patch( - f"/api/data/data_connectors/storage/{storage_id}", headers=headers, json=patch - ) - - assert response.status_code == 422, response.text - - async def test_get_all_dc_links( sanic_client: SanicASGITestClient, regular_user: UserInfo, @@ -3387,215 +2708,3 @@ async def test_get_all_dc_links( assert res.status_code == 200, res.text assert len(res.json) == 2 assert p2["id"] not in [i["id"] for i in res.json] - - -# --- PATCH Project Storage Allow Tests --- - - -@pytest.mark.asyncio -async def test_patch_storage_allow_success( - sanic_client: SanicASGITestClient, create_project, admin_headers: dict[str, str], user_headers: dict[str, str] -) -> None: - project = await create_project(sanic_client, "Test Project") - project_id = project["id"] - - payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) - assert response.status_code == 201, response.text - - # Get the allow entry to retrieve the etag (use user_headers as admin may not have read access) - _, response = await sanic_client.get(f"/api/data/data_connectors/storage/allow/{project_id}", headers=user_headers) - assert response.status_code == 200, response.text - etag = response.headers["ETag"] - - # Patch the max_size - headers = merge_headers(admin_headers, {"If-Match": etag}) - patch = {"max_size": 20} - _, response = await sanic_client.patch( - f"/api/data/data_connectors/storage/allow/{project_id}", headers=headers, json=patch - ) - - assert response.status_code == 200, response.text - assert response.json is not None - updated_allow = response.json - assert updated_allow.get("project_id") == project_id - assert updated_allow.get("max_size") == 20 - - -@pytest.mark.asyncio -async def test_patch_storage_allow_requires_admin( - sanic_client: SanicASGITestClient, create_project, admin_headers: dict[str, str], user_headers: dict[str, str] -) -> None: - project = await create_project(sanic_client, "Test Project") - project_id = project["id"] - - payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) - assert response.status_code == 201, response.text - - # Get the allow entry to retrieve the etag (use user_headers as admin may not have read access) - _, response = await sanic_client.get(f"/api/data/data_connectors/storage/allow/{project_id}", headers=admin_headers) - assert response.status_code == 200, response.text - etag = response.headers["ETag"] - - # Try to patch as non-admin - headers = merge_headers(user_headers, {"If-Match": etag}) - patch = {"max_size": 20} - _, response = await sanic_client.patch( - f"/api/data/data_connectors/storage/allow/{project_id}", headers=headers, json=patch - ) - - assert response.status_code == 403, response.text - - -@pytest.mark.asyncio -async def test_patch_storage_allow_without_if_match_header( - sanic_client: SanicASGITestClient, create_project, admin_headers: dict[str, str] -) -> None: - project = await create_project(sanic_client, "Test Project") - project_id = project["id"] - - payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) - assert response.status_code == 201, response.text - - # Patch without If-Match header - patch = {"max_size": 20} - _, response = await sanic_client.patch( - f"/api/data/data_connectors/storage/allow/{project_id}", headers=admin_headers, json=patch - ) - - assert response.status_code == 428, response.text - assert "If-Match header not provided" in response.text - - -@pytest.mark.asyncio -async def test_patch_storage_allow_with_invalid_etag( - sanic_client: SanicASGITestClient, create_project, admin_headers: dict[str, str], user_headers: dict[str, str] -) -> None: - project = await create_project(sanic_client, "Test Project") - project_id = project["id"] - - payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) - assert response.status_code == 201, response.text - - # Get the allow entry to retrieve the etag (use user_headers as admin may not have read access) - _, response = await sanic_client.get(f"/api/data/data_connectors/storage/allow/{project_id}", headers=user_headers) - assert response.status_code == 200, response.text - correct_etag = response.headers["ETag"] - - # Patch with wrong ETag - headers = merge_headers(admin_headers, {"If-Match": "wrong-etag"}) - patch = {"max_size": 20} - _, response = await sanic_client.patch( - f"/api/data/data_connectors/storage/allow/{project_id}", headers=headers, json=patch - ) - - assert response.status_code == 409, response.text - - # Verify the etag changed after a successful patch - headers = merge_headers(admin_headers, {"If-Match": correct_etag}) - patch = {"max_size": 15} - _, response = await sanic_client.patch( - f"/api/data/data_connectors/storage/allow/{project_id}", headers=headers, json=patch - ) - assert response.status_code == 200, response.text - new_etag = response.headers["ETag"] - assert new_etag != correct_etag - - -@pytest.mark.asyncio -async def test_patch_storage_allow_not_in_list( - sanic_client: SanicASGITestClient, create_project, admin_headers: dict[str, str] -) -> None: - project = await create_project(sanic_client, "Test Project") - project_id = project["id"] - - headers = merge_headers(admin_headers, {"If-Match": "some-etag"}) - patch = {"max_size": 20} - _, response = await sanic_client.patch( - f"/api/data/data_connectors/storage/allow/{project_id}", headers=headers, json=patch - ) - - assert response.status_code == 404, response.text - - -@pytest.mark.asyncio -async def test_patch_storage_allow_unauthenticated_fails( - sanic_client: SanicASGITestClient, create_project, admin_headers: dict[str, str] -) -> None: - project = await create_project(sanic_client, "Test Project") - project_id = project["id"] - - payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) - assert response.status_code == 201, response.text - - # Patch without authentication - _, response = await sanic_client.patch( - f"/api/data/data_connectors/storage/allow/{project_id}", json={"max_size": 20} - ) - - assert response.status_code == 401, response.text - - -@pytest.mark.asyncio -async def test_patch_storage_allow_min_size( - sanic_client: SanicASGITestClient, create_project, admin_headers: dict[str, str], user_headers: dict[str, str] -) -> None: - project = await create_project(sanic_client, "Test Project") - project_id = project["id"] - - payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) - assert response.status_code == 201, response.text - - # Get the allow entry to retrieve the etag (use user_headers as admin may not have read access) - _, response = await sanic_client.get(f"/api/data/data_connectors/storage/allow/{project_id}", headers=user_headers) - assert response.status_code == 200, response.text - etag = response.headers["ETag"] - - # Try to set max_size below minimum (1GB) - headers = merge_headers(admin_headers, {"If-Match": etag}) - patch = {"max_size": 0} - _, response = await sanic_client.patch( - f"/api/data/data_connectors/storage/allow/{project_id}", headers=headers, json=patch - ) - - assert response.status_code == 422, response.text - assert "at least 1GB" in response.text - - -@pytest.mark.asyncio -async def test_get_all_storage_allow( - sanic_client: SanicASGITestClient, create_project, admin_headers: dict[str, str], user_headers: dict[str, str] -) -> None: - project = await create_project(sanic_client, "Test Project") - project_id = project["id"] - - payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) - assert response.status_code == 201, response.text - - _, response = await sanic_client.get("/api/data/data_connectors/storage/allow", headers=admin_headers) - assert response.status_code == 200, response.text - assert response.json is not None, f"No json response body: {response.text}" - assert isinstance(response.json, list) - assert len(response.json) == 1 - assert response.json[0]["project_id"] == project_id - assert response.json[0]["max_size"] == 10 diff --git a/test/bases/renku_data_services/data_api/test_storage.py b/test/bases/renku_data_services/data_api/test_storage.py index d76c77f75e..6d2f349881 100644 --- a/test/bases/renku_data_services/data_api/test_storage.py +++ b/test/bases/renku_data_services/data_api/test_storage.py @@ -28,6 +28,14 @@ } +def merge_headers(*headers: dict[str, str]) -> dict[str, str]: + """Merge multiple headers.""" + all_headers = dict() + for h in headers: + all_headers.update(**h) + return all_headers + + @pytest.fixture def valid_storage_payload() -> dict[str, Any]: return _valid_storage @@ -229,3 +237,891 @@ async def test_storage_validate_connection_supports_doi(storage_test_client) -> payload = {"configuration": {"type": "doi", "doi": "10.5281/zenodo.15174623"}, "source_path": ""} _, res = await storage_test_client.post("/api/data/storage_schema/test_connection", json=payload) assert res.status_code == 204, res.text + + +@pytest.mark.asyncio +async def test_post_storage_success( + sanic_client: SanicASGITestClient, + create_project, + admin_headers: dict[str, str], + user_headers: dict[str, str], +) -> None: + project = await create_project(sanic_client, "Test Project") + namespace = f"{project['namespace']}/{project['slug']}" + project_id = project["id"] + + payload = {"project_id": project_id, "max_size": 10} + _, response = await sanic_client.post( + "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload + ) + assert response.status_code == 201, response.text + + payload = {"namespace": namespace, "size": 10, "mount_path": "/data"} + _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) + + assert response.status_code == 201, response.text + assert response.json is not None + storage = response.json + assert storage.get("project_id") == project["id"] + assert storage.get("size") == 10 + assert storage.get("mount_path") == "/data" + assert storage.get("created_by") == "user" + assert "ETag" in response.headers + + +@pytest.mark.asyncio +async def test_post_storage_unauthenticated_fails( + sanic_client: SanicASGITestClient, create_project, admin_headers: dict[str, str] +) -> None: + project = await create_project(sanic_client, "Test Project") + namespace = f"{project['namespace']}/{project['slug']}" + + project_id = project["id"] + payload = {"project_id": project_id, "max_size": 10} + _, response = await sanic_client.post( + "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload + ) + assert response.status_code == 201, response.text + + payload = {"namespace": namespace, "size": 10, "mount_path": "/data"} + _, response = await sanic_client.post("/api/data/data_connectors/storage", json=payload) + + assert response.status_code == 401, response.text + + +@pytest.mark.asyncio +async def test_post_storage_not_allowed_fails( + sanic_client: SanicASGITestClient, create_project, user_headers: dict[str, str] +) -> None: + project = await create_project(sanic_client, "Test Project") + namespace = f"{project['namespace']}/{project['slug']}" + + payload = {"namespace": namespace, "size": 10, "mount_path": "/data"} + _, response = await sanic_client.post("/api/data/data_connectors/storage", json=payload, headers=user_headers) + + assert response.status_code == 403, response.text + + +@pytest.mark.asyncio +async def test_post_storage_duplicate_fails( + sanic_client: SanicASGITestClient, create_project, user_headers: dict[str, str], admin_headers: dict[str, str] +) -> None: + project = await create_project(sanic_client, "Test Project") + namespace = f"{project['namespace']}/{project['slug']}" + + project_id = project["id"] + payload = {"project_id": project_id, "max_size": 10} + _, response = await sanic_client.post( + "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload + ) + assert response.status == 201 + + payload = {"namespace": namespace, "size": 10, "mount_path": "/data"} + _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) + assert response.status_code == 201, response.text + + _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) + + assert response.status_code == 422, response.text + + +@pytest.mark.asyncio +async def test_get_one_storage_success( + sanic_client: SanicASGITestClient, create_project, user_headers: dict[str, str], admin_headers: dict[str, str] +) -> None: + project = await create_project(sanic_client, "Test Project") + namespace = f"{project['namespace']}/{project['slug']}" + + project_id = project["id"] + payload = {"project_id": project_id, "max_size": 10} + _, response = await sanic_client.post( + "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload + ) + assert response.status_code == 201, response.text + + payload = {"namespace": namespace, "size": 10, "mount_path": "/data"} + _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) + assert response.status_code == 201, response.text + storage_id = response.json["id"] + + _, response = await sanic_client.get(f"/api/data/data_connectors/storage/{storage_id}", headers=user_headers) + + assert response.status_code == 200, response.text + assert response.json is not None + storage = response.json + assert storage.get("id") == storage_id + assert storage.get("project_id") == project["id"] + assert storage.get("size") == 10 + assert storage.get("mount_path") == "/data" + + +@pytest.mark.asyncio +async def test_get_one_storage_not_found(sanic_client: SanicASGITestClient, user_headers: dict[str, str]) -> None: + from ulid import ULID + + non_existent_id = str(ULID()) + _, response = await sanic_client.get(f"/api/data/data_connectors/storage/{non_existent_id}", headers=user_headers) + + assert response.status_code == 404, response.text + + +@pytest.mark.asyncio +async def test_get_one_storage_etag( + sanic_client: SanicASGITestClient, create_project, user_headers: dict[str, str], admin_headers: dict[str, str] +) -> None: + project = await create_project(sanic_client, "Test Project") + namespace = f"{project['namespace']}/{project['slug']}" + + project_id = project["id"] + payload = {"project_id": project_id, "max_size": 10} + _, response = await sanic_client.post( + "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload + ) + assert response.status_code == 201, response.text + + payload = {"namespace": namespace, "size": 10, "mount_path": "/data"} + _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) + assert response.status_code == 201, response.text + storage_id = response.json["id"] + etag = response.headers["ETag"] + + headers = merge_headers(user_headers, {"If-None-Match": etag}) + _, response = await sanic_client.get(f"/api/data/data_connectors/storage/{storage_id}", headers=headers) + + assert response.status_code == 304, response.text + + +@pytest.mark.asyncio +async def test_get_storage_to_project_for_no_storage( + sanic_client: SanicASGITestClient, create_project, user_headers: dict[str, str] +) -> None: + project = await create_project(sanic_client, "Test Project") + + _, response = await sanic_client.get(f"/api/data/projects/{project['id']}/storage", headers=user_headers) + + assert response.status_code == 200, response.text + assert response.json == [] + + +@pytest.mark.asyncio +async def test_get_storage_to_project_success( + sanic_client: SanicASGITestClient, create_project, user_headers: dict[str, str], admin_headers: dict[str, str] +) -> None: + project = await create_project(sanic_client, "Test Project") + namespace = f"{project['namespace']}/{project['slug']}" + + project_id = project["id"] + payload = {"project_id": project_id, "max_size": 10} + _, response = await sanic_client.post( + "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload + ) + assert response.status == 201 + + payload = {"namespace": namespace, "size": 10, "mount_path": "/data"} + _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) + assert response.status_code == 201, response.text + + _, response = await sanic_client.get(f"/api/data/projects/{project['id']}/storage", headers=user_headers) + + assert response.status_code == 200, response.text + assert len(response.json) == 1 + storage = response.json[0] + assert storage.get("project_id") == project["id"] + assert storage.get("size") == 10 + assert storage.get("mount_path") == "/data" + + +@pytest.mark.asyncio +async def test_delete_storage_success( + sanic_client: SanicASGITestClient, + create_project, + user_headers: dict[str, str], + admin_headers: dict[str, str], + cluster, +) -> None: + project = await create_project(sanic_client, "Test Project") + namespace = f"{project['namespace']}/{project['slug']}" + + project_id = project["id"] + payload = {"project_id": project_id, "max_size": 10} + _, response = await sanic_client.post( + "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload + ) + assert response.status_code == 201, response.text + + payload = {"namespace": namespace, "size": 10, "mount_path": "/data"} + _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) + assert response.status_code == 201, response.text + storage_id = response.json["id"] + + _, response = await sanic_client.delete(f"/api/data/data_connectors/storage/{storage_id}", headers=user_headers) + assert response.status_code == 204, response.text + + _, response = await sanic_client.get(f"/api/data/projects/{project['id']}/storage", headers=user_headers) + assert response.status_code == 200, response.text + assert response.json == [] + + +@pytest.mark.asyncio +async def test_delete_storage_unauthenticated_fails( + sanic_client: SanicASGITestClient, create_project, user_headers: dict[str, str], admin_headers: dict[str, str] +) -> None: + project = await create_project(sanic_client, "Test Project") + namespace = f"{project['namespace']}/{project['slug']}" + + project_id = project["id"] + payload = {"project_id": project_id, "max_size": 10} + _, response = await sanic_client.post( + "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload + ) + assert response.status_code == 201, response.text + + payload = {"namespace": namespace, "size": 10, "mount_path": "/data"} + _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) + assert response.status_code == 201, response.text + storage_id = response.json["id"] + + _, response = await sanic_client.delete(f"/api/data/data_connectors/storage/{storage_id}") + + assert response.status_code == 401, response.text + + +@pytest.mark.asyncio +async def test_post_storage_allow_success( + sanic_client: SanicASGITestClient, create_project, admin_headers: dict[str, str] +) -> None: + project = await create_project(sanic_client, "Test Project") + project_id = project["id"] + + payload = {"project_id": project_id, "max_size": 10} + _, response = await sanic_client.post( + "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload + ) + + assert response.status_code == 201, response.text + assert response.json is not None + allow = response.json + assert allow.get("project_id") == project_id + assert allow.get("max_size") == 10 + + +@pytest.mark.asyncio +async def test_post_storage_allow_requires_admin( + sanic_client: SanicASGITestClient, create_project, user_headers: dict[str, str] +) -> None: + project = await create_project(sanic_client, "Test Project") + + payload = {"project_id": project["id"], "max_size": 10} + _, response = await sanic_client.post("/api/data/data_connectors/storage/allow", headers=user_headers, json=payload) + + assert response.status_code == 403, response.text + + +@pytest.mark.asyncio +async def test_post_storage_allow_unauthenticated_fails( + sanic_client: SanicASGITestClient, create_project, user_headers: dict[str, str] +) -> None: + project = await create_project(sanic_client, "Test Project") + + payload = {"project_id": project["id"], "max_size": 10} + _, response = await sanic_client.post("/api/data/data_connectors/storage/allow", json=payload) + + assert response.status_code == 401, response.text + + +@pytest.mark.asyncio +async def test_post_storage_allow_duplicate_fails( + sanic_client: SanicASGITestClient, create_project, admin_headers: dict[str, str] +) -> None: + project = await create_project(sanic_client, "Test Project") + project_id = project["id"] + + payload = {"project_id": project_id, "max_size": 10} + _, response = await sanic_client.post( + "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload + ) + assert response.status_code == 201, response.text + + _, response = await sanic_client.post( + "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload + ) + + assert response.status_code == 422, response.text + + +@pytest.mark.asyncio +async def test_delete_storage_allow_success( + sanic_client: SanicASGITestClient, create_project, admin_headers: dict[str, str] +) -> None: + project = await create_project(sanic_client, "Test Project") + project_id = project["id"] + + payload = {"project_id": project_id, "max_size": 10} + _, response = await sanic_client.post( + "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload + ) + assert response.status_code == 201, response.text + + _, response = await sanic_client.delete( + f"/api/data/data_connectors/storage/allow/{project_id}", headers=admin_headers + ) + + assert response.status_code == 204, response.text + + # Re-adding after deletion should succeed + _, response = await sanic_client.post( + "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload + ) + assert response.status_code == 201, response.text + + +@pytest.mark.asyncio +async def test_delete_storage_allow_requires_admin( + sanic_client: SanicASGITestClient, create_project, admin_headers: dict[str, str], user_headers: dict[str, str] +) -> None: + project = await create_project(sanic_client, "Test Project") + project_id = project["id"] + + payload = {"project_id": project_id, "max_size": 10} + _, response = await sanic_client.post( + "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload + ) + assert response.status_code == 201, response.text + + _, response = await sanic_client.delete( + f"/api/data/data_connectors/storage/allow/{project_id}", headers=user_headers + ) + + assert response.status_code == 403, response.text + + +@pytest.mark.asyncio +async def test_get_storage_allow_success( + sanic_client: SanicASGITestClient, create_project, admin_headers: dict[str, str], user_headers: dict[str, str] +) -> None: + project = await create_project(sanic_client, "Test Project") + project_id = project["id"] + + payload = {"project_id": project_id, "max_size": 10} + _, response = await sanic_client.post( + "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload + ) + assert response.status_code == 201, response.text + + _, response = await sanic_client.get(f"/api/data/data_connectors/storage/allow/{project_id}", headers=user_headers) + + assert response.status_code == 200, response.text + assert response.json is not None + assert response.json.get("project_id") == project_id + assert response.json.get("max_size") == 10 + + +@pytest.mark.asyncio +async def test_get_storage_allow_not_in_list( + sanic_client: SanicASGITestClient, create_project, user_headers: dict[str, str] +) -> None: + project = await create_project(sanic_client, "Test Project") + project_id = project["id"] + + _, response = await sanic_client.get(f"/api/data/data_connectors/storage/allow/{project_id}", headers=user_headers) + + assert response.status_code == 404, response.text + + +@pytest.mark.asyncio +async def test_get_storage_allow_unauthenticated( + sanic_client: SanicASGITestClient, create_project, admin_headers: dict[str, str] +) -> None: + project = await create_project(sanic_client, "Test Project") + project_id = project["id"] + + payload = {"project_id": project_id, "max_size": 10} + _, response = await sanic_client.post( + "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload + ) + assert response.status_code == 201, response.text + + _, response = await sanic_client.get(f"/api/data/data_connectors/storage/allow/{project_id}") + + assert response.status_code == 401, response.text + + +@pytest.mark.asyncio +async def test_patch_storage_success( + sanic_client: SanicASGITestClient, create_project, user_headers: dict[str, str], admin_headers: dict[str, str] +) -> None: + project = await create_project(sanic_client, "Test Project") + namespace = f"{project['namespace']}/{project['slug']}" + + project_id = project["id"] + payload = {"project_id": project_id, "max_size": 10} + _, response = await sanic_client.post( + "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload + ) + assert response.status_code == 201, response.text + + payload = {"namespace": namespace, "size": 5, "mount_path": "/data"} + _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) + assert response.status_code == 201, response.text + storage = response.json + storage_id = storage["id"] + original_etag = response.headers["ETag"] + + # Patch the size + headers = merge_headers(user_headers, {"If-Match": original_etag}) + patch = {"size": 8} + _, response = await sanic_client.patch( + f"/api/data/data_connectors/storage/{storage_id}", headers=headers, json=patch + ) + + assert response.status_code == 200, response.text + assert response.json is not None + updated_storage = response.json + assert updated_storage.get("id") == storage_id + assert updated_storage.get("size") == 8 + assert updated_storage.get("mount_path") == "/data" + + +@pytest.mark.asyncio +async def test_patch_storage_mount_path( + sanic_client: SanicASGITestClient, create_project, user_headers: dict[str, str], admin_headers: dict[str, str] +) -> None: + project = await create_project(sanic_client, "Test Project") + namespace = f"{project['namespace']}/{project['slug']}" + + project_id = project["id"] + payload = {"project_id": project_id, "max_size": 10} + _, response = await sanic_client.post( + "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload + ) + assert response.status_code == 201, response.text + + payload = {"namespace": namespace, "size": 10, "mount_path": "/data"} + _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) + assert response.status_code == 201, response.text + storage = response.json + storage_id = storage["id"] + + # Patch the mount path + headers = merge_headers(user_headers, {"If-Match": response.headers["ETag"]}) + patch = {"mount_path": "/new/mount"} + _, response = await sanic_client.patch( + f"/api/data/data_connectors/storage/{storage_id}", headers=headers, json=patch + ) + + assert response.status_code == 200, response.text + assert response.json is not None + updated_storage = response.json + assert updated_storage.get("id") == storage_id + assert updated_storage.get("size") == 10 + assert updated_storage.get("mount_path") == "/new/mount" + + +@pytest.mark.asyncio +async def test_patch_storage_both_fields( + sanic_client: SanicASGITestClient, create_project, user_headers: dict[str, str], admin_headers: dict[str, str] +) -> None: + project = await create_project(sanic_client, "Test Project") + namespace = f"{project['namespace']}/{project['slug']}" + + project_id = project["id"] + payload = {"project_id": project_id, "max_size": 10} + _, response = await sanic_client.post( + "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload + ) + assert response.status_code == 201, response.text + + payload = {"namespace": namespace, "size": 5, "mount_path": "/data"} + _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) + assert response.status_code == 201, response.text + storage = response.json + storage_id = storage["id"] + + # Patch both fields at once + headers = merge_headers(user_headers, {"If-Match": response.headers["ETag"]}) + patch = {"size": 8, "mount_path": "/new/mount"} + _, response = await sanic_client.patch( + f"/api/data/data_connectors/storage/{storage_id}", headers=headers, json=patch + ) + + assert response.status_code == 200, response.text + assert response.json is not None + updated_storage = response.json + assert updated_storage.get("id") == storage_id + assert updated_storage.get("size") == 8 + assert updated_storage.get("mount_path") == "/new/mount" + + +@pytest.mark.asyncio +async def test_patch_storage_without_if_match_header( + sanic_client: SanicASGITestClient, create_project, user_headers: dict[str, str], admin_headers: dict[str, str] +) -> None: + project = await create_project(sanic_client, "Test Project") + namespace = f"{project['namespace']}/{project['slug']}" + + project_id = project["id"] + payload = {"project_id": project_id, "max_size": 10} + _, response = await sanic_client.post( + "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload + ) + assert response.status_code == 201, response.text + + payload = {"namespace": namespace, "size": 10, "mount_path": "/data"} + _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) + assert response.status_code == 201, response.text + storage_id = response.json["id"] + + # Patch without If-Match header + patch = {"size": 20} + _, response = await sanic_client.patch( + f"/api/data/data_connectors/storage/{storage_id}", headers=user_headers, json=patch + ) + + assert response.status_code == 428, response.text + assert "If-Match header not provided" in response.text + + +@pytest.mark.asyncio +async def test_patch_storage_with_invalid_etag( + sanic_client: SanicASGITestClient, create_project, user_headers: dict[str, str], admin_headers: dict[str, str] +) -> None: + project = await create_project(sanic_client, "Test Project") + namespace = f"{project['namespace']}/{project['slug']}" + + project_id = project["id"] + payload = {"project_id": project_id, "max_size": 10} + _, response = await sanic_client.post( + "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload + ) + assert response.status_code == 201, response.text + + payload = {"namespace": namespace, "size": 5, "mount_path": "/data"} + _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) + assert response.status_code == 201, response.text + storage_id = response.json["id"] + correct_etag = response.headers["ETag"] + + # Patch with wrong ETag + headers = merge_headers(user_headers, {"If-Match": "wrong-etag"}) + patch = {"size": 8} + _, response = await sanic_client.patch( + f"/api/data/data_connectors/storage/{storage_id}", headers=headers, json=patch + ) + + assert response.status_code == 409, response.text + + # Verify the etag changed after a successful patch + headers = merge_headers(user_headers, {"If-Match": correct_etag}) + patch = {"size": 6} + _, response = await sanic_client.patch( + f"/api/data/data_connectors/storage/{storage_id}", headers=headers, json=patch + ) + assert response.status_code == 200, response.text + new_etag = response.headers["ETag"] + assert new_etag != correct_etag + + +@pytest.mark.asyncio +async def test_patch_storage_not_found(sanic_client: SanicASGITestClient, user_headers: dict[str, str]) -> None: + from ulid import ULID + + non_existent_id = str(ULID()) + headers = merge_headers(user_headers, {"If-Match": "some-etag"}) + patch = {"size": 20} + _, response = await sanic_client.patch( + f"/api/data/data_connectors/storage/{non_existent_id}", headers=headers, json=patch + ) + + assert response.status_code == 404, response.text + + +@pytest.mark.asyncio +async def test_patch_storage_unauthenticated_fails( + sanic_client: SanicASGITestClient, create_project, admin_headers: dict[str, str], user_headers: dict[str, str] +) -> None: + project = await create_project(sanic_client, "Test Project") + namespace = f"{project['namespace']}/{project['slug']}" + + project_id = project["id"] + payload = {"project_id": project_id, "max_size": 10} + _, response = await sanic_client.post( + "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload + ) + assert response.status_code == 201, response.text + + payload = {"namespace": namespace, "size": 5, "mount_path": "/data"} + _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) + assert response.status_code == 201, response.text + storage_id = response.json["id"] + + # Patch without authentication + _, response = await sanic_client.patch(f"/api/data/data_connectors/storage/{storage_id}", json={"size": 8}) + + assert response.status_code == 401, response.text + + +@pytest.mark.asyncio +async def test_patch_storage_exceeds_max_size( + sanic_client: SanicASGITestClient, create_project, user_headers: dict[str, str], admin_headers: dict[str, str] +) -> None: + project = await create_project(sanic_client, "Test Project") + namespace = f"{project['namespace']}/{project['slug']}" + + project_id = project["id"] + payload = {"project_id": project_id, "max_size": 10} + _, response = await sanic_client.post( + "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload + ) + assert response.status_code == 201, response.text + + payload = {"namespace": namespace, "size": 5, "mount_path": "/data"} + _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) + assert response.status_code == 201, response.text + storage_id = response.json["id"] + + # Try to patch size beyond the allowed max (10GB) + headers = merge_headers(user_headers, {"If-Match": response.headers["ETag"]}) + patch = {"size": 11} + _, response = await sanic_client.patch( + f"/api/data/data_connectors/storage/{storage_id}", headers=headers, json=patch + ) + + assert response.status_code == 422, response.text + + +@pytest.mark.asyncio +async def test_patch_storage_invalid_mount_path( + sanic_client: SanicASGITestClient, create_project, user_headers: dict[str, str], admin_headers: dict[str, str] +) -> None: + project = await create_project(sanic_client, "Test Project") + namespace = f"{project['namespace']}/{project['slug']}" + + project_id = project["id"] + payload = {"project_id": project_id, "max_size": 10} + _, response = await sanic_client.post( + "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload + ) + assert response.status_code == 201, response.text + + payload = {"namespace": namespace, "size": 10, "mount_path": "/data"} + _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) + assert response.status_code == 201, response.text + storage_id = response.json["id"] + + # Try to patch with invalid mount path + headers = merge_headers(user_headers, {"If-Match": response.headers["ETag"]}) + patch = {"mount_path": "/etc/passwd"} + _, response = await sanic_client.patch( + f"/api/data/data_connectors/storage/{storage_id}", headers=headers, json=patch + ) + + assert response.status_code == 422, response.text + + +@pytest.mark.asyncio +async def test_patch_storage_allow_success( + sanic_client: SanicASGITestClient, create_project, admin_headers: dict[str, str], user_headers: dict[str, str] +) -> None: + project = await create_project(sanic_client, "Test Project") + project_id = project["id"] + + payload = {"project_id": project_id, "max_size": 10} + _, response = await sanic_client.post( + "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload + ) + assert response.status_code == 201, response.text + + # Get the allow entry to retrieve the etag (use user_headers as admin may not have read access) + _, response = await sanic_client.get(f"/api/data/data_connectors/storage/allow/{project_id}", headers=user_headers) + assert response.status_code == 200, response.text + etag = response.headers["ETag"] + + # Patch the max_size + headers = merge_headers(admin_headers, {"If-Match": etag}) + patch = {"max_size": 20} + _, response = await sanic_client.patch( + f"/api/data/data_connectors/storage/allow/{project_id}", headers=headers, json=patch + ) + + assert response.status_code == 200, response.text + assert response.json is not None + updated_allow = response.json + assert updated_allow.get("project_id") == project_id + assert updated_allow.get("max_size") == 20 + + +@pytest.mark.asyncio +async def test_patch_storage_allow_requires_admin( + sanic_client: SanicASGITestClient, create_project, admin_headers: dict[str, str], user_headers: dict[str, str] +) -> None: + project = await create_project(sanic_client, "Test Project") + project_id = project["id"] + + payload = {"project_id": project_id, "max_size": 10} + _, response = await sanic_client.post( + "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload + ) + assert response.status_code == 201, response.text + + # Get the allow entry to retrieve the etag (use user_headers as admin may not have read access) + _, response = await sanic_client.get(f"/api/data/data_connectors/storage/allow/{project_id}", headers=admin_headers) + assert response.status_code == 200, response.text + etag = response.headers["ETag"] + + # Try to patch as non-admin + headers = merge_headers(user_headers, {"If-Match": etag}) + patch = {"max_size": 20} + _, response = await sanic_client.patch( + f"/api/data/data_connectors/storage/allow/{project_id}", headers=headers, json=patch + ) + + assert response.status_code == 403, response.text + + +@pytest.mark.asyncio +async def test_patch_storage_allow_without_if_match_header( + sanic_client: SanicASGITestClient, create_project, admin_headers: dict[str, str] +) -> None: + project = await create_project(sanic_client, "Test Project") + project_id = project["id"] + + payload = {"project_id": project_id, "max_size": 10} + _, response = await sanic_client.post( + "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload + ) + assert response.status_code == 201, response.text + + # Patch without If-Match header + patch = {"max_size": 20} + _, response = await sanic_client.patch( + f"/api/data/data_connectors/storage/allow/{project_id}", headers=admin_headers, json=patch + ) + + assert response.status_code == 428, response.text + assert "If-Match header not provided" in response.text + + +@pytest.mark.asyncio +async def test_patch_storage_allow_with_invalid_etag( + sanic_client: SanicASGITestClient, create_project, admin_headers: dict[str, str], user_headers: dict[str, str] +) -> None: + project = await create_project(sanic_client, "Test Project") + project_id = project["id"] + + payload = {"project_id": project_id, "max_size": 10} + _, response = await sanic_client.post( + "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload + ) + assert response.status_code == 201, response.text + + # Get the allow entry to retrieve the etag (use user_headers as admin may not have read access) + _, response = await sanic_client.get(f"/api/data/data_connectors/storage/allow/{project_id}", headers=user_headers) + assert response.status_code == 200, response.text + correct_etag = response.headers["ETag"] + + # Patch with wrong ETag + headers = merge_headers(admin_headers, {"If-Match": "wrong-etag"}) + patch = {"max_size": 20} + _, response = await sanic_client.patch( + f"/api/data/data_connectors/storage/allow/{project_id}", headers=headers, json=patch + ) + + assert response.status_code == 409, response.text + + # Verify the etag changed after a successful patch + headers = merge_headers(admin_headers, {"If-Match": correct_etag}) + patch = {"max_size": 15} + _, response = await sanic_client.patch( + f"/api/data/data_connectors/storage/allow/{project_id}", headers=headers, json=patch + ) + assert response.status_code == 200, response.text + new_etag = response.headers["ETag"] + assert new_etag != correct_etag + + +@pytest.mark.asyncio +async def test_patch_storage_allow_not_in_list( + sanic_client: SanicASGITestClient, create_project, admin_headers: dict[str, str] +) -> None: + project = await create_project(sanic_client, "Test Project") + project_id = project["id"] + + headers = merge_headers(admin_headers, {"If-Match": "some-etag"}) + patch = {"max_size": 20} + _, response = await sanic_client.patch( + f"/api/data/data_connectors/storage/allow/{project_id}", headers=headers, json=patch + ) + + assert response.status_code == 404, response.text + + +@pytest.mark.asyncio +async def test_patch_storage_allow_unauthenticated_fails( + sanic_client: SanicASGITestClient, create_project, admin_headers: dict[str, str] +) -> None: + project = await create_project(sanic_client, "Test Project") + project_id = project["id"] + + payload = {"project_id": project_id, "max_size": 10} + _, response = await sanic_client.post( + "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload + ) + assert response.status_code == 201, response.text + + # Patch without authentication + _, response = await sanic_client.patch( + f"/api/data/data_connectors/storage/allow/{project_id}", json={"max_size": 20} + ) + + assert response.status_code == 401, response.text + + +@pytest.mark.asyncio +async def test_patch_storage_allow_min_size( + sanic_client: SanicASGITestClient, create_project, admin_headers: dict[str, str], user_headers: dict[str, str] +) -> None: + project = await create_project(sanic_client, "Test Project") + project_id = project["id"] + + payload = {"project_id": project_id, "max_size": 10} + _, response = await sanic_client.post( + "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload + ) + assert response.status_code == 201, response.text + + # Get the allow entry to retrieve the etag (use user_headers as admin may not have read access) + _, response = await sanic_client.get(f"/api/data/data_connectors/storage/allow/{project_id}", headers=user_headers) + assert response.status_code == 200, response.text + etag = response.headers["ETag"] + + # Try to set max_size below minimum (1GB) + headers = merge_headers(admin_headers, {"If-Match": etag}) + patch = {"max_size": 0} + _, response = await sanic_client.patch( + f"/api/data/data_connectors/storage/allow/{project_id}", headers=headers, json=patch + ) + + assert response.status_code == 422, response.text + assert "at least 1GB" in response.text + + +@pytest.mark.asyncio +async def test_get_all_storage_allow( + sanic_client: SanicASGITestClient, create_project, admin_headers: dict[str, str], user_headers: dict[str, str] +) -> None: + project = await create_project(sanic_client, "Test Project") + project_id = project["id"] + + payload = {"project_id": project_id, "max_size": 10} + _, response = await sanic_client.post( + "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload + ) + assert response.status_code == 201, response.text + + _, response = await sanic_client.get("/api/data/data_connectors/storage/allow", headers=admin_headers) + assert response.status_code == 200, response.text + assert response.json is not None, f"No json response body: {response.text}" + assert isinstance(response.json, list) + assert len(response.json) == 1 + assert response.json[0]["project_id"] == project_id + assert response.json[0]["max_size"] == 10 diff --git a/test/components/renku_data_services/search/test_reprovision.py b/test/components/renku_data_services/search/test_reprovision.py index e956bf2fc7..a081983b82 100644 --- a/test/components/renku_data_services/search/test_reprovision.py +++ b/test/components/renku_data_services/search/test_reprovision.py @@ -10,7 +10,6 @@ from renku_data_services.authz.models import Visibility from renku_data_services.base_models.core import APIUser, NamespacePath from renku_data_services.base_models.metrics import MetricsService -from renku_data_services.data_connectors.config import ProjectStorageConfig from renku_data_services.data_connectors.db import DataConnectorRepository from renku_data_services.data_connectors.models import ( CloudStorageCore, @@ -26,6 +25,7 @@ from renku_data_services.project.models import Project, UnsavedProject from renku_data_services.search.db import SearchUpdatesRepo from renku_data_services.search.reprovision import SearchReprovision +from renku_data_services.storage.config import ProjectStorageConfig from renku_data_services.users.db import UserRepo admin = APIUser(id="the-admin-1", is_admin=True) diff --git a/test/utils.py b/test/utils.py index 1c0fde1c8e..90c4b61f9e 100644 --- a/test/utils.py +++ b/test/utils.py @@ -37,7 +37,6 @@ from renku_data_services.data_connectors.db import DataConnectorRepository, DataConnectorSecretRepository from renku_data_services.data_connectors.deposits.envidat import EnvidatClient from renku_data_services.data_connectors.deposits.zenodo import ZenodoAPIClient -from renku_data_services.data_connectors.project_storage_k8s import ProjectStorageK8s from renku_data_services.db_config.config import DBConfig from renku_data_services.git.gitlab import DummyGitlabAPI from renku_data_services.k8s.clients import ( @@ -75,6 +74,8 @@ from renku_data_services.session.constants import BUILD_RUN_GVK, TASK_RUN_GVK from renku_data_services.session.db import SessionRepository from renku_data_services.session.k8s_client import ShipwrightClient +from renku_data_services.storage.db import ProjectStorageRepository +from renku_data_services.storage.project_storage_k8s import ProjectStorageK8s from renku_data_services.users import models as user_preferences_models from renku_data_services.users.db import UserPreferencesRepository from renku_data_services.users.db import UserRepo as KcUserRepo @@ -325,7 +326,6 @@ def from_env( project_repo=project_repo, group_repo=group_repo, search_updates_repo=search_updates_repo, - project_storage_config=config.project_storage_config, ) data_connector_secret_repo = DataConnectorSecretRepository( session_maker=config.db.async_session_maker, @@ -334,6 +334,13 @@ def from_env( secret_service_public_key=config.secrets.public_key, authz=authz, ) + project_storage_repo = ProjectStorageRepository( + session_maker=config.db.async_session_maker, + authz=authz, + project_repo=project_repo, + group_repo=group_repo, + project_storage_config=config.project_storage_config, + ) search_reprovisioning = SearchReprovision( search_updates_repo=search_updates_repo, reprovisioning_repo=reprovisioning_repo, @@ -420,6 +427,7 @@ def from_env( internal_token_mint=internal_token_mint, internal_scope_verifier=internal_scope_verifier, project_storage_k8s=project_storage_k8s, + project_storage_repo=project_storage_repo, ) def __post_init__(self) -> None: From 1755f91f566776ac219b4e830b66863e21497eee Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Fri, 31 Jul 2026 10:42:11 +0200 Subject: [PATCH 3/7] Change path to /storage --- .../renku_data_services/storage/blueprints.py | 20 +- .../data_api/__snapshots__/test_storage.ambr | 235 --------------- .../data_api/test_storage.py | 268 ++++++------------ 3 files changed, 94 insertions(+), 429 deletions(-) diff --git a/components/renku_data_services/storage/blueprints.py b/components/renku_data_services/storage/blueprints.py index d5f6e102c8..a86e81341a 100644 --- a/components/renku_data_services/storage/blueprints.py +++ b/components/renku_data_services/storage/blueprints.py @@ -124,7 +124,7 @@ async def _get_one(_: Request, user: base_models.APIUser, storage_id: ULID, etag headers=headers, ) - return "/data_connectors/storage/", ["GET"], _get_one + return "/storage/", ["GET"], _get_one def get_storage_to_project(self) -> BlueprintFactoryResponse: """List all project storage to a given project.""" @@ -153,7 +153,7 @@ async def _get_project_config(_: Request, user: base_models.APIUser) -> JSONResp ) return validated_json(apispec.ProjectStorageConfig, result) - return "/data_connectors/storage/config", ["GET"], _get_project_config + return "/storage/config", ["GET"], _get_project_config def delete_storage(self) -> BlueprintFactoryResponse: """Delete a specific project storage.""" @@ -170,7 +170,7 @@ async def _delete_storage( await self.project_storage_k8s.delete_volume(deleted) return HTTPResponse(status=204) - return "/data_connectors/storage/", ["DELETE"], _delete_storage + return "/storage/", ["DELETE"], _delete_storage def post_storage(self) -> BlueprintFactoryResponse: """Create a new shared project storage.""" @@ -188,7 +188,7 @@ async def _post_storage( apispec.ProjectStorage, self._dump_project_storage(result), headers=headers, status=201 ) - return "/data_connectors/storage", ["POST"], _post_storage + return "/storage", ["POST"], _post_storage def patch_storage(self) -> BlueprintFactoryResponse: """Partially update a project storage entry.""" @@ -219,7 +219,7 @@ async def _patch_storage( headers=headers, ) - return "/data_connectors/storage/", ["PATCH"], _patch_storage + return "/storage/", ["PATCH"], _patch_storage def get_all_storage_allows(self) -> BlueprintFactoryResponse: """List all projects in the storage allow list.""" @@ -246,7 +246,7 @@ async def _get_all_storage_allows( for a in allows ], total - return "/data_connectors/storage/allow", ["GET"], _get_all_storage_allows + return "/storage/allow", ["GET"], _get_all_storage_allows def post_storage_allow(self) -> BlueprintFactoryResponse: """Add a project to the storage allow list.""" @@ -265,7 +265,7 @@ async def _post_storage_allow( status=201, ) - return "/data_connectors/storage/allow", ["POST"], _post_storage_allow + return "/storage/allow", ["POST"], _post_storage_allow def patch_storage_allow(self) -> BlueprintFactoryResponse: """Partially update a project storage allow entry.""" @@ -295,7 +295,7 @@ async def _patch( apispec.ProjectStorageAllow, self._dump_project_storage_allow_detail(pse_update.new), headers=headers ) - return "/data_connectors/storage/allow/", ["PATCH"], _patch + return "/storage/allow/", ["PATCH"], _patch def get_storage_allow(self) -> BlueprintFactoryResponse: """Get the storage allow entry for a project.""" @@ -312,7 +312,7 @@ async def _get_storage_allow(_: Request, user: base_models.APIUser, project_id: apispec.ProjectStorageAllow, self._dump_project_storage_allow_detail(allow), headers=headers ) - return "/data_connectors/storage/allow/", ["GET"], _get_storage_allow + return "/storage/allow/", ["GET"], _get_storage_allow def delete_storage_allow(self) -> BlueprintFactoryResponse: """Remove a project from the storage allow list.""" @@ -325,7 +325,7 @@ async def _delete_storage_allow(_: Request, user: base_models.APIUser, project_i await self.project_storage_k8s.delete_volume(deleted) return HTTPResponse(status=204) - return "/data_connectors/storage/allow/", ["DELETE"], _delete_storage_allow + return "/storage/allow/", ["DELETE"], _delete_storage_allow @staticmethod def _dump_project_storage(ps: models.ProjectStorage) -> apispec.ProjectStorage: diff --git a/test/bases/renku_data_services/data_api/__snapshots__/test_storage.ambr b/test/bases/renku_data_services/data_api/__snapshots__/test_storage.ambr index 51a57c1c12..9835c82953 100644 --- a/test/bases/renku_data_services/data_api/__snapshots__/test_storage.ambr +++ b/test/bases/renku_data_services/data_api/__snapshots__/test_storage.ambr @@ -1,239 +1,4 @@ # serializer version: 1 -# name: test_storage_creation[payload0-201-s3] - dict({ - 'sensitive_fields': list([ - ]), - 'storage': dict({ - 'configuration': dict({ - 'provider': 'AWS', - 'region': 'us-east-1', - 'type': 's3', - }), - 'name': 'mystorage', - 'project_id': '123456', - 'readonly': True, - 'source_path': 'bucket/myfolder', - 'storage_type': 's3', - 'target_path': 'my/target', - }), - }) -# --- -# name: test_storage_creation[payload1-201-s3] - dict({ - 'sensitive_fields': list([ - ]), - 'storage': dict({ - 'configuration': dict({ - 'provider': 'AWS', - 'region': 'us-east-1', - 'type': 's3', - }), - 'name': 'mystorage', - 'project_id': '123456', - 'readonly': True, - 'source_path': 'bucket/myfolder', - 'storage_type': 's3', - 'target_path': 'my/target', - }), - }) -# --- -# name: test_storage_creation[payload10-201-s3] - dict({ - 'sensitive_fields': list([ - dict({ - 'advanced': False, - 'default': '', - 'default_str': '', - 'exclusive': False, - 'help': ''' - AWS Secret Access Key (password). - - Leave blank for anonymous access or runtime credentials. - ''', - 'ispassword': False, - 'name': 'secret_access_key', - 'required': False, - 'sensitive': True, - 'type': 'string', - }), - ]), - 'storage': dict({ - 'configuration': dict({ - 'provider': 'AWS', - 'secret_access_key': '', - 'type': 's3', - }), - 'name': 'mystorage', - 'project_id': '123456', - 'readonly': True, - 'source_path': 'bucket/myfolder', - 'storage_type': 's3', - 'target_path': 'my/target', - }), - }) -# --- -# name: test_storage_creation[payload2-201-s3] - dict({ - 'sensitive_fields': list([ - ]), - 'storage': dict({ - 'configuration': dict({ - 'provider': 'AWS', - 'region': 'us-east-2', - 'type': 's3', - }), - 'name': 'mystorage', - 'project_id': '123456', - 'readonly': True, - 'source_path': 'mybucket/myfolder', - 'storage_type': 's3', - 'target_path': 'my/target', - }), - }) -# --- -# name: test_storage_creation[payload3-201-s3] - dict({ - 'sensitive_fields': list([ - ]), - 'storage': dict({ - 'configuration': dict({ - 'provider': 'AWS', - 'type': 's3', - }), - 'name': 'mystorage', - 'project_id': '123456', - 'readonly': True, - 'source_path': 'giab', - 'storage_type': 's3', - 'target_path': 'my/target', - }), - }) -# --- -# name: test_storage_creation[payload4-201-s3] - dict({ - 'sensitive_fields': list([ - ]), - 'storage': dict({ - 'configuration': dict({ - 'provider': 'AWS', - 'region': 'us-east-2', - 'type': 's3', - }), - 'name': 'mystorage', - 'project_id': '123456', - 'readonly': False, - 'source_path': 'mybucket/myfolder', - 'storage_type': 's3', - 'target_path': 'my/target', - }), - }) -# --- -# name: test_storage_creation[payload5-201-s3] - dict({ - 'sensitive_fields': list([ - ]), - 'storage': dict({ - 'configuration': dict({ - 'endpoint': 'my.provider.com', - 'type': 's3', - }), - 'name': 'mystorage', - 'project_id': '123456', - 'readonly': True, - 'source_path': 'mybucket/myfolder', - 'storage_type': 's3', - 'target_path': 'my/target', - }), - }) -# --- -# name: test_storage_creation[payload6-201-azureblob] - dict({ - 'sensitive_fields': list([ - ]), - 'storage': dict({ - 'configuration': dict({ - 'type': 'azureblob', - }), - 'name': 'mystorage', - 'project_id': '123456', - 'readonly': True, - 'source_path': 'mycontainer/myfolder', - 'storage_type': 'azureblob', - 'target_path': 'my/target', - }), - }) -# --- -# name: test_storage_creation[payload7-201-azureblob] - dict({ - 'sensitive_fields': list([ - ]), - 'storage': dict({ - 'configuration': dict({ - 'account': 'myaccount', - 'type': 'azureblob', - }), - 'name': 'mystorage', - 'project_id': '123456', - 'readonly': True, - 'source_path': 'myfolder', - 'storage_type': 'azureblob', - 'target_path': 'my/target', - }), - }) -# --- -# name: test_storage_creation[payload8-201-azureblob] - dict({ - 'sensitive_fields': list([ - ]), - 'storage': dict({ - 'configuration': dict({ - 'account': 'myaccount', - 'type': 'azureblob', - }), - 'name': 'mystorage', - 'project_id': '123456', - 'readonly': True, - 'source_path': 'myfolder', - 'storage_type': 'azureblob', - 'target_path': 'my/target', - }), - }) -# --- -# name: test_storage_creation[payload9-201-s3] - dict({ - 'sensitive_fields': list([ - dict({ - 'advanced': False, - 'default': '', - 'default_str': '', - 'exclusive': False, - 'help': ''' - AWS Secret Access Key (password). - - Leave blank for anonymous access or runtime credentials. - ''', - 'ispassword': False, - 'name': 'secret_access_key', - 'required': False, - 'sensitive': True, - 'type': 'string', - }), - ]), - 'storage': dict({ - 'configuration': dict({ - 'provider': 'AWS', - 'secret_access_key': '', - 'type': 's3', - }), - 'name': 'mystorage', - 'project_id': '123456', - 'readonly': True, - 'source_path': 'bucket/myfolder', - 'storage_type': 's3', - 'target_path': 'my/target', - }), - }) -# --- # name: test_storage_schema_patches list([ dict({ diff --git a/test/bases/renku_data_services/data_api/test_storage.py b/test/bases/renku_data_services/data_api/test_storage.py index 6d2f349881..7a4292f48b 100644 --- a/test/bases/renku_data_services/data_api/test_storage.py +++ b/test/bases/renku_data_services/data_api/test_storage.py @@ -251,13 +251,11 @@ async def test_post_storage_success( project_id = project["id"] payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) + _, response = await sanic_client.post("/api/data/storage/allow", headers=admin_headers, json=payload) assert response.status_code == 201, response.text payload = {"namespace": namespace, "size": 10, "mount_path": "/data"} - _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) + _, response = await sanic_client.post("/api/data/storage", headers=user_headers, json=payload) assert response.status_code == 201, response.text assert response.json is not None @@ -278,13 +276,11 @@ async def test_post_storage_unauthenticated_fails( project_id = project["id"] payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) + _, response = await sanic_client.post("/api/data/storage/allow", headers=admin_headers, json=payload) assert response.status_code == 201, response.text payload = {"namespace": namespace, "size": 10, "mount_path": "/data"} - _, response = await sanic_client.post("/api/data/data_connectors/storage", json=payload) + _, response = await sanic_client.post("/api/data/storage", json=payload) assert response.status_code == 401, response.text @@ -297,7 +293,7 @@ async def test_post_storage_not_allowed_fails( namespace = f"{project['namespace']}/{project['slug']}" payload = {"namespace": namespace, "size": 10, "mount_path": "/data"} - _, response = await sanic_client.post("/api/data/data_connectors/storage", json=payload, headers=user_headers) + _, response = await sanic_client.post("/api/data/storage", json=payload, headers=user_headers) assert response.status_code == 403, response.text @@ -311,16 +307,14 @@ async def test_post_storage_duplicate_fails( project_id = project["id"] payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) + _, response = await sanic_client.post("/api/data/storage/allow", headers=admin_headers, json=payload) assert response.status == 201 payload = {"namespace": namespace, "size": 10, "mount_path": "/data"} - _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) + _, response = await sanic_client.post("/api/data/storage", headers=user_headers, json=payload) assert response.status_code == 201, response.text - _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) + _, response = await sanic_client.post("/api/data/storage", headers=user_headers, json=payload) assert response.status_code == 422, response.text @@ -334,17 +328,15 @@ async def test_get_one_storage_success( project_id = project["id"] payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) + _, response = await sanic_client.post("/api/data/storage/allow", headers=admin_headers, json=payload) assert response.status_code == 201, response.text payload = {"namespace": namespace, "size": 10, "mount_path": "/data"} - _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) + _, response = await sanic_client.post("/api/data/storage", headers=user_headers, json=payload) assert response.status_code == 201, response.text storage_id = response.json["id"] - _, response = await sanic_client.get(f"/api/data/data_connectors/storage/{storage_id}", headers=user_headers) + _, response = await sanic_client.get(f"/api/data/storage/{storage_id}", headers=user_headers) assert response.status_code == 200, response.text assert response.json is not None @@ -360,7 +352,7 @@ async def test_get_one_storage_not_found(sanic_client: SanicASGITestClient, user from ulid import ULID non_existent_id = str(ULID()) - _, response = await sanic_client.get(f"/api/data/data_connectors/storage/{non_existent_id}", headers=user_headers) + _, response = await sanic_client.get(f"/api/data/storage/{non_existent_id}", headers=user_headers) assert response.status_code == 404, response.text @@ -374,19 +366,17 @@ async def test_get_one_storage_etag( project_id = project["id"] payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) + _, response = await sanic_client.post("/api/data/storage/allow", headers=admin_headers, json=payload) assert response.status_code == 201, response.text payload = {"namespace": namespace, "size": 10, "mount_path": "/data"} - _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) + _, response = await sanic_client.post("/api/data/storage", headers=user_headers, json=payload) assert response.status_code == 201, response.text storage_id = response.json["id"] etag = response.headers["ETag"] headers = merge_headers(user_headers, {"If-None-Match": etag}) - _, response = await sanic_client.get(f"/api/data/data_connectors/storage/{storage_id}", headers=headers) + _, response = await sanic_client.get(f"/api/data/storage/{storage_id}", headers=headers) assert response.status_code == 304, response.text @@ -412,13 +402,11 @@ async def test_get_storage_to_project_success( project_id = project["id"] payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) + _, response = await sanic_client.post("/api/data/storage/allow", headers=admin_headers, json=payload) assert response.status == 201 payload = {"namespace": namespace, "size": 10, "mount_path": "/data"} - _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) + _, response = await sanic_client.post("/api/data/storage", headers=user_headers, json=payload) assert response.status_code == 201, response.text _, response = await sanic_client.get(f"/api/data/projects/{project['id']}/storage", headers=user_headers) @@ -444,17 +432,15 @@ async def test_delete_storage_success( project_id = project["id"] payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) + _, response = await sanic_client.post("/api/data/storage/allow", headers=admin_headers, json=payload) assert response.status_code == 201, response.text payload = {"namespace": namespace, "size": 10, "mount_path": "/data"} - _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) + _, response = await sanic_client.post("/api/data/storage", headers=user_headers, json=payload) assert response.status_code == 201, response.text storage_id = response.json["id"] - _, response = await sanic_client.delete(f"/api/data/data_connectors/storage/{storage_id}", headers=user_headers) + _, response = await sanic_client.delete(f"/api/data/storage/{storage_id}", headers=user_headers) assert response.status_code == 204, response.text _, response = await sanic_client.get(f"/api/data/projects/{project['id']}/storage", headers=user_headers) @@ -471,17 +457,15 @@ async def test_delete_storage_unauthenticated_fails( project_id = project["id"] payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) + _, response = await sanic_client.post("/api/data/storage/allow", headers=admin_headers, json=payload) assert response.status_code == 201, response.text payload = {"namespace": namespace, "size": 10, "mount_path": "/data"} - _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) + _, response = await sanic_client.post("/api/data/storage", headers=user_headers, json=payload) assert response.status_code == 201, response.text storage_id = response.json["id"] - _, response = await sanic_client.delete(f"/api/data/data_connectors/storage/{storage_id}") + _, response = await sanic_client.delete(f"/api/data/storage/{storage_id}") assert response.status_code == 401, response.text @@ -494,9 +478,7 @@ async def test_post_storage_allow_success( project_id = project["id"] payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) + _, response = await sanic_client.post("/api/data/storage/allow", headers=admin_headers, json=payload) assert response.status_code == 201, response.text assert response.json is not None @@ -512,7 +494,7 @@ async def test_post_storage_allow_requires_admin( project = await create_project(sanic_client, "Test Project") payload = {"project_id": project["id"], "max_size": 10} - _, response = await sanic_client.post("/api/data/data_connectors/storage/allow", headers=user_headers, json=payload) + _, response = await sanic_client.post("/api/data/storage/allow", headers=user_headers, json=payload) assert response.status_code == 403, response.text @@ -524,7 +506,7 @@ async def test_post_storage_allow_unauthenticated_fails( project = await create_project(sanic_client, "Test Project") payload = {"project_id": project["id"], "max_size": 10} - _, response = await sanic_client.post("/api/data/data_connectors/storage/allow", json=payload) + _, response = await sanic_client.post("/api/data/storage/allow", json=payload) assert response.status_code == 401, response.text @@ -537,14 +519,10 @@ async def test_post_storage_allow_duplicate_fails( project_id = project["id"] payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) + _, response = await sanic_client.post("/api/data/storage/allow", headers=admin_headers, json=payload) assert response.status_code == 201, response.text - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) + _, response = await sanic_client.post("/api/data/storage/allow", headers=admin_headers, json=payload) assert response.status_code == 422, response.text @@ -557,21 +535,15 @@ async def test_delete_storage_allow_success( project_id = project["id"] payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) + _, response = await sanic_client.post("/api/data/storage/allow", headers=admin_headers, json=payload) assert response.status_code == 201, response.text - _, response = await sanic_client.delete( - f"/api/data/data_connectors/storage/allow/{project_id}", headers=admin_headers - ) + _, response = await sanic_client.delete(f"/api/data/storage/allow/{project_id}", headers=admin_headers) assert response.status_code == 204, response.text # Re-adding after deletion should succeed - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) + _, response = await sanic_client.post("/api/data/storage/allow", headers=admin_headers, json=payload) assert response.status_code == 201, response.text @@ -583,14 +555,10 @@ async def test_delete_storage_allow_requires_admin( project_id = project["id"] payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) + _, response = await sanic_client.post("/api/data/storage/allow", headers=admin_headers, json=payload) assert response.status_code == 201, response.text - _, response = await sanic_client.delete( - f"/api/data/data_connectors/storage/allow/{project_id}", headers=user_headers - ) + _, response = await sanic_client.delete(f"/api/data/storage/allow/{project_id}", headers=user_headers) assert response.status_code == 403, response.text @@ -603,12 +571,10 @@ async def test_get_storage_allow_success( project_id = project["id"] payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) + _, response = await sanic_client.post("/api/data/storage/allow", headers=admin_headers, json=payload) assert response.status_code == 201, response.text - _, response = await sanic_client.get(f"/api/data/data_connectors/storage/allow/{project_id}", headers=user_headers) + _, response = await sanic_client.get(f"/api/data/storage/allow/{project_id}", headers=user_headers) assert response.status_code == 200, response.text assert response.json is not None @@ -623,7 +589,7 @@ async def test_get_storage_allow_not_in_list( project = await create_project(sanic_client, "Test Project") project_id = project["id"] - _, response = await sanic_client.get(f"/api/data/data_connectors/storage/allow/{project_id}", headers=user_headers) + _, response = await sanic_client.get(f"/api/data/storage/allow/{project_id}", headers=user_headers) assert response.status_code == 404, response.text @@ -636,12 +602,10 @@ async def test_get_storage_allow_unauthenticated( project_id = project["id"] payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) + _, response = await sanic_client.post("/api/data/storage/allow", headers=admin_headers, json=payload) assert response.status_code == 201, response.text - _, response = await sanic_client.get(f"/api/data/data_connectors/storage/allow/{project_id}") + _, response = await sanic_client.get(f"/api/data/storage/allow/{project_id}") assert response.status_code == 401, response.text @@ -655,13 +619,11 @@ async def test_patch_storage_success( project_id = project["id"] payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) + _, response = await sanic_client.post("/api/data/storage/allow", headers=admin_headers, json=payload) assert response.status_code == 201, response.text payload = {"namespace": namespace, "size": 5, "mount_path": "/data"} - _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) + _, response = await sanic_client.post("/api/data/storage", headers=user_headers, json=payload) assert response.status_code == 201, response.text storage = response.json storage_id = storage["id"] @@ -670,9 +632,7 @@ async def test_patch_storage_success( # Patch the size headers = merge_headers(user_headers, {"If-Match": original_etag}) patch = {"size": 8} - _, response = await sanic_client.patch( - f"/api/data/data_connectors/storage/{storage_id}", headers=headers, json=patch - ) + _, response = await sanic_client.patch(f"/api/data/storage/{storage_id}", headers=headers, json=patch) assert response.status_code == 200, response.text assert response.json is not None @@ -691,13 +651,11 @@ async def test_patch_storage_mount_path( project_id = project["id"] payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) + _, response = await sanic_client.post("/api/data/storage/allow", headers=admin_headers, json=payload) assert response.status_code == 201, response.text payload = {"namespace": namespace, "size": 10, "mount_path": "/data"} - _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) + _, response = await sanic_client.post("/api/data/storage", headers=user_headers, json=payload) assert response.status_code == 201, response.text storage = response.json storage_id = storage["id"] @@ -705,9 +663,7 @@ async def test_patch_storage_mount_path( # Patch the mount path headers = merge_headers(user_headers, {"If-Match": response.headers["ETag"]}) patch = {"mount_path": "/new/mount"} - _, response = await sanic_client.patch( - f"/api/data/data_connectors/storage/{storage_id}", headers=headers, json=patch - ) + _, response = await sanic_client.patch(f"/api/data/storage/{storage_id}", headers=headers, json=patch) assert response.status_code == 200, response.text assert response.json is not None @@ -726,13 +682,11 @@ async def test_patch_storage_both_fields( project_id = project["id"] payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) + _, response = await sanic_client.post("/api/data/storage/allow", headers=admin_headers, json=payload) assert response.status_code == 201, response.text payload = {"namespace": namespace, "size": 5, "mount_path": "/data"} - _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) + _, response = await sanic_client.post("/api/data/storage", headers=user_headers, json=payload) assert response.status_code == 201, response.text storage = response.json storage_id = storage["id"] @@ -740,9 +694,7 @@ async def test_patch_storage_both_fields( # Patch both fields at once headers = merge_headers(user_headers, {"If-Match": response.headers["ETag"]}) patch = {"size": 8, "mount_path": "/new/mount"} - _, response = await sanic_client.patch( - f"/api/data/data_connectors/storage/{storage_id}", headers=headers, json=patch - ) + _, response = await sanic_client.patch(f"/api/data/storage/{storage_id}", headers=headers, json=patch) assert response.status_code == 200, response.text assert response.json is not None @@ -761,21 +713,17 @@ async def test_patch_storage_without_if_match_header( project_id = project["id"] payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) + _, response = await sanic_client.post("/api/data/storage/allow", headers=admin_headers, json=payload) assert response.status_code == 201, response.text payload = {"namespace": namespace, "size": 10, "mount_path": "/data"} - _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) + _, response = await sanic_client.post("/api/data/storage", headers=user_headers, json=payload) assert response.status_code == 201, response.text storage_id = response.json["id"] # Patch without If-Match header patch = {"size": 20} - _, response = await sanic_client.patch( - f"/api/data/data_connectors/storage/{storage_id}", headers=user_headers, json=patch - ) + _, response = await sanic_client.patch(f"/api/data/storage/{storage_id}", headers=user_headers, json=patch) assert response.status_code == 428, response.text assert "If-Match header not provided" in response.text @@ -790,13 +738,11 @@ async def test_patch_storage_with_invalid_etag( project_id = project["id"] payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) + _, response = await sanic_client.post("/api/data/storage/allow", headers=admin_headers, json=payload) assert response.status_code == 201, response.text payload = {"namespace": namespace, "size": 5, "mount_path": "/data"} - _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) + _, response = await sanic_client.post("/api/data/storage", headers=user_headers, json=payload) assert response.status_code == 201, response.text storage_id = response.json["id"] correct_etag = response.headers["ETag"] @@ -804,18 +750,14 @@ async def test_patch_storage_with_invalid_etag( # Patch with wrong ETag headers = merge_headers(user_headers, {"If-Match": "wrong-etag"}) patch = {"size": 8} - _, response = await sanic_client.patch( - f"/api/data/data_connectors/storage/{storage_id}", headers=headers, json=patch - ) + _, response = await sanic_client.patch(f"/api/data/storage/{storage_id}", headers=headers, json=patch) assert response.status_code == 409, response.text # Verify the etag changed after a successful patch headers = merge_headers(user_headers, {"If-Match": correct_etag}) patch = {"size": 6} - _, response = await sanic_client.patch( - f"/api/data/data_connectors/storage/{storage_id}", headers=headers, json=patch - ) + _, response = await sanic_client.patch(f"/api/data/storage/{storage_id}", headers=headers, json=patch) assert response.status_code == 200, response.text new_etag = response.headers["ETag"] assert new_etag != correct_etag @@ -828,9 +770,7 @@ async def test_patch_storage_not_found(sanic_client: SanicASGITestClient, user_h non_existent_id = str(ULID()) headers = merge_headers(user_headers, {"If-Match": "some-etag"}) patch = {"size": 20} - _, response = await sanic_client.patch( - f"/api/data/data_connectors/storage/{non_existent_id}", headers=headers, json=patch - ) + _, response = await sanic_client.patch(f"/api/data/storage/{non_existent_id}", headers=headers, json=patch) assert response.status_code == 404, response.text @@ -844,18 +784,16 @@ async def test_patch_storage_unauthenticated_fails( project_id = project["id"] payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) + _, response = await sanic_client.post("/api/data/storage/allow", headers=admin_headers, json=payload) assert response.status_code == 201, response.text payload = {"namespace": namespace, "size": 5, "mount_path": "/data"} - _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) + _, response = await sanic_client.post("/api/data/storage", headers=user_headers, json=payload) assert response.status_code == 201, response.text storage_id = response.json["id"] # Patch without authentication - _, response = await sanic_client.patch(f"/api/data/data_connectors/storage/{storage_id}", json={"size": 8}) + _, response = await sanic_client.patch(f"/api/data/storage/{storage_id}", json={"size": 8}) assert response.status_code == 401, response.text @@ -869,22 +807,18 @@ async def test_patch_storage_exceeds_max_size( project_id = project["id"] payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) + _, response = await sanic_client.post("/api/data/storage/allow", headers=admin_headers, json=payload) assert response.status_code == 201, response.text payload = {"namespace": namespace, "size": 5, "mount_path": "/data"} - _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) + _, response = await sanic_client.post("/api/data/storage", headers=user_headers, json=payload) assert response.status_code == 201, response.text storage_id = response.json["id"] # Try to patch size beyond the allowed max (10GB) headers = merge_headers(user_headers, {"If-Match": response.headers["ETag"]}) patch = {"size": 11} - _, response = await sanic_client.patch( - f"/api/data/data_connectors/storage/{storage_id}", headers=headers, json=patch - ) + _, response = await sanic_client.patch(f"/api/data/storage/{storage_id}", headers=headers, json=patch) assert response.status_code == 422, response.text @@ -898,22 +832,18 @@ async def test_patch_storage_invalid_mount_path( project_id = project["id"] payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) + _, response = await sanic_client.post("/api/data/storage/allow", headers=admin_headers, json=payload) assert response.status_code == 201, response.text payload = {"namespace": namespace, "size": 10, "mount_path": "/data"} - _, response = await sanic_client.post("/api/data/data_connectors/storage", headers=user_headers, json=payload) + _, response = await sanic_client.post("/api/data/storage", headers=user_headers, json=payload) assert response.status_code == 201, response.text storage_id = response.json["id"] # Try to patch with invalid mount path headers = merge_headers(user_headers, {"If-Match": response.headers["ETag"]}) patch = {"mount_path": "/etc/passwd"} - _, response = await sanic_client.patch( - f"/api/data/data_connectors/storage/{storage_id}", headers=headers, json=patch - ) + _, response = await sanic_client.patch(f"/api/data/storage/{storage_id}", headers=headers, json=patch) assert response.status_code == 422, response.text @@ -926,22 +856,18 @@ async def test_patch_storage_allow_success( project_id = project["id"] payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) + _, response = await sanic_client.post("/api/data/storage/allow", headers=admin_headers, json=payload) assert response.status_code == 201, response.text # Get the allow entry to retrieve the etag (use user_headers as admin may not have read access) - _, response = await sanic_client.get(f"/api/data/data_connectors/storage/allow/{project_id}", headers=user_headers) + _, response = await sanic_client.get(f"/api/data/storage/allow/{project_id}", headers=user_headers) assert response.status_code == 200, response.text etag = response.headers["ETag"] # Patch the max_size headers = merge_headers(admin_headers, {"If-Match": etag}) patch = {"max_size": 20} - _, response = await sanic_client.patch( - f"/api/data/data_connectors/storage/allow/{project_id}", headers=headers, json=patch - ) + _, response = await sanic_client.patch(f"/api/data/storage/allow/{project_id}", headers=headers, json=patch) assert response.status_code == 200, response.text assert response.json is not None @@ -958,22 +884,18 @@ async def test_patch_storage_allow_requires_admin( project_id = project["id"] payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) + _, response = await sanic_client.post("/api/data/storage/allow", headers=admin_headers, json=payload) assert response.status_code == 201, response.text # Get the allow entry to retrieve the etag (use user_headers as admin may not have read access) - _, response = await sanic_client.get(f"/api/data/data_connectors/storage/allow/{project_id}", headers=admin_headers) + _, response = await sanic_client.get(f"/api/data/storage/allow/{project_id}", headers=admin_headers) assert response.status_code == 200, response.text etag = response.headers["ETag"] # Try to patch as non-admin headers = merge_headers(user_headers, {"If-Match": etag}) patch = {"max_size": 20} - _, response = await sanic_client.patch( - f"/api/data/data_connectors/storage/allow/{project_id}", headers=headers, json=patch - ) + _, response = await sanic_client.patch(f"/api/data/storage/allow/{project_id}", headers=headers, json=patch) assert response.status_code == 403, response.text @@ -986,16 +908,12 @@ async def test_patch_storage_allow_without_if_match_header( project_id = project["id"] payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) + _, response = await sanic_client.post("/api/data/storage/allow", headers=admin_headers, json=payload) assert response.status_code == 201, response.text # Patch without If-Match header patch = {"max_size": 20} - _, response = await sanic_client.patch( - f"/api/data/data_connectors/storage/allow/{project_id}", headers=admin_headers, json=patch - ) + _, response = await sanic_client.patch(f"/api/data/storage/allow/{project_id}", headers=admin_headers, json=patch) assert response.status_code == 428, response.text assert "If-Match header not provided" in response.text @@ -1009,31 +927,25 @@ async def test_patch_storage_allow_with_invalid_etag( project_id = project["id"] payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) + _, response = await sanic_client.post("/api/data/storage/allow", headers=admin_headers, json=payload) assert response.status_code == 201, response.text # Get the allow entry to retrieve the etag (use user_headers as admin may not have read access) - _, response = await sanic_client.get(f"/api/data/data_connectors/storage/allow/{project_id}", headers=user_headers) + _, response = await sanic_client.get(f"/api/data/storage/allow/{project_id}", headers=user_headers) assert response.status_code == 200, response.text correct_etag = response.headers["ETag"] # Patch with wrong ETag headers = merge_headers(admin_headers, {"If-Match": "wrong-etag"}) patch = {"max_size": 20} - _, response = await sanic_client.patch( - f"/api/data/data_connectors/storage/allow/{project_id}", headers=headers, json=patch - ) + _, response = await sanic_client.patch(f"/api/data/storage/allow/{project_id}", headers=headers, json=patch) assert response.status_code == 409, response.text # Verify the etag changed after a successful patch headers = merge_headers(admin_headers, {"If-Match": correct_etag}) patch = {"max_size": 15} - _, response = await sanic_client.patch( - f"/api/data/data_connectors/storage/allow/{project_id}", headers=headers, json=patch - ) + _, response = await sanic_client.patch(f"/api/data/storage/allow/{project_id}", headers=headers, json=patch) assert response.status_code == 200, response.text new_etag = response.headers["ETag"] assert new_etag != correct_etag @@ -1048,9 +960,7 @@ async def test_patch_storage_allow_not_in_list( headers = merge_headers(admin_headers, {"If-Match": "some-etag"}) patch = {"max_size": 20} - _, response = await sanic_client.patch( - f"/api/data/data_connectors/storage/allow/{project_id}", headers=headers, json=patch - ) + _, response = await sanic_client.patch(f"/api/data/storage/allow/{project_id}", headers=headers, json=patch) assert response.status_code == 404, response.text @@ -1063,15 +973,11 @@ async def test_patch_storage_allow_unauthenticated_fails( project_id = project["id"] payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) + _, response = await sanic_client.post("/api/data/storage/allow", headers=admin_headers, json=payload) assert response.status_code == 201, response.text # Patch without authentication - _, response = await sanic_client.patch( - f"/api/data/data_connectors/storage/allow/{project_id}", json={"max_size": 20} - ) + _, response = await sanic_client.patch(f"/api/data/storage/allow/{project_id}", json={"max_size": 20}) assert response.status_code == 401, response.text @@ -1084,22 +990,18 @@ async def test_patch_storage_allow_min_size( project_id = project["id"] payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) + _, response = await sanic_client.post("/api/data/storage/allow", headers=admin_headers, json=payload) assert response.status_code == 201, response.text # Get the allow entry to retrieve the etag (use user_headers as admin may not have read access) - _, response = await sanic_client.get(f"/api/data/data_connectors/storage/allow/{project_id}", headers=user_headers) + _, response = await sanic_client.get(f"/api/data/storage/allow/{project_id}", headers=user_headers) assert response.status_code == 200, response.text etag = response.headers["ETag"] # Try to set max_size below minimum (1GB) headers = merge_headers(admin_headers, {"If-Match": etag}) patch = {"max_size": 0} - _, response = await sanic_client.patch( - f"/api/data/data_connectors/storage/allow/{project_id}", headers=headers, json=patch - ) + _, response = await sanic_client.patch(f"/api/data/storage/allow/{project_id}", headers=headers, json=patch) assert response.status_code == 422, response.text assert "at least 1GB" in response.text @@ -1113,12 +1015,10 @@ async def test_get_all_storage_allow( project_id = project["id"] payload = {"project_id": project_id, "max_size": 10} - _, response = await sanic_client.post( - "/api/data/data_connectors/storage/allow", headers=admin_headers, json=payload - ) + _, response = await sanic_client.post("/api/data/storage/allow", headers=admin_headers, json=payload) assert response.status_code == 201, response.text - _, response = await sanic_client.get("/api/data/data_connectors/storage/allow", headers=admin_headers) + _, response = await sanic_client.get("/api/data/storage/allow", headers=admin_headers) assert response.status_code == 200, response.text assert response.json is not None, f"No json response body: {response.text}" assert isinstance(response.json, list) From cf5125344de8c5119ec85ae8741a57a42773bcda Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Fri, 31 Jul 2026 11:40:38 +0200 Subject: [PATCH 4/7] Fix alembic migration sequence --- .../migrations/versions/fa511acd4581_add_project_storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/renku_data_services/migrations/versions/fa511acd4581_add_project_storage.py b/components/renku_data_services/migrations/versions/fa511acd4581_add_project_storage.py index 5b9a5ae5de..395e8df238 100644 --- a/components/renku_data_services/migrations/versions/fa511acd4581_add_project_storage.py +++ b/components/renku_data_services/migrations/versions/fa511acd4581_add_project_storage.py @@ -13,7 +13,7 @@ # revision identifiers, used by Alembic. revision = "fa511acd4581" -down_revision = "01k4dy9r2we4" +down_revision = "36435401b2e7" branch_labels = None depends_on = None From 7b9d4bf35c0dc6b8d2cb2ce3c81c4faaa27507f8 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Fri, 31 Jul 2026 12:05:12 +0200 Subject: [PATCH 5/7] register storage orms --- components/renku_data_services/migrations/env.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/renku_data_services/migrations/env.py b/components/renku_data_services/migrations/env.py index ebe4f15665..c4807e42bc 100644 --- a/components/renku_data_services/migrations/env.py +++ b/components/renku_data_services/migrations/env.py @@ -17,6 +17,7 @@ from renku_data_services.search.orm import BaseORM as search from renku_data_services.secrets.orm import BaseORM as secrets from renku_data_services.session.orm import BaseORM as sessions +from renku_data_services.storage.orm import BaseORM as storage from renku_data_services.users.orm import BaseORM as users all_metadata = [ @@ -35,6 +36,7 @@ search.metadata, secrets.metadata, sessions.metadata, + storage.metadata, users.metadata, resource_usage.metadata, ] From 00a37a78f6a7ce2267e85bc224a89c6b02c02e98 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Fri, 31 Jul 2026 13:12:42 +0200 Subject: [PATCH 6/7] Fix type error --- test/components/renku_data_services/search/test_reprovision.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/components/renku_data_services/search/test_reprovision.py b/test/components/renku_data_services/search/test_reprovision.py index a081983b82..96cb030806 100644 --- a/test/components/renku_data_services/search/test_reprovision.py +++ b/test/components/renku_data_services/search/test_reprovision.py @@ -25,7 +25,6 @@ from renku_data_services.project.models import Project, UnsavedProject from renku_data_services.search.db import SearchUpdatesRepo from renku_data_services.search.reprovision import SearchReprovision -from renku_data_services.storage.config import ProjectStorageConfig from renku_data_services.users.db import UserRepo admin = APIUser(id="the-admin-1", is_admin=True) @@ -55,7 +54,7 @@ def make_setup(app_manager_instance, solr_config) -> Setup: gr = GroupRepository(sess, authz, search_updates) ur = UserRepo(sess, gr, search_updates, None, MagicMock(spec=MetricsService), authz) pr = ProjectRepository(sess, gr, search_updates, authz) - dcr = DataConnectorRepository(sess, authz, pr, gr, search_updates, ProjectStorageConfig.from_env()) + dcr = DataConnectorRepository(sess, authz, pr, gr, search_updates) sr = SearchReprovision( search_updates_repo=search_updates, reprovisioning_repo=ReprovisioningRepository(sess), From bb241ca805a810b019ea2f65b513e37a0d0b0d10 Mon Sep 17 00:00:00 2001 From: Eike Kettner Date: Fri, 31 Jul 2026 15:20:53 +0200 Subject: [PATCH 7/7] Change api tags to storage --- .../renku_data_services/storage/api.spec.yaml | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/components/renku_data_services/storage/api.spec.yaml b/components/renku_data_services/storage/api.spec.yaml index 0c296970d7..4163d6dc61 100644 --- a/components/renku_data_services/storage/api.spec.yaml +++ b/components/renku_data_services/storage/api.spec.yaml @@ -109,7 +109,7 @@ paths: default: $ref: "#/components/responses/Error" tags: - - data_connectors + - storage /data_connectors/storage/config: get: summary: Get the current configuration for project storage @@ -123,7 +123,7 @@ paths: default: $ref: "#/components/responses/Error" tags: - - data_connectors + - storage /data_connectors/storage/allow: get: summary: List all projects in the storage allow list @@ -166,7 +166,7 @@ paths: default: $ref: "#/components/responses/Error" tags: - - data_connectors + - storage post: summary: Add a project to the storage allow list requestBody: @@ -185,7 +185,7 @@ paths: default: $ref: "#/components/responses/Error" tags: - - data_connectors + - storage /data_connectors/storage/allow/{project_id}: parameters: - in: path @@ -211,7 +211,7 @@ paths: default: $ref: "#/components/responses/Error" tags: - - data_connectors + - storage patch: summary: Change the maximum size for a project storage parameters: @@ -238,7 +238,7 @@ paths: default: $ref: "#/components/responses/Error" tags: - - data_connectors + - storage delete: summary: Remove a project from the storage allow list responses: @@ -247,7 +247,7 @@ paths: default: $ref: "#/components/responses/Error" tags: - - data_connectors + - storage /data_connectors/storage/{storage_id}: parameters: - in: path @@ -273,7 +273,7 @@ paths: default: $ref: "#/components/responses/Error" tags: - - data_connectors + - storage patch: summary: Change the size or mount path of a project storage parameters: @@ -300,7 +300,7 @@ paths: default: $ref: "#/components/responses/Error" tags: - - data_connectors + - storage delete: summary: Delete a specific project storage responses: @@ -309,7 +309,7 @@ paths: default: $ref: "#/components/responses/Error" tags: - - data_connectors + - storage /projects/{project_id}/storage: parameters: - in: path