Moved more code

Author: eikek

bases/renku_data_services/data_api/app.py

@@ -38,7 +38,7 @@
 from renku_data_services.search.reprovision import SearchReprovision
 from renku_data_services.search.solr_user_query import UsernameResolve
 from renku_data_services.session.blueprints import BuildsBP, EnvironmentsBP, SessionLaunchersBP
-from renku_data_services.storage.blueprints import StorageSchemaBP
+from renku_data_services.storage.blueprints import ProjectStorageBP, StorageSchemaBP
 from renku_data_services.users.blueprints import KCUsersBP, UserPreferencesBP, UserSecretsBP
 
 
@@ -222,6 +222,7 @@ def register_all_handlers(app: Sanic, dm: DependencyManager) -> Sanic:
         oauth_client_factory=dm.oauth_http_client_factory,
         project_repo=dm.project_repo,
         project_session_secret_repo=dm.project_session_secret_repo,
+        project_storage_repo=dm.project_storage_repo,
         rp_repo=dm.rp_repo,
         session_repo=dm.session_repo,
         user_repo=dm.kc_user_repo,
@@ -280,7 +281,6 @@ def register_all_handlers(app: Sanic, dm: DependencyManager) -> Sanic:
         data_service_base_url=dm.config.nb_config.data_service_url,
         k8s_client=dm.k8s_client,
         deposit_config=dm.config.deposit_config,
-        project_storage_k8s=dm.project_storage_k8s,
     )
     notifications = NotificationsBP(
         name="notifications",
@@ -311,6 +311,13 @@ def register_all_handlers(app: Sanic, dm: DependencyManager) -> Sanic:
         internal_token_mint=dm.internal_token_mint,
         internal_scope_verifier=dm.internal_scope_verifier,
     )
+    project_storage = ProjectStorageBP(
+        name="project_storage",
+        url_prefix=url_prefix,
+        project_storage_k8s=dm.project_storage_k8s,
+        project_storage_repo=dm.project_storage_repo,
+        authenticator=dm.authenticator,
+    )
     app.blueprint(
         [
             resource_pools.blueprint(),
@@ -342,6 +349,7 @@ def register_all_handlers(app: Sanic, dm: DependencyManager) -> Sanic:
             capacity_reservation.blueprint(),
             resource_usage.blueprint(),
             internal_authentication.blueprint(),
+            project_storage.blueprint(),
         ]
     )
     if builds is not None:

bases/renku_data_services/data_api/config.py

@@ -14,12 +14,13 @@
 )
 from renku_data_services.app_config.logging import Config as LoggingConfig
 from renku_data_services.authz.config import AuthzConfig
-from renku_data_services.data_connectors.config import DepositConfig, ProjectStorageConfig
+from renku_data_services.data_connectors.config import DepositConfig
 from renku_data_services.db_config.config import DBConfig
 from renku_data_services.notebooks.config import NotebooksConfig
 from renku_data_services.secrets.config import PublicSecretsConfig
 from renku_data_services.session.config import BuildsConfig
 from renku_data_services.solr.solr_client import SolrClientConfig
+from renku_data_services.storage.config import ProjectStorageConfig
 from renku_data_services.users.config import UserPreferencesConfig
 
 

bases/renku_data_services/data_api/dependencies.py

@@ -40,7 +40,6 @@
 )
 from renku_data_services.data_connectors.deposits.envidat import EnvidatClient
 from renku_data_services.data_connectors.deposits.zenodo import ZenodoAPIClient
-from renku_data_services.data_connectors.project_storage_k8s import ProjectStorageK8s
 from renku_data_services.git.gitlab import DummyGitlabAPI, EmptyGitlabAPI, GitlabAPI
 from renku_data_services.k8s.client_interfaces import K8sClient
 from renku_data_services.k8s.clients import (
@@ -79,6 +78,8 @@
 from renku_data_services.session.constants import BUILD_RUN_GVK, TASK_RUN_GVK
 from renku_data_services.session.db import SessionRepository
 from renku_data_services.session.k8s_client import ShipwrightClient
+from renku_data_services.storage.db import ProjectStorageRepository
+from renku_data_services.storage.project_storage_k8s import ProjectStorageK8s
 from renku_data_services.users.db import UserPreferencesRepository
 from renku_data_services.users.db import UserRepo as KcUserRepo
 from renku_data_services.users.dummy_kc_api import DummyKeycloakAPI
@@ -177,6 +178,7 @@ class DependencyManager:
     internal_token_mint: RenkuSelfTokenMint
     internal_scope_verifier: ScopeVerifier
     project_storage_k8s: ProjectStorageK8s
+    project_storage_repo: ProjectStorageRepository
 
     spec: dict[str, Any] = field(init=False, repr=False, default_factory=dict)
     app_name: str = "renku_data_services"
@@ -367,6 +369,13 @@ def from_env(cls) -> DependencyManager:
             member_repo=member_repo,
         )
         project_storage_k8s = ProjectStorageK8s(config.nb_config.k8s_v2_client)
+        project_storage_repo = ProjectStorageRepository(
+            session_maker=config.db.async_session_maker,
+            authz=authz,
+            project_repo=project_repo,
+            group_repo=group_repo,
+            project_storage_config=config.project_storage_config,
+        )
         reprovisioning_repo = ReprovisioningRepository(session_maker=config.db.async_session_maker)
 
         git_repositories_repo = GitRepositoriesRepository(
@@ -423,7 +432,6 @@ def from_env(cls) -> DependencyManager:
             project_repo=project_repo,
             group_repo=group_repo,
             search_updates_repo=search_updates_repo,
-            project_storage_config=config.project_storage_config,
         )
         data_connector_secret_repo = DataConnectorSecretRepository(
             session_maker=config.db.async_session_maker,
@@ -518,4 +526,5 @@ def from_env(cls) -> DependencyManager:
             internal_token_mint=internal_token_mint,
             internal_scope_verifier=internal_scope_verifier,
             project_storage_k8s=project_storage_k8s,
+            project_storage_repo=project_storage_repo,
         )

components/renku_data_services/data_connectors/blueprints.py

@@ -11,7 +11,6 @@
 from renku_data_services import base_models, errors
 from renku_data_services.base_api.auth import (
     authenticate,
-    only_admins,
     only_authenticated,
 )
 from renku_data_services.base_api.blueprint import BlueprintFactoryResponse, CustomBlueprint
@@ -54,7 +53,6 @@
 )
 from renku_data_services.data_connectors.deposits.envidat import EnvidatClient
 from renku_data_services.data_connectors.deposits.zenodo import ZenodoAPIClient
-from renku_data_services.data_connectors.project_storage_k8s import ProjectStorageK8s
 from renku_data_services.k8s.client_interfaces import K8sClient, SecretClient
 from renku_data_services.k8s.clients import DepositUploadJobClient
 from renku_data_services.notebooks.data_sources import DataSourceRepository
@@ -79,7 +77,6 @@ class DataConnectorsBP(CustomBlueprint):
     data_service_base_url: str
     k8s_client: K8sClient
     deposit_config: DepositConfig
-    project_storage_k8s: ProjectStorageK8s
 
     def get_all(self) -> BlueprintFactoryResponse:
         """List data connectors."""
@@ -399,52 +396,6 @@ async def _delete_project_link(
             _delete_project_link,
         )
 
-    def get_storage_to_project(self) -> BlueprintFactoryResponse:
-        """List all project storage to a given project."""
-
-        @authenticate(self.authenticator)
-        async def _get_all_storage_to_project(
-            _: Request,
-            user: base_models.APIUser,
-            project_id: ULID,
-        ) -> JSONResponse:
-            project_storage = await self.data_connector_repo.get_storage_to(user=user, project_id=project_id)
-            result = [self._dump_project_storage(project_storage)] if project_storage else []
-            return validated_json(apispec.ProjectStorageList, result)
-
-        return "/projects/<project_id:ulid>/storage", ["GET"], _get_all_storage_to_project
-
-    def get_storage_config(self) -> BlueprintFactoryResponse:
-        """Get the current config used for project storage."""
-
-        @authenticate(self.authenticator)
-        @only_admins
-        async def _get_project_config(_: Request, user: base_models.APIUser) -> JSONResponse:
-            storage_config = self.data_connector_repo.get_project_storage_config()
-            result = apispec.ProjectStorageConfig(
-                enabled=storage_config.enabled, max_size=int(storage_config.maximum_size.to_gibi())
-            )
-            return validated_json(apispec.ProjectStorageConfig, result)
-
-        return "/data_connectors/storage/config", ["GET"], _get_project_config
-
-    def delete_storage(self) -> BlueprintFactoryResponse:
-        """Delete a specific project storage."""
-
-        @authenticate(self.authenticator)
-        @only_authenticated
-        async def _delete_storage(
-            _: Request,
-            user: base_models.APIUser,
-            storage_id: ULID,
-        ) -> HTTPResponse:
-            deleted = await self.data_connector_repo.delete_project_storage(user=user, storage_id=storage_id)
-            if deleted:
-                await self.project_storage_k8s.delete_volume(deleted)
-            return HTTPResponse(status=204)
-
-        return "/data_connectors/storage/<storage_id:ulid>", ["DELETE"], _delete_storage
-
     def get_all_data_connectors_links_to_project(self) -> BlueprintFactoryResponse:
         """List all links from data connectors to a given project."""
 

components/renku_data_services/data_connectors/db.py

@@ -29,7 +29,6 @@
 )
 from renku_data_services.data_connectors import apispec, models
 from renku_data_services.data_connectors import orm as schemas
-from renku_data_services.data_connectors.config import ProjectStorageConfig
 from renku_data_services.data_connectors.core import validate_unsaved_global_data_connector
 from renku_data_services.data_connectors.doi.models import DOI
 from renku_data_services.k8s.constants import DEFAULT_K8S_CLUSTER
@@ -58,14 +57,12 @@ def __init__(
         project_repo: ProjectRepository,
         group_repo: GroupRepository,
         search_updates_repo: SearchUpdatesRepo,
-        project_storage_config: ProjectStorageConfig,
     ) -> None:
         self.session_maker = session_maker
         self.authz = authz
         self.project_repo = project_repo
         self.group_repo = group_repo
         self.search_updates_repo = search_updates_repo
-        self.project_storage_config = project_storage_config
 
     async def get_data_connectors(
         self,

components/renku_data_services/notebooks/blueprints.py

@@ -40,6 +40,7 @@
 from renku_data_services.resource_usage.db import ResourceRequestsRepo
 from renku_data_services.session.config import BuildsConfig
 from renku_data_services.session.db import SessionRepository
+from renku_data_services.storage.db import ProjectStorageRepository
 from renku_data_services.users.db import UserRepo
 
 logger = logging.getLogger(__name__)
@@ -62,6 +63,7 @@ class NotebooksNewBP(CustomBlueprint):
     image_check_repo: ImageCheckRepository
     project_repo: ProjectRepository
     project_session_secret_repo: ProjectSessionSecretRepository
+    project_storage_repo: ProjectStorageRepository
     rp_repo: ResourcePoolRepository
     session_repo: SessionRepository
     user_repo: UserRepo
@@ -96,6 +98,7 @@ async def _handler(
                 data_connector_repo=self.data_connector_repo,
                 project_repo=self.project_repo,
                 project_session_secret_repo=self.project_session_secret_repo,
+                project_storage_repo=self.project_storage_repo,
                 rp_repo=self.rp_repo,
                 session_repo=self.session_repo,
                 user_repo=self.user_repo,

components/renku_data_services/notebooks/core_sessions.py

@@ -44,7 +44,6 @@
     DataConnectorSecretRepository,
 )
 from renku_data_services.data_connectors.models import DataConnectorSecret, DataConnectorWithSecrets
-from renku_data_services.data_connectors.project_storage_k8s import ProjectStorageK8s
 from renku_data_services.errors import ValidationError, errors
 from renku_data_services.k8s.models import ClusterConnection, K8sSecret, sanitizer
 from renku_data_services.notebooks import apispec
@@ -125,6 +124,8 @@
 from renku_data_services.session.config import BuildsConfig
 from renku_data_services.session.db import SessionRepository
 from renku_data_services.session.models import Environment, SessionLauncher
+from renku_data_services.storage.db import ProjectStorageRepository
+from renku_data_services.storage.project_storage_k8s import ProjectStorageK8s
 from renku_data_services.users.db import UserRepo
 from renku_data_services.utils.core import get_effective_quota
 
@@ -191,14 +192,14 @@ async def get_extra_containers(
 async def get_project_storage(
     user: APIUser,
     project_storage_k8s: ProjectStorageK8s,
-    data_connector_repo: DataConnectorRepository,
+    project_storage_repo: ProjectStorageRepository,
     project_id: ULID,
     storage_mount: PurePosixPath,
     cluster: ClusterConnection,
     authz: Authz,
 ) -> SessionExtraResources:
     """If applicable, fetch the project storage and return it as SessionExtras."""
-    project_storage = await data_connector_repo.get_storage_to(user, project_id)
+    project_storage = await project_storage_repo.get_storage_to(user, project_id)
     if not project_storage:
         logger.debug(f"Project {project_id} has no project storage.")
         return SessionExtraResources()
@@ -929,6 +930,7 @@ async def start_session(
     data_connector_secret_repo: DataConnectorSecretRepository,
     project_repo: ProjectRepository,
     project_session_secret_repo: ProjectSessionSecretRepository,
+    project_storage_repo: ProjectStorageRepository,
     rp_repo: ResourcePoolRepository,
     session_repo: SessionRepository,
     user_repo: UserRepo,
@@ -1069,7 +1071,7 @@ async def start_session(
     project_storage_k8s = ProjectStorageK8s(nb_config.k8s_v2_client)
     session_extras = session_extras.concat(
         await get_project_storage(
-            user, project_storage_k8s, data_connector_repo, project.id, storage_mount, cluster, authz
+            user, project_storage_k8s, project_storage_repo, project.id, storage_mount, cluster, authz
         )
     )
 

components/renku_data_services/project/blueprints.py

@@ -23,7 +23,6 @@
 from renku_data_services.base_models.metrics import MetricsService, ProjectCreationType
 from renku_data_services.base_models.validation import validate_and_dump, validated_json
 from renku_data_services.data_connectors.db import DataConnectorRepository
-from renku_data_services.data_connectors.project_storage_k8s import ProjectStorageK8s
 from renku_data_services.errors import errors
 from renku_data_services.project import apispec
 from renku_data_services.project import models as project_models
@@ -42,6 +41,7 @@
     ProjectSessionSecretRepository,
 )
 from renku_data_services.session.db import SessionRepository
+from renku_data_services.storage.project_storage_k8s import ProjectStorageK8s
 from renku_data_services.users.db import UserRepo
 
 

components/renku_data_services/storage/blueprints.py

@@ -3,7 +3,7 @@
 from dataclasses import dataclass
 from typing import Any
 
-from sanic import HTTPResponse, Request, empty
+from sanic import Request, empty
 from sanic.response import HTTPResponse, JSONResponse
 from sanic_ext import validate
 from ulid import ULID
@@ -20,7 +20,6 @@
 from renku_data_services.base_api.misc import validate_query
 from renku_data_services.base_api.pagination import PaginationRequest, paginate
 from renku_data_services.base_models.validation import validate_and_dump, validated_json
-from renku_data_services.data_connectors.project_storage_k8s import ProjectStorageK8s
 from renku_data_services.notebooks.data_sources import DataSourceRepository
 from renku_data_services.storage import apispec, models
 from renku_data_services.storage.core import (
@@ -30,6 +29,7 @@
     validate_unsaved_project_storage,
 )
 from renku_data_services.storage.db import ProjectStorageRepository
+from renku_data_services.storage.project_storage_k8s import ProjectStorageK8s
 from renku_data_services.storage.rclone import RCloneValidator
 
 
@@ -98,6 +98,8 @@ async def _obscure(
 
 @dataclass(kw_only=True)
 class ProjectStorageBP(CustomBlueprint):
+    """Handler for project storage."""
+
     project_storage_k8s: ProjectStorageK8s
     project_storage_repo: ProjectStorageRepository
     authenticator: base_models.Authenticator
@@ -124,6 +126,52 @@ async def _get_one(_: Request, user: base_models.APIUser, storage_id: ULID, etag
 
         return "/data_connectors/storage/<storage_id:ulid>", ["GET"], _get_one
 
+    def get_storage_to_project(self) -> BlueprintFactoryResponse:
+        """List all project storage to a given project."""
+
+        @authenticate(self.authenticator)
+        async def _get_all_storage_to_project(
+            _: Request,
+            user: base_models.APIUser,
+            project_id: ULID,
+        ) -> JSONResponse:
+            project_storage = await self.project_storage_repo.get_storage_to(user=user, project_id=project_id)
+            result = [self._dump_project_storage(project_storage)] if project_storage else []
+            return validated_json(apispec.ProjectStorageList, result)
+
+        return "/projects/<project_id:ulid>/storage", ["GET"], _get_all_storage_to_project
+
+    def get_storage_config(self) -> BlueprintFactoryResponse:
+        """Get the current config used for project storage."""
+
+        @authenticate(self.authenticator)
+        @only_admins
+        async def _get_project_config(_: Request, user: base_models.APIUser) -> JSONResponse:
+            storage_config = self.project_storage_repo.get_project_storage_config()
+            result = apispec.ProjectStorageConfig(
+                enabled=storage_config.enabled, max_size=int(storage_config.maximum_size.to_gibi())
+            )
+            return validated_json(apispec.ProjectStorageConfig, result)
+
+        return "/data_connectors/storage/config", ["GET"], _get_project_config
+
+    def delete_storage(self) -> BlueprintFactoryResponse:
+        """Delete a specific project storage."""
+
+        @authenticate(self.authenticator)
+        @only_authenticated
+        async def _delete_storage(
+            _: Request,
+            user: base_models.APIUser,
+            storage_id: ULID,
+        ) -> HTTPResponse:
+            deleted = await self.project_storage_repo.delete_project_storage(user=user, storage_id=storage_id)
+            if deleted:
+                await self.project_storage_k8s.delete_volume(deleted)
+            return HTTPResponse(status=204)
+
+        return "/data_connectors/storage/<storage_id:ulid>", ["DELETE"], _delete_storage
+
     def post_storage(self) -> BlueprintFactoryResponse:
         """Create a new shared project storage."""