feat: add data connectors (#478)

Co-authored-by: Samuel Gaist <samuel.gaist@idiap.ch>
Co-authored-by: Ralf Grubenmann <ralf.grubenmann@sdsc.ethz.ch>
Co-authored-by: Flora Thiebaut <flora.thiebaut@sdsc.ethz.ch>

Author: 4 people

Makefile

@@ -1,7 +1,7 @@
 .PHONY: schemas tests test_setup main_tests schemathesis_tests collect_coverage style_checks pre_commit_checks run download_avro check_avro avro_models update_avro k3d_cluster install_amaltheas all
 
-AMALTHEA_JS_VERSION ?= 0.12.2
-AMALTHEA_SESSIONS_VERSION ?= 0.0.10-new-operator-chart
+AMALTHEA_JS_VERSION ?= 0.13.0
+AMALTHEA_SESSIONS_VERSION ?= 0.13.0
 codegen_params = --input-file-type openapi --output-model-type pydantic_v2.BaseModel --use-double-quotes --target-python-version 3.12 --collapse-root-models --field-constraints --strict-nullable --set-default-enum-member --openapi-scopes schemas paths parameters --set-default-enum-member --use-one-literal-as-default --use-default
 
 define test_apispec_up_to_date
@@ -166,5 +166,5 @@ install_amaltheas:  ## Installs both version of amalthea in the. NOTE: It uses t
 
 # TODO: Add the version variables from the top of the file here when the charts are fully published
 amalthea_schema:  ## Updates generates pydantic classes from CRDs
-	curl https://raw.githubusercontent.com/SwissDataScienceCenter/amalthea/feat-add-cloud-storage/config/crd/bases/amalthea.dev_amaltheasessions.yaml | yq '.spec.versions[0].schema.openAPIV3Schema' | poetry run datamodel-codegen --input-file-type jsonschema --output-model-type pydantic_v2.BaseModel --output components/renku_data_services/notebooks/cr_amalthea_session.py --use-double-quotes --target-python-version 3.12 --collapse-root-models --field-constraints --strict-nullable --base-class renku_data_services.notebooks.cr_base.BaseCRD --allow-extra-fields --use-default-kwarg
+	curl https://raw.githubusercontent.com/SwissDataScienceCenter/amalthea/main/config/crd/bases/amalthea.dev_amaltheasessions.yaml | yq '.spec.versions[0].schema.openAPIV3Schema' | poetry run datamodel-codegen --input-file-type jsonschema --output-model-type pydantic_v2.BaseModel --output components/renku_data_services/notebooks/cr_amalthea_session.py --use-double-quotes --target-python-version 3.12 --collapse-root-models --field-constraints --strict-nullable --base-class renku_data_services.notebooks.cr_base.BaseCRD --allow-extra-fields --use-default-kwarg
 	curl https://raw.githubusercontent.com/SwissDataScienceCenter/amalthea/main/controller/crds/jupyter_server.yaml | yq '.spec.versions[0].schema.openAPIV3Schema' | poetry run datamodel-codegen --input-file-type jsonschema --output-model-type pydantic_v2.BaseModel --output components/renku_data_services/notebooks/cr_jupyter_server.py --use-double-quotes --target-python-version 3.12 --collapse-root-models --field-constraints --strict-nullable --base-class renku_data_services.notebooks.cr_base.BaseCRD --allow-extra-fields --use-default-kwarg

bases/renku_data_services/data_api/app.py

@@ -149,6 +149,9 @@ def register_all_handlers(app: Sanic, config: Config) -> Sanic:
         session_repo=config.session_repo,
         storage_repo=config.storage_repo,
         rp_repo=config.rp_repo,
+        data_connector_repo=config.data_connector_repo,
+        data_connector_project_link_repo=config.data_connector_to_project_link_repo,
+        data_connector_secret_repo=config.data_connector_secret_repo,
         internal_gitlab_authenticator=config.gitlab_authenticator,
     )
     platform_config = PlatformConfigBP(

components/renku_data_services/app_config/config.py

@@ -459,6 +459,7 @@ def data_connector_secret_repo(self) -> DataConnectorSecretRepository:
                 data_connector_repo=self.data_connector_repo,
                 user_repo=self.kc_user_repo,
                 secret_service_public_key=self.secrets_service_public_key,
+                authz=self.authz,
             )
         return self._data_connector_secret_repo
 

components/renku_data_services/authn/keycloak.py

@@ -98,7 +98,7 @@ async def authenticate(
             user = base_models.AuthenticatedAPIUser(
                 is_admin=is_admin,
                 id=id,
-                access_token=access_token,
+                access_token=token,
                 full_name=parsed.get("name"),
                 first_name=parsed.get("given_name"),
                 last_name=parsed.get("family_name"),

components/renku_data_services/base_api/auth.py

@@ -71,30 +71,6 @@ async def decorated_function(request: Request, *args: _P.args, **kwargs: _P.kwar
     return decorator
 
 
-def validate_path_project_id(
-    f: Callable[Concatenate[Request, _P], Coroutine[Any, Any, _T]],
-) -> Callable[Concatenate[Request, _P], Coroutine[Any, Any, _T]]:
-    """Decorator for a Sanic handler that validates the project_id path parameter."""
-    _path_project_id_regex = re.compile(r"^[A-Za-z0-9]{26}$")
-
-    @wraps(f)
-    async def decorated_function(request: Request, *args: _P.args, **kwargs: _P.kwargs) -> _T:
-        project_id = cast(str | None, kwargs.get("project_id"))
-        if not project_id:
-            raise errors.ProgrammingError(
-                message="Could not find 'project_id' in the keyword arguments for the handler in order to validate it."
-            )
-        if not _path_project_id_regex.match(project_id):
-            raise errors.ValidationError(
-                message=f"The 'project_id' path parameter {project_id} does not match the required "
-                f"regex {_path_project_id_regex}"
-            )
-
-        return await f(request, *args, **kwargs)
-
-    return decorated_function
-
-
 def validate_path_user_id(
     f: Callable[Concatenate[Request, _P], Coroutine[Any, Any, _T]],
 ) -> Callable[Concatenate[Request, _P], Coroutine[Any, Any, _T]]:

components/renku_data_services/crc/db.py

@@ -12,7 +12,7 @@
 from functools import wraps
 from typing import Any, Concatenate, Optional, ParamSpec, TypeVar, cast
 
-from sqlalchemy import NullPool, delete, select
+from sqlalchemy import NullPool, delete, false, select, true
 from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
 from sqlalchemy.orm import selectinload
 from sqlalchemy.sql import Select, and_, not_, or_

components/renku_data_services/data_connectors/api.spec.yaml

@@ -619,10 +619,10 @@ components:
         exclusive:
           type: boolean
           description: if true, only values from 'examples' can be used
-        datatype:
+        type:
           type: string
           description: data type of option value. RClone has more options but they map to the ones listed here.
-          enum: ["int", "bool", "string", "Time"]
+          enum: ["int", "bool", "string", "Time", "Duration", "MultiEncoder", "SizeSuffix", "SpaceSepList", "CommaSepList", "Tristate"]
     Ulid:
       description: ULID identifier
       type: string

components/renku_data_services/data_connectors/apispec.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api.spec.yaml
-#   timestamp: 2024-10-22T07:46:54+00:00
+#   timestamp: 2024-10-28T20:03:14+00:00
 
 from __future__ import annotations
 
@@ -23,11 +23,17 @@ class Example(BaseAPISpec):
     )
 
 
-class Datatype(Enum):
+class Type(Enum):
     int = "int"
     bool = "bool"
     string = "string"
     Time = "Time"
+    Duration = "Duration"
+    MultiEncoder = "MultiEncoder"
+    SizeSuffix = "SizeSuffix"
+    SpaceSepList = "SpaceSepList"
+    CommaSepList = "CommaSepList"
+    Tristate = "Tristate"
 
 
 class RCloneOption(BaseAPISpec):
@@ -65,7 +71,7 @@ class RCloneOption(BaseAPISpec):
     exclusive: Optional[bool] = Field(
         None, description="if true, only values from 'examples' can be used"
     )
-    datatype: Optional[Datatype] = Field(
+    type: Optional[Type] = Field(
         None,
         description="data type of option value. RClone has more options but they map to the ones listed here.",
     )

components/renku_data_services/data_connectors/db.py

@@ -1,6 +1,6 @@
 """Adapters for data connectors database classes."""
 
-from collections.abc import Callable
+from collections.abc import AsyncIterator, Callable
 from typing import TypeVar
 
 from cryptography.hazmat.primitives.asymmetric import rsa
@@ -477,11 +477,44 @@ def __init__(
         data_connector_repo: DataConnectorRepository,
         user_repo: UserRepo,
         secret_service_public_key: rsa.RSAPublicKey,
+        authz: Authz,
     ) -> None:
         self.session_maker = session_maker
         self.data_connector_repo = data_connector_repo
         self.user_repo = user_repo
         self.secret_service_public_key = secret_service_public_key
+        self.authz = authz
+
+    async def get_data_connectors_with_secrets(
+        self,
+        user: base_models.APIUser,
+        project_id: ULID,
+    ) -> AsyncIterator[models.DataConnectorWithSecrets]:
+        """Get all data connectors and their secrets for a project."""
+        if user.id is None:
+            raise errors.UnauthorizedError(message="You do not have the required permissions for this operation.")
+
+        can_read_project = await self.authz.has_permission(user, ResourceType.project, project_id, Scope.READ)
+        if not can_read_project:
+            raise errors.MissingResourceError(
+                message=f"The project ID with {project_id} does not exist or you dont have permission to access it"
+            )
+
+        data_connector_ids = await self.authz.resources_with_permission(
+            user, user.id, ResourceType.data_connector, Scope.READ
+        )
+
+        async with self.session_maker() as session:
+            stmt = select(schemas.DataConnectorORM).where(
+                schemas.DataConnectorORM.project_links.any(
+                    schemas.DataConnectorToProjectLinkORM.project_id == project_id
+                ),
+                schemas.DataConnectorORM.id.in_(data_connector_ids),
+            )
+            results = await session.stream_scalars(stmt)
+            async for dc in results:
+                secrets = await self.get_data_connector_secrets(user, dc.id)
+                yield models.DataConnectorWithSecrets(dc.dump(), secrets)
 
     async def get_data_connector_secrets(
         self,

components/renku_data_services/data_connectors/models.py

@@ -150,3 +150,11 @@ class DataConnectorPermissions:
     write: bool
     delete: bool
     change_membership: bool
+
+
+@dataclass
+class DataConnectorWithSecrets:
+    """A data connector with its secrets."""
+
+    data_connector: DataConnector
+    secrets: list[DataConnectorSecret] = field(default_factory=list)