feat: handle private images from gitlab (#996)

Co-authored-by: Mohammad Alisafaee <mohammad.alisafaee@epfl.ch>

Author: olevski

bases/renku_data_services/data_api/app.py

@@ -216,6 +216,7 @@ def register_all_handlers(app: Sanic, dm: DependencyManager) -> Sanic:
         cluster_repo=dm.cluster_repo,
         internal_gitlab_authenticator=dm.gitlab_authenticator,
         metrics=dm.metrics,
+        connected_svcs_repo=dm.connected_services_repo,
     )
     platform_config = PlatformConfigBP(
         name="platform_config",

bases/renku_data_services/data_api/config.py

@@ -44,7 +44,7 @@ class Config:
     @classmethod
     def from_env(cls, db: DBConfig | None = None) -> Self:
         """Load config from environment."""
-        enable_internal_gitlab = os.getenv("ENABLE_V1_SERVICES", "true").lower() == "true"
+        enable_internal_gitlab = os.getenv("ENABLE_INTERNAL_GITLAB", "true").lower() == "true"
 
         dummy_stores = os.environ.get("DUMMY_STORES", "false").lower() == "true"
         if db is None:

components/renku_data_services/connected_services/api.spec.yaml

@@ -272,6 +272,8 @@ components:
           $ref: "#/components/schemas/ProviderUrl"
         use_pkce:
           $ref: "#/components/schemas/UsePKCE"
+        image_registry_url:
+          $ref: "#/components/schemas/ImageRegistryUrl"
       required:
         - id
         - kind
@@ -303,6 +305,8 @@ components:
           $ref: "#/components/schemas/ProviderUrl"
         use_pkce:
           $ref: "#/components/schemas/UsePKCE"
+        image_registry_url:
+          $ref: "#/components/schemas/ImageRegistryUrl"
       required:
         - id
         - kind
@@ -330,6 +334,8 @@ components:
           $ref: "#/components/schemas/ProviderUrl"
         use_pkce:
           $ref: "#/components/schemas/UsePKCE"
+        image_registry_url:
+          $ref: "#/components/schemas/ImageRegistryUrl"
     ConnectionList:
       type: array
       items:
@@ -468,6 +474,12 @@ components:
           minimum: 1
           maximum: 100
           default: 20
+    ImageRegistryUrl:
+      type: string
+      description: |
+        This should contain no paths, just the domain for the registry and the scheme
+        (http or https) to access the image registry API.
+      example: https://registry.gitlab.com
     ErrorResponse:
       type: object
       properties:

components/renku_data_services/connected_services/apispec.py

@@ -1,6 +1,6 @@
 # generated by datamodel-codegen:
 #   filename:  api.spec.yaml
-#   timestamp: 2025-03-19T10:21:11+00:00
+#   timestamp: 2025-09-01T13:55:20+00:00
 
 from __future__ import annotations
 
@@ -120,6 +120,11 @@ class Provider(BaseAPISpec):
         description="Whether or not to use PKCE during authorization flows.\n",
         examples=[False],
     )
+    image_registry_url: Optional[str] = Field(
+        None,
+        description="This should contain no paths, just the domain for the registry and the scheme\n(http or https) to access the image registry API.\n",
+        examples=["https://registry.gitlab.com"],
+    )
 
 
 class ProviderPost(BaseAPISpec):
@@ -155,10 +160,15 @@ class ProviderPost(BaseAPISpec):
         examples=["https://example.org"],
     )
     use_pkce: Optional[bool] = Field(
-        None,
+        False,
         description="Whether or not to use PKCE during authorization flows.\n",
         examples=[False],
     )
+    image_registry_url: Optional[str] = Field(
+        None,
+        description="This should contain no paths, just the domain for the registry and the scheme\n(http or https) to access the image registry API.\n",
+        examples=["https://registry.gitlab.com"],
+    )
 
 
 class ProviderPatch(BaseAPISpec):
@@ -189,10 +199,15 @@ class ProviderPatch(BaseAPISpec):
         examples=["https://example.org"],
     )
     use_pkce: Optional[bool] = Field(
-        None,
+        False,
         description="Whether or not to use PKCE during authorization flows.\n",
         examples=[False],
     )
+    image_registry_url: Optional[str] = Field(
+        None,
+        description="This should contain no paths, just the domain for the registry and the scheme\n(http or https) to access the image registry API.\n",
+        examples=["https://registry.gitlab.com"],
+    )
 
 
 class Connection(BaseAPISpec):

components/renku_data_services/connected_services/blueprints.py

@@ -18,7 +18,7 @@
 from renku_data_services.base_models.validation import validate_and_dump, validated_json
 from renku_data_services.connected_services import apispec
 from renku_data_services.connected_services.apispec_base import AuthorizeParams, CallbackParams
-from renku_data_services.connected_services.core import validate_oauth2_client_patch
+from renku_data_services.connected_services.core import validate_oauth2_client_patch, validate_unsaved_oauth2_client
 from renku_data_services.connected_services.db import ConnectedServicesRepository
 
 logger = logging.getLogger(__name__)
@@ -59,7 +59,8 @@ def post(self) -> BlueprintFactoryResponse:
         @only_admins
         @validate(json=apispec.ProviderPost)
         async def _post(_: Request, user: base_models.APIUser, body: apispec.ProviderPost) -> JSONResponse:
-            client = await self.connected_services_repo.insert_oauth2_client(user=user, new_client=body)
+            new_client = validate_unsaved_oauth2_client(body)
+            client = await self.connected_services_repo.insert_oauth2_client(user=user, new_client=new_client)
             return validated_json(apispec.Provider, client, 201)
 
         return "/oauth2/providers", ["POST"], _post

components/renku_data_services/connected_services/core.py

@@ -1,10 +1,15 @@
 """Business logic for connected services."""
 
+from urllib.parse import urlparse
+
 from renku_data_services.connected_services import apispec, models
+from renku_data_services.errors import errors
 
 
 def validate_oauth2_client_patch(patch: apispec.ProviderPatch) -> models.OAuth2ClientPatch:
     """Validate the update to a OAuth2 Client."""
+    if patch.image_registry_url is not None and len(patch.image_registry_url) > 0:
+        validate_image_registry_url(patch.image_registry_url)
     return models.OAuth2ClientPatch(
         kind=patch.kind,
         app_slug=patch.app_slug,
@@ -14,4 +19,39 @@ def validate_oauth2_client_patch(patch: apispec.ProviderPatch) -> models.OAuth2C
         scope=patch.scope,
         url=patch.url,
         use_pkce=patch.use_pkce,
+        image_registry_url=patch.image_registry_url,
+    )
+
+
+def validate_unsaved_oauth2_client(clnt: apispec.ProviderPost) -> models.UnsavedOAuth2Client:
+    """Validate the the creation of a new OAuth2 Client."""
+    if clnt.image_registry_url is not None:
+        validate_image_registry_url(clnt.image_registry_url)
+    return models.UnsavedOAuth2Client(
+        id=clnt.id,
+        kind=clnt.kind,
+        app_slug=clnt.app_slug or "",
+        client_id=clnt.client_id,
+        client_secret=clnt.client_secret,
+        display_name=clnt.display_name,
+        scope=clnt.scope,
+        url=clnt.url,
+        use_pkce=clnt.use_pkce or False,
+        image_registry_url=clnt.image_registry_url,
     )
+
+
+def validate_image_registry_url(url: str) -> None:
+    """Validate an image registry url."""
+    parsed = urlparse(url)
+    if not parsed.netloc:
+        raise errors.ValidationError(
+            message=f"The host for the image registry url {url} is not valid, expected a non-empty value.",
+            quiet=True,
+        )
+    accepted_schemes = ["https"]
+    if parsed.scheme not in accepted_schemes:
+        raise errors.ValidationError(
+            message=f"The scheme for the image registry url {url} is not valid, expected one of {accepted_schemes}",
+            quiet=True,
+        )

components/renku_data_services/connected_services/db.py

@@ -4,20 +4,20 @@
 from collections.abc import AsyncGenerator, Callable
 from contextlib import asynccontextmanager
 from typing import Any
-from urllib.parse import urljoin
+from urllib.parse import urljoin, urlparse
 
 from authlib.integrations.base_client import InvalidTokenError
 from authlib.integrations.httpx_client import AsyncOAuth2Client, OAuthError
 from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
-from sqlalchemy.orm import selectinload
+from sqlalchemy.orm import joinedload, selectinload
 from ulid import ULID
 
 import renku_data_services.base_models as base_models
 from renku_data_services import errors
 from renku_data_services.app_config import logging
 from renku_data_services.base_api.pagination import PaginationRequest
-from renku_data_services.connected_services import apispec, models
+from renku_data_services.connected_services import models
 from renku_data_services.connected_services import orm as schemas
 from renku_data_services.connected_services.apispec import ConnectionStatus, ProviderKind
 from renku_data_services.connected_services.provider_adapters import (
@@ -26,6 +26,7 @@
     get_provider_adapter,
 )
 from renku_data_services.connected_services.utils import generate_code_verifier
+from renku_data_services.notebooks.api.classes.image import Image, ImageRepoDockerAPI
 from renku_data_services.utils.cryptography import decrypt_string, encrypt_string
 
 logger = logging.getLogger(__name__)
@@ -68,9 +69,7 @@ async def get_oauth2_client(self, provider_id: str, user: base_models.APIUser) -
             return client.dump(user_is_admin=user.is_admin)
 
     async def insert_oauth2_client(
-        self,
-        user: base_models.APIUser,
-        new_client: apispec.ProviderPost,
+        self, user: base_models.APIUser, new_client: models.UnsavedOAuth2Client
     ) -> models.OAuth2Client:
         """Insert a new OAuth2 Client environment."""
         if user.id is None:
@@ -93,6 +92,7 @@ async def insert_oauth2_client(
             url=new_client.url,
             use_pkce=new_client.use_pkce or False,
             created_by_id=user.id,
+            image_registry_url=new_client.image_registry_url,
         )
 
         async with self.session_maker() as session, session.begin():
@@ -144,6 +144,12 @@ async def update_oauth2_client(
                 client.url = patch.url
             if patch.use_pkce is not None:
                 client.use_pkce = patch.use_pkce
+            if patch.image_registry_url:
+                # Patching with a string of at least length 1 updates the value
+                client.image_registry_url = patch.image_registry_url
+            elif patch.image_registry_url == "":
+                # Patching with "", removes the value
+                client.image_registry_url = None
 
             await session.flush()
             await session.refresh(client)
@@ -272,7 +278,7 @@ async def authorize_callback(self, state: str, raw_url: str, callback_url: str)
                     adapter.token_endpoint_url, authorization_response=raw_url, code_verifier=code_verifier
                 )
 
-                logger.info(f"Token for client {client.id} has keys: {", ".join(token.keys())}")
+                logger.info(f"Token for client {client.id} has keys: {', '.join(token.keys())}")
 
                 next_url = connection.next_url
 
@@ -356,6 +362,40 @@ async def get_oauth2_connection_token(
             token_model = models.OAuth2TokenSet.from_dict(oauth2_client.token)
             return token_model
 
+    async def get_docker_client(
+        self, user: base_models.APIUser, image: Image
+    ) -> tuple[ImageRepoDockerAPI, ULID] | tuple[None, None]:
+        """Search for clients and connections that can work with the specific image and return a docker client."""
+        async with self.session_maker() as session:
+            registry_urls = [f"http://{image.hostname}", f"https://{image.hostname}"]
+            stmt = (
+                select(schemas.OAuth2ConnectionORM)
+                .where(schemas.OAuth2ConnectionORM.user_id == user.id)
+                .where(
+                    schemas.OAuth2ConnectionORM.client.has(
+                        schemas.OAuth2ClientORM.image_registry_url.in_(registry_urls)
+                    )
+                )
+                .options(joinedload(schemas.OAuth2ConnectionORM.client))
+            )
+            conn = await session.scalar(stmt)
+        if not conn:
+            return None, None
+        if conn.client.kind != ProviderKind.gitlab:
+            # NOTE: Only Gitlab is currently supported for this
+            return None, None
+        url = conn.client.image_registry_url
+        if not url:
+            return None, None
+        token_set = await self.get_oauth2_connection_token(conn.id, user)
+        url_parsed = urlparse(url)
+        access_token = token_set.access_token
+        if not access_token:
+            return None, None
+        return ImageRepoDockerAPI(
+            hostname=url_parsed.netloc, scheme=url_parsed.scheme, oauth2_token=access_token
+        ), conn.id
+
     async def get_oauth2_app_installations(
         self, connection_id: ULID, user: base_models.APIUser, pagination: PaginationRequest
     ) -> models.AppInstallationList:

components/renku_data_services/connected_services/models.py

@@ -10,7 +10,7 @@
 
 
 @dataclass(frozen=True, eq=True, kw_only=True)
-class OAuth2Client:
+class UnsavedOAuth2Client:
     """OAuth2 Client model."""
 
     id: str
@@ -22,6 +22,13 @@ class OAuth2Client:
     scope: str
     url: str
     use_pkce: bool
+    image_registry_url: str | None = None
+
+
+@dataclass(frozen=True, eq=True, kw_only=True)
+class OAuth2Client(UnsavedOAuth2Client):
+    """OAuth2 Client model."""
+
     created_by_id: str
     creation_date: datetime
     updated_at: datetime
@@ -39,6 +46,7 @@ class OAuth2ClientPatch:
     scope: str | None
     url: str | None
     use_pkce: bool | None
+    image_registry_url: str | None
 
 
 @dataclass(frozen=True, eq=True, kw_only=True)

components/renku_data_services/connected_services/orm.py

@@ -49,6 +49,7 @@ class OAuth2ClientORM(BaseORM):
         onupdate=func.now(),
         nullable=False,
     )
+    image_registry_url: Mapped[str | None] = mapped_column(default=None, nullable=True, server_default=None)
 
     def dump(self, user_is_admin: bool = False) -> models.OAuth2Client:
         """Create an OAuth2 Client model from the OAuth2ClientORM.
@@ -68,6 +69,7 @@ def dump(self, user_is_admin: bool = False) -> models.OAuth2Client:
             created_by_id=self.created_by_id,
             creation_date=self.creation_date,
             updated_at=self.updated_at,
+            image_registry_url=self.image_registry_url,
         )
 
 

components/renku_data_services/migrations/versions/35ea9d8f54e8_add_image_registry_url.py

@@ -0,0 +1,30 @@
+"""add image registry url
+
+Revision ID: 35ea9d8f54e8
+Revises: c8061499b966
+Create Date: 2025-08-27 14:34:34.190341
+
+"""
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "35ea9d8f54e8"
+down_revision = "c8061499b966"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "oauth2_clients", sa.Column("image_registry_url", sa.String(), nullable=True), schema="connected_services"
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column("oauth2_clients", "image_registry_url", schema="connected_services")
+    # ### end Alembic commands ###