feat: launch sessions using the new operator

.devcontainer/devcontainer.json

@@ -20,7 +20,10 @@
 		"ghcr.io/devcontainers/features/kubectl-helm-minikube:1": {
 			"minikube": "none"
 		},
-		"ghcr.io/eitsupi/devcontainer-features/jq-likes:2": {},
+		"ghcr.io/eitsupi/devcontainer-features/jq-likes:2": {
+			"jqVersion": "latest",
+			"yqVersion": "latest"
+		},
 		"ghcr.io/dhoeric/features/k9s:1": {},
 		"ghcr.io/EliiseS/devcontainer-features/bash-profile:1": {
 			"command": "alias k=kubectl"

.devcontainer/docker-compose.yml

@@ -23,6 +23,8 @@ services:
       ZED_TOKEN: renku
       ZED_INSECURE: "true"
       POETRY_CACHE_DIR: "/poetry_cache"
+      NB_SERVER_OPTIONS__DEFAULTS_PATH: /workspace/server_defaults.json
+      NB_SERVER_OPTIONS__UI_CHOICES_PATH: /workspace/server_options.json
     network_mode: service:db
     depends_on:
       - db
@@ -43,6 +45,7 @@ services:
       - "8080:8080"
       - "5678:5678"
       - "50051:50051"
+      - "8888:80"
 
   swagger:
     image: swaggerapi/swagger-ui

.github/workflows/acceptance-tests.yml

@@ -25,13 +25,15 @@ jobs:
       renku-graph: ${{ steps.deploy-comment.outputs.renku-graph}}
       renku-notebooks: ${{ steps.deploy-comment.outputs.renku-notebooks}}
       renku-ui: ${{ steps.deploy-comment.outputs.renku-ui}}
+      amalthea-sessions: ${{ steps.deploy-comment.outputs.amalthea-sessions}}
+      amalthea: ${{ steps.deploy-comment.outputs.amalthea}}
       test-enabled: ${{ steps.deploy-comment.outputs.test-enabled}}
       test-cypress-enabled: ${{ steps.deploy-comment.outputs.test-cypress-enabled}}
       persist: ${{ steps.deploy-comment.outputs.persist}}
       extra-values: ${{ steps.deploy-comment.outputs.extra-values}}
     steps:
       - id: deploy-comment
-        uses: SwissDataScienceCenter/renku-actions/check-pr-description@v1.11.3
+        uses: SwissDataScienceCenter/renku-actions/check-pr-description@v1.12.1
         with:
           string: /deploy
           pr_ref: ${{ github.event.number }}
@@ -61,7 +63,7 @@ jobs:
           body: |
             You can access the deployment of this PR at https://renku-ci-ds-${{ github.event.number }}.dev.renku.ch
       - name: Build and deploy
-        uses: SwissDataScienceCenter/renku-actions/deploy-renku@v1.11.3
+        uses: SwissDataScienceCenter/renku-actions/deploy-renku@v1.12.1
         env:
           RANCHER_PROJECT_ID: ${{ secrets.CI_RANCHER_PROJECT }}
           DOCKER_PASSWORD: ${{ secrets.RENKU_DOCKER_PASSWORD }}
@@ -84,14 +86,16 @@ jobs:
           renku_graph: "${{ needs.check-deploy.outputs.renku-graph }}"
           renku_notebooks: "${{ needs.check-deploy.outputs.renku-notebooks }}"
           renku_data_services: "@${{ github.head_ref }}"
+          amalthea: "${{ needs.check-deploy.outputs.amalthea }}"
+          amalthea-sessions: "${{ needs.check-deploy.outputs.amalthea-sessions }}"
           extra_values: "${{ needs.check-deploy.outputs.extra-values }}"
 
   selenium-acceptance-tests:
     needs: [check-deploy, deploy-pr]
     if: github.event.action != 'closed' && needs.check-deploy.outputs.pr-contains-string == 'true' && needs.check-deploy.outputs.test-enabled == 'true'
     runs-on: ubuntu-22.04
     steps:
-      - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.11.3
+      - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.12.1
         with:
           kubeconfig: ${{ secrets.RENKUBOT_DEV_KUBECONFIG }}
           renku-release: renku-ci-ds-${{ github.event.number }}
@@ -118,7 +122,7 @@ jobs:
     steps:
       - name: Extract Renku repository reference
         run: echo "RENKU_REFERENCE=`echo '${{ needs.check-deploy.outputs.renku }}' | cut -d'@' -f2`" >> $GITHUB_ENV
-      - uses: SwissDataScienceCenter/renku-actions/test-renku-cypress@v1.11.3
+      - uses: SwissDataScienceCenter/renku-actions/test-renku-cypress@v1.12.1
         with:
           e2e-target: ${{ matrix.tests }}
           renku-reference: ${{ env.RENKU_REFERENCE }}
@@ -146,7 +150,7 @@ jobs:
           body: |
             Tearing down the temporary RenkuLab deplyoment for this PR.
       - name: renku teardown
-        uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.11.3
+        uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.12.1
         env:
           HELM_RELEASE_REGEX: "^renku-ci-ds-${{ github.event.number }}$"
           GITLAB_TOKEN: ${{ secrets.DEV_GITLAB_TOKEN }}

.github/workflows/save_cache.yml

@@ -0,0 +1,41 @@
+name: Create cache from commits on main
+
+on:
+  push:
+    branches:
+      - main
+      - chore-add-kind
+  workflow_dispatch:
+
+
+jobs:
+  save-poetry-cache:
+    runs-on: ubuntu-latest
+    env:
+      CACHE_KEY: main-branch-poetry-cache-ubuntu
+      CACHE_PATH: .devcontainer/.poetry_cache
+      DEVCONTAINER_IMAGE_CACHE: ghcr.io/swissdatasciencecenter/renku-data-services/devcontainer
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install python deps
+        uses: devcontainers/ci@v0.3
+        with:
+          runCmd: poetry install --with dev
+          push: always
+          skipContainerUserIdUpdate: false
+          imageName: ${{ env.DEVCONTAINER_IMAGE_CACHE }}
+          cacheFrom: ${{ env.DEVCONTAINER_IMAGE_CACHE }}
+      - uses: actions/cache/save@v3
+        name: Create cache
+        with:
+          path: ${{ env.CACHE_PATH }}
+          key: ${{ env.CACHE_KEY }}

.github/workflows/test_publish.yml

@@ -71,6 +71,11 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+      - uses: actions/cache/restore@v3
+        name: Restore cache
+        with:
+          path: ${{ env.CACHE_PATH }}
+          key: ${{ env.CACHE_KEY }}
       - name: Set Git config
         shell: bash
         run: |
@@ -99,6 +104,11 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+      - uses: actions/cache/restore@v3
+        name: Restore cache
+        with:
+          path: ${{ env.CACHE_PATH }}
+          key: ${{ env.CACHE_KEY }}
       - name: Set Git config
         shell: bash
         run: |
@@ -143,6 +153,11 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+      - uses: actions/cache/restore@v3
+        name: Restore cache
+        with:
+          path: ${{ env.CACHE_PATH }}
+          key: ${{ env.CACHE_KEY }}
       - name: Set Git config
         shell: bash
         run: |

Makefile

@@ -1,7 +1,7 @@
 .PHONY: schemas tests test_setup main_tests schemathesis_tests collect_coverage style_checks pre_commit_checks run download_avro check_avro avro_models update_avro kind_cluster install_amaltheas all
 
-AMALTHEA_JS_VERSION ?= 0.11.0
-AMALTHEA_SESSIONS_VERSION ?= 0.0.1-new-operator-chart
+AMALTHEA_JS_VERSION ?= 0.12.2
+AMALTHEA_SESSIONS_VERSION ?= 0.0.9-new-operator-chart
 codegen_params = --input-file-type openapi --output-model-type pydantic_v2.BaseModel --use-double-quotes --target-python-version 3.12 --collapse-root-models --field-constraints --strict-nullable --set-default-enum-member --openapi-scopes schemas paths parameters --set-default-enum-member --use-one-literal-as-default --use-default
 
 define test_apispec_up_to_date
@@ -41,7 +41,17 @@ components/renku_data_services/platform/apispec.py: components/renku_data_servic
 
 ##@ Apispec
 
-schemas: components/renku_data_services/crc/apispec.py components/renku_data_services/storage/apispec.py components/renku_data_services/users/apispec.py components/renku_data_services/project/apispec.py components/renku_data_services/namespace/apispec.py components/renku_data_services/secrets/apispec.py components/renku_data_services/connected_services/apispec.py components/renku_data_services/repositories/apispec.py components/renku_data_services/notebooks/apispec.py components/renku_data_services/platform/apispec.py  ## Generate pydantic classes from apispec yaml files
+schemas: components/renku_data_services/crc/apispec.py \
+components/renku_data_services/storage/apispec.py \
+components/renku_data_services/users/apispec.py \
+components/renku_data_services/project/apispec.py \
+components/renku_data_services/session/apispec.py \
+components/renku_data_services/namespace/apispec.py \
+components/renku_data_services/secrets/apispec.py \
+components/renku_data_services/connected_services/apispec.py \
+components/renku_data_services/repositories/apispec.py \
+components/renku_data_services/notebooks/apispec.py \
+components/renku_data_services/platform/apispec.py  ## Generate pydantic classes from apispec yaml files
 	@echo "generated classes based on ApiSpec"
 
 ##@ Avro schemas
@@ -84,6 +94,8 @@ style_checks:  ## Run linting and style checks
 	@$(call test_apispec_up_to_date,"notebooks")
 	@echo "checking platform apispec is up to date"
 	@$(call test_apispec_up_to_date,"platform")
+	@echo "checking session apispec is up to date"
+	@$(call test_apispec_up_to_date,"session")
 	poetry run mypy
 	poetry run ruff format --check
 	poetry run ruff check .
@@ -141,7 +153,13 @@ kind_cluster:  ## Creates a kind cluster for testing
 	sleep 15
 	kubectl wait --namespace ingress-nginx --for=condition=ready pod --selector=app.kubernetes.io/component=controller --timeout=90s
 
-install_amaltheas:  ## Installs both version of amalthea in the currently active k8s context.
+install_amaltheas:  ## Installs both version of amalthea in the. NOTE: It uses the currently active k8s context.
 	helm repo add renku https://swissdatasciencecenter.github.io/helm-charts
-	helm install amalthea-js renku/amalthea --version $(AMALTHEA_JS_VERSION)
-	helm install amalthea-sessions renku/amalthea-sessions --version $(AMALTHEA_SESSIONS_VERSION)
+	helm repo update
+	helm upgrade --install amalthea-js renku/amalthea --version $(AMALTHEA_JS_VERSION)
+	helm upgrade --install amalthea-sessions amalthea-sessions-0.0.9-new-operator-chart.tgz --version $(AMALTHEA_SESSIONS_VERSION)
+
+# TODO: Add the version variables from the top of the file here when the charts are fully published
+amalthea_schema:  ## Updates generates pydantic classes from CRDs
+	curl https://raw.githubusercontent.com/SwissDataScienceCenter/amalthea/fix-missing-status/config/crd/bases/amalthea.dev_amaltheasessions.yaml | yq '.spec.versions[0].schema.openAPIV3Schema' | poetry run datamodel-codegen --input-file-type jsonschema --output-model-type pydantic_v2.BaseModel --output components/renku_data_services/notebooks/cr_amalthea_session.py --use-double-quotes --target-python-version 3.12 --collapse-root-models --field-constraints --strict-nullable --base-class renku_data_services.notebooks.cr_base.BaseCRD --allow-extra-fields --use-default-kwarg
+	curl https://raw.githubusercontent.com/SwissDataScienceCenter/amalthea/main/controller/crds/jupyter_server.yaml | yq '.spec.versions[0].schema.openAPIV3Schema' | poetry run datamodel-codegen --input-file-type jsonschema --output-model-type pydantic_v2.BaseModel --output components/renku_data_services/notebooks/cr_jupyter_server.py --use-double-quotes --target-python-version 3.12 --collapse-root-models --field-constraints --strict-nullable --base-class renku_data_services.notebooks.cr_base.BaseCRD --allow-extra-fields --use-default-kwarg

bases/renku_data_services/data_api/app.py

@@ -16,6 +16,7 @@
     UserResourcePoolsBP,
 )
 from renku_data_services.namespace.blueprints import GroupsBP
+from renku_data_services.notebooks.blueprints import NotebooksBP, NotebooksNewBP
 from renku_data_services.platform.blueprints import PlatformConfigBP
 from renku_data_services.project.blueprints import ProjectsBP
 from renku_data_services.repositories.blueprints import RepositoriesBP
@@ -134,6 +135,24 @@ def register_all_handlers(app: Sanic, config: Config) -> Sanic:
         authenticator=config.authenticator,
         internal_gitlab_authenticator=config.gitlab_authenticator,
     )
+    notebooks = NotebooksBP(
+        name="notebooks_old",
+        url_prefix=url_prefix,
+        authenticator=config.authenticator,
+        nb_config=config.nb_config,
+        internal_gitlab_authenticator=config.gitlab_authenticator,
+        git_repo=config.git_repositories_repo,
+    )
+    notebooks_new = NotebooksNewBP(
+        name="notebooks",
+        url_prefix=url_prefix,
+        authenticator=config.authenticator,
+        nb_config=config.nb_config,
+        project_repo=config.project_repo,
+        session_repo=config.session_repo,
+        rp_repo=config.rp_repo,
+        internal_gitlab_authenticator=config.gitlab_authenticator,
+    )
     platform_config = PlatformConfigBP(
         name="platform_config",
         url_prefix=url_prefix,
@@ -161,6 +180,8 @@ def register_all_handlers(app: Sanic, config: Config) -> Sanic:
             oauth2_clients.blueprint(),
             oauth2_connections.blueprint(),
             repositories.blueprint(),
+            notebooks.blueprint(),
+            notebooks_new.blueprint(),
             platform_config.blueprint(),
         ]
     )

components/renku_data_services/app_config/config.py

@@ -52,6 +52,7 @@
 from renku_data_services.message_queue.interface import IMessageQueue
 from renku_data_services.message_queue.redis_queue import RedisQueue
 from renku_data_services.namespace.db import GroupRepository
+from renku_data_services.notebooks.config import _NotebooksConfig
 from renku_data_services.platform.db import PlatformRepository
 from renku_data_services.project.db import ProjectMemberRepository, ProjectRepository
 from renku_data_services.repositories.db import GitRepositoriesRepository
@@ -144,6 +145,7 @@ class Config:
     kc_api: IKeycloakAPI
     message_queue: IMessageQueue
     gitlab_url: str | None
+    nb_config: _NotebooksConfig
 
     secrets_service_public_key: rsa.RSAPublicKey
     """The public key of the secrets service, used to encrypt user secrets that only it can decrypt."""
@@ -208,6 +210,9 @@ def __post_init__(self) -> None:
         with open(spec_file) as f:
             repositories = safe_load(f)
 
+        spec_file = Path(renku_data_services.notebooks.__file__).resolve().parent / "api.spec.yaml"
+        with open(spec_file) as f:
+            repositories = safe_load(f)
         spec_file = Path(renku_data_services.platform.__file__).resolve().parent / "api.spec.yaml"
         with open(spec_file) as f:
             platform = safe_load(f)
@@ -408,8 +413,8 @@ def from_env(cls, prefix: str = "") -> "Config":
         gitlab_client: base_models.GitlabAPIProtocol
         user_preferences_config: UserPreferencesConfig
         version = os.environ.get(f"{prefix}VERSION", "0.0.1")
-        server_options_file = os.environ.get("SERVER_OPTIONS")
-        server_defaults_file = os.environ.get("SERVER_DEFAULTS")
+        server_options_file = os.environ.get("NB_SERVER_OPTIONS__UI_CHOICES_PATH")
+        server_defaults_file = os.environ.get("NB_SERVER_OPTIONS__DEFAULTS_PATH")
         k8s_namespace = os.environ.get("K8S_NAMESPACE", "default")
         max_pinned_projects = int(os.environ.get(f"{prefix}MAX_PINNED_PROJECTS", "10"))
         user_preferences_config = UserPreferencesConfig(max_pinned_projects=max_pinned_projects)
@@ -491,6 +496,7 @@ def from_env(cls, prefix: str = "") -> "Config":
         sentry = SentryConfig.from_env(prefix)
         trusted_proxies = TrustedProxiesConfig.from_env(prefix)
         message_queue = RedisQueue(redis)
+        nb_config = _NotebooksConfig.from_env(db)
 
         return cls(
             version=version,
@@ -511,4 +517,5 @@ def from_env(cls, prefix: str = "") -> "Config":
             encryption_key=encryption_key,
             secrets_service_public_key=secrets_service_public_key,
             gitlab_url=gitlab_url,
+            nb_config=nb_config,
         )

components/renku_data_services/authn/dummy.py

@@ -57,11 +57,12 @@ async def authenticate(access_token: str, request: Request) -> base_models.APIUs
         )
 
         return base_models.APIUser(
-            is_admin=user_props.get("is_admin", False),
+            is_admin_init=user_props.get("is_admin", False),
             id=user_props.get("id", "some-id") if is_set else None,
             access_token=access_token,
             first_name=user_props.get("first_name", "John") if is_set else None,
             last_name=user_props.get("last_name", "Doe") if is_set else None,
             email=user_props.get("email", "john.doe@gmail.com") if is_set else None,
             full_name=user_props.get("full_name", "John Doe") if is_set else None,
+            refresh_token=request.headers.get("Renku-Auth-Refresh-Token"),
         )

components/renku_data_services/authn/gitlab.py

@@ -1,11 +1,18 @@
 """Gitlab authenticator."""
 
+import base64
 import contextlib
+import json
+import re
 import urllib.parse as parse
+from contextlib import suppress
 from dataclasses import dataclass
+from datetime import datetime
+from typing import Any
 
 import gitlab
 from sanic import Request
+from sanic.compat import Header
 
 import renku_data_services.base_models as base_models
 from renku_data_services import errors
@@ -36,10 +43,10 @@ async def authenticate(self, access_token: str, request: Request) -> base_models
         if self.token_field != "Authorization":  # nosec: B105
             access_token = str(request.headers.get(self.token_field))
 
-        result = await self._get_gitlab_api_user(access_token)
+        result = await self._get_gitlab_api_user(access_token, request.headers)
         return result
 
-    async def _get_gitlab_api_user(self, access_token: str) -> base_models.APIUser:
+    async def _get_gitlab_api_user(self, access_token: str, headers: Header) -> base_models.APIUser:
         """Get and validate a Gitlab API User."""
         client = gitlab.Gitlab(self.gitlab_url, oauth_token=access_token)
         try:
@@ -69,12 +76,35 @@ async def _get_gitlab_api_user(self, access_token: str) -> base_models.APIUser:
             if len(name_parts) >= 1:
                 last_name = " ".join(name_parts)
 
+        _, _, _, expires_at = self.git_creds_from_headers(headers)
         return base_models.APIUser(
-            is_admin=False,
             id=str(user_id),
             access_token=access_token,
             first_name=first_name,
             last_name=last_name,
             email=email,
             full_name=full_name,
+            access_token_expires_at=expires_at,
+        )
+
+    @staticmethod
+    def git_creds_from_headers(headers: Header) -> tuple[Any, Any, Any, datetime | None]:
+        """Extract git credentials from the encoded header sent by the gateway."""
+        parsed_dict = json.loads(base64.decodebytes(headers["Renku-Auth-Git-Credentials"].encode()))
+        git_url, git_credentials = next(iter(parsed_dict.items()))
+        token_match = re.match(r"^[^\s]+\ ([^\s]+)$", git_credentials["AuthorizationHeader"])
+        git_token = token_match.group(1) if token_match is not None else None
+        git_token_expires_at_raw = git_credentials["AccessTokenExpiresAt"]
+        git_token_expires_at_num: float | None = None
+        with suppress(ValueError, TypeError):
+            git_token_expires_at_num = float(git_token_expires_at_raw)
+        git_token_expires_at: datetime | None = None
+        if git_token_expires_at_num is not None and git_token_expires_at_num > 0:
+            with suppress(ValueError):
+                git_token_expires_at = datetime.fromtimestamp(git_token_expires_at_num)
+        return (
+            git_url,
+            git_credentials["AuthorizationHeader"],
+            git_token,
+            git_token_expires_at,
         )