squashme: notebooks changes

This includes major edits to the notebooks code to work with the data
service.

Author: olevski

.devcontainer/devcontainer.json

@@ -20,7 +20,10 @@
 		"ghcr.io/devcontainers/features/kubectl-helm-minikube:1": {
 			"minikube": "none"
 		},
-		"ghcr.io/eitsupi/devcontainer-features/jq-likes:2": {},
+		"ghcr.io/eitsupi/devcontainer-features/jq-likes:2": {
+			"jqVersion": "latest",
+			"yqVersion": "latest"
+		},
 		"ghcr.io/dhoeric/features/k9s:1": {},
 		"ghcr.io/EliiseS/devcontainer-features/bash-profile:1": {
 			"command": "alias k=kubectl"

.devcontainer/docker-compose.yml

@@ -23,6 +23,8 @@ services:
       ZED_TOKEN: renku
       ZED_INSECURE: "true"
       POETRY_CACHE_DIR: "/poetry_cache"
+      NB_SERVER_OPTIONS__DEFAULTS_PATH: /workspace/server_defaults.json
+      NB_SERVER_OPTIONS__UI_CHOICES_PATH: /workspace/server_options.json
     network_mode: service:db
     depends_on:
       - db
@@ -43,6 +45,7 @@ services:
       - "8080:8080"
       - "5678:5678"
       - "50051:50051"
+      - "8888:80"
 
   swagger:
     image: swaggerapi/swagger-ui

.github/workflows/acceptance-tests.yml

@@ -25,13 +25,15 @@ jobs:
       renku-graph: ${{ steps.deploy-comment.outputs.renku-graph}}
       renku-notebooks: ${{ steps.deploy-comment.outputs.renku-notebooks}}
       renku-ui: ${{ steps.deploy-comment.outputs.renku-ui}}
+      amalthea-sessions: ${{ steps.deploy-comment.outputs.amalthea-sessions}}
+      amalthea: ${{ steps.deploy-comment.outputs.amalthea}}
       test-enabled: ${{ steps.deploy-comment.outputs.test-enabled}}
       test-cypress-enabled: ${{ steps.deploy-comment.outputs.test-cypress-enabled}}
       persist: ${{ steps.deploy-comment.outputs.persist}}
       extra-values: ${{ steps.deploy-comment.outputs.extra-values}}
     steps:
       - id: deploy-comment
-        uses: SwissDataScienceCenter/renku-actions/check-pr-description@v1.11.3
+        uses: SwissDataScienceCenter/renku-actions/check-pr-description@v1.12.1
         with:
           string: /deploy
           pr_ref: ${{ github.event.number }}
@@ -61,7 +63,7 @@ jobs:
           body: |
             You can access the deployment of this PR at https://renku-ci-ds-${{ github.event.number }}.dev.renku.ch
       - name: Build and deploy
-        uses: SwissDataScienceCenter/renku-actions/deploy-renku@v1.11.3
+        uses: SwissDataScienceCenter/renku-actions/deploy-renku@v1.12.1
         env:
           RANCHER_PROJECT_ID: ${{ secrets.CI_RANCHER_PROJECT }}
           DOCKER_PASSWORD: ${{ secrets.RENKU_DOCKER_PASSWORD }}
@@ -84,14 +86,16 @@ jobs:
           renku_graph: "${{ needs.check-deploy.outputs.renku-graph }}"
           renku_notebooks: "${{ needs.check-deploy.outputs.renku-notebooks }}"
           renku_data_services: "@${{ github.head_ref }}"
+          amalthea: "${{ needs.check-deploy.outputs.amalthea }}"
+          amalthea-sessions: "${{ needs.check-deploy.outputs.amalthea-sessions }}"
           extra_values: "${{ needs.check-deploy.outputs.extra-values }}"
 
   selenium-acceptance-tests:
     needs: [check-deploy, deploy-pr]
     if: github.event.action != 'closed' && needs.check-deploy.outputs.pr-contains-string == 'true' && needs.check-deploy.outputs.test-enabled == 'true'
     runs-on: ubuntu-22.04
     steps:
-      - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.11.3
+      - uses: SwissDataScienceCenter/renku-actions/test-renku@v1.12.1
         with:
           kubeconfig: ${{ secrets.RENKUBOT_DEV_KUBECONFIG }}
           renku-release: renku-ci-ds-${{ github.event.number }}
@@ -118,7 +122,7 @@ jobs:
     steps:
       - name: Extract Renku repository reference
         run: echo "RENKU_REFERENCE=`echo '${{ needs.check-deploy.outputs.renku }}' | cut -d'@' -f2`" >> $GITHUB_ENV
-      - uses: SwissDataScienceCenter/renku-actions/test-renku-cypress@v1.11.3
+      - uses: SwissDataScienceCenter/renku-actions/test-renku-cypress@v1.12.1
         with:
           e2e-target: ${{ matrix.tests }}
           renku-reference: ${{ env.RENKU_REFERENCE }}
@@ -146,7 +150,7 @@ jobs:
           body: |
             Tearing down the temporary RenkuLab deplyoment for this PR.
       - name: renku teardown
-        uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.11.3
+        uses: SwissDataScienceCenter/renku-actions/cleanup-renku-ci-deployments@v1.12.1
         env:
           HELM_RELEASE_REGEX: "^renku-ci-ds-${{ github.event.number }}$"
           GITLAB_TOKEN: ${{ secrets.DEV_GITLAB_TOKEN }}

.github/workflows/save_cache.yml

@@ -0,0 +1,41 @@
+name: Create cache from commits on main
+
+on:
+  push:
+    branches:
+      - main
+      - chore-add-kind
+  workflow_dispatch:
+
+
+jobs:
+  save-poetry-cache:
+    runs-on: ubuntu-latest
+    env:
+      CACHE_KEY: main-branch-poetry-cache-ubuntu
+      CACHE_PATH: .devcontainer/.poetry_cache
+      DEVCONTAINER_IMAGE_CACHE: ghcr.io/swissdatasciencecenter/renku-data-services/devcontainer
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install python deps
+        uses: devcontainers/ci@v0.3
+        with:
+          runCmd: poetry install --with dev
+          push: always
+          skipContainerUserIdUpdate: false
+          imageName: ${{ env.DEVCONTAINER_IMAGE_CACHE }}
+          cacheFrom: ${{ env.DEVCONTAINER_IMAGE_CACHE }}
+      - uses: actions/cache/save@v3
+        name: Create cache
+        with:
+          path: ${{ env.CACHE_PATH }}
+          key: ${{ env.CACHE_KEY }}

.github/workflows/test_publish.yml

@@ -71,6 +71,58 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+      - name: Docker image metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.DEVCONTAINER_REGISTRY }}/${{ env.DEVCONTAINER_IMAGE_NAME }}
+          tags: |
+            type=ref,event=pr,prefix=cache-pr-,priority=600
+            type=ref,event=branch,prefix=cache-,priority=500
+            type=ref,event=tag,prefix=cache-,priority=500
+          flavor: |
+            latest=false
+      - name: Extract Docker image name
+        id: docker_image
+        env:
+          IMAGE_TAGS: ${{ steps.meta.outputs.tags }}
+        run: |
+          IMAGE=$(echo "$IMAGE_TAGS" | cut -d" " -f1)
+          IMAGE_REPOSITORY=$(echo "$IMAGE" | cut -d":" -f1)
+          IMAGE_TAG=$(echo "$IMAGE" | cut -d":" -f2)
+          echo "image=$IMAGE" >> "$GITHUB_OUTPUT"
+          echo "image_repository=$IMAGE_REPOSITORY" >> "$GITHUB_OUTPUT"
+          echo "image_tag=$IMAGE_TAG" >> "$GITHUB_OUTPUT"
+      - uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Pre-build devcontainer
+        uses: devcontainers/ci@v0.3
+        continue-on-error: true
+        with:
+          push: always
+          skipContainerUserIdUpdate: false
+          imageName: ${{ steps.docker_image.outputs.image_repository }}
+          imageTag: ${{ steps.docker_image.outputs.image_tag }}
+          cacheFrom: |
+            ${{ steps.docker_image.outputs.image_repository }}:${{ steps.docker_image.outputs.image_tag }}
+            ${{ steps.docker_image.outputs.image_repository }}:${{ env.DEVCONTAINER_IMAGE_TAG_MAIN }}
+
+  style-checks:
+    runs-on: ubuntu-latest
+    needs:
+      - build-devcontainer
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: actions/cache/restore@v3
+        name: Restore cache
+        with:
+          path: ${{ env.CACHE_PATH }}
+          key: ${{ env.CACHE_KEY }}
       - name: Set Git config
         shell: bash
         run: |
@@ -99,6 +151,11 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+      - uses: actions/cache/restore@v3
+        name: Restore cache
+        with:
+          path: ${{ env.CACHE_PATH }}
+          key: ${{ env.CACHE_KEY }}
       - name: Set Git config
         shell: bash
         run: |
@@ -143,6 +200,11 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+      - uses: actions/cache/restore@v3
+        name: Restore cache
+        with:
+          path: ${{ env.CACHE_PATH }}
+          key: ${{ env.CACHE_KEY }}
       - name: Set Git config
         shell: bash
         run: |

Makefile

@@ -1,7 +1,7 @@
 .PHONY: schemas tests test_setup main_tests schemathesis_tests collect_coverage style_checks pre_commit_checks run download_avro check_avro avro_models update_avro kind_cluster install_amaltheas all
 
-AMALTHEA_JS_VERSION ?= 0.11.0
-AMALTHEA_SESSIONS_VERSION ?= 0.0.1-new-operator-chart
+AMALTHEA_JS_VERSION ?= 0.12.2
+AMALTHEA_SESSIONS_VERSION ?= 0.0.9-new-operator-chart
 codegen_params = --input-file-type openapi --output-model-type pydantic_v2.BaseModel --use-double-quotes --target-python-version 3.12 --collapse-root-models --field-constraints --strict-nullable --set-default-enum-member --openapi-scopes schemas paths parameters --set-default-enum-member --use-one-literal-as-default --use-default
 
 define test_apispec_up_to_date
@@ -153,7 +153,13 @@ kind_cluster:  ## Creates a kind cluster for testing
 	sleep 15
 	kubectl wait --namespace ingress-nginx --for=condition=ready pod --selector=app.kubernetes.io/component=controller --timeout=90s
 
-install_amaltheas:  ## Installs both version of amalthea in the currently active k8s context.
+install_amaltheas:  ## Installs both version of amalthea in the. NOTE: It uses the currently active k8s context.
 	helm repo add renku https://swissdatasciencecenter.github.io/helm-charts
-	helm install amalthea-js renku/amalthea --version $(AMALTHEA_JS_VERSION)
-	helm install amalthea-sessions renku/amalthea-sessions --version $(AMALTHEA_SESSIONS_VERSION)
+	helm repo update
+	helm upgrade --install amalthea-js renku/amalthea --version $(AMALTHEA_JS_VERSION)
+	helm upgrade --install amalthea-sessions amalthea-sessions-0.0.9-new-operator-chart.tgz --version $(AMALTHEA_SESSIONS_VERSION)
+
+# TODO: Add the version variables from the top of the file here when the charts are fully published
+amalthea_schema:  ## Updates generates pydantic classes from CRDs
+	curl https://raw.githubusercontent.com/SwissDataScienceCenter/amalthea/fix-missing-status/config/crd/bases/amalthea.dev_amaltheasessions.yaml | yq '.spec.versions[0].schema.openAPIV3Schema' | poetry run datamodel-codegen --input-file-type jsonschema --output-model-type pydantic_v2.BaseModel --output components/renku_data_services/notebooks/cr_amalthea_session.py --use-double-quotes --target-python-version 3.12 --collapse-root-models --field-constraints --strict-nullable --base-class renku_data_services.notebooks.cr_base.BaseCRD --allow-extra-fields --use-default-kwarg
+	curl https://raw.githubusercontent.com/SwissDataScienceCenter/amalthea/main/controller/crds/jupyter_server.yaml | yq '.spec.versions[0].schema.openAPIV3Schema' | poetry run datamodel-codegen --input-file-type jsonschema --output-model-type pydantic_v2.BaseModel --output components/renku_data_services/notebooks/cr_jupyter_server.py --use-double-quotes --target-python-version 3.12 --collapse-root-models --field-constraints --strict-nullable --base-class renku_data_services.notebooks.cr_base.BaseCRD --allow-extra-fields --use-default-kwarg

bases/renku_data_services/data_api/app.py

@@ -16,6 +16,7 @@
     UserResourcePoolsBP,
 )
 from renku_data_services.namespace.blueprints import GroupsBP
+from renku_data_services.notebooks.blueprints import NotebooksBP, NotebooksNewBP
 from renku_data_services.platform.blueprints import PlatformConfigBP
 from renku_data_services.project.blueprints import ProjectsBP
 from renku_data_services.repositories.blueprints import RepositoriesBP
@@ -134,6 +135,24 @@ def register_all_handlers(app: Sanic, config: Config) -> Sanic:
         authenticator=config.authenticator,
         internal_gitlab_authenticator=config.gitlab_authenticator,
     )
+    notebooks = NotebooksBP(
+        name="notebooks_old",
+        url_prefix=url_prefix,
+        authenticator=config.authenticator,
+        nb_config=config.nb_config,
+        internal_gitlab_authenticator=config.gitlab_authenticator,
+        git_repo=config.git_repositories_repo,
+    )
+    notebooks_new = NotebooksNewBP(
+        name="notebooks",
+        url_prefix=url_prefix,
+        authenticator=config.authenticator,
+        nb_config=config.nb_config,
+        project_repo=config.project_repo,
+        session_repo=config.session_repo,
+        rp_repo=config.rp_repo,
+        internal_gitlab_authenticator=config.gitlab_authenticator,
+    )
     platform_config = PlatformConfigBP(
         name="platform_config",
         url_prefix=url_prefix,
@@ -161,6 +180,8 @@ def register_all_handlers(app: Sanic, config: Config) -> Sanic:
             oauth2_clients.blueprint(),
             oauth2_connections.blueprint(),
             repositories.blueprint(),
+            notebooks.blueprint(),
+            notebooks_new.blueprint(),
             platform_config.blueprint(),
         ]
     )

components/renku_data_services/app_config/config.py

@@ -52,6 +52,7 @@
 from renku_data_services.message_queue.interface import IMessageQueue
 from renku_data_services.message_queue.redis_queue import RedisQueue
 from renku_data_services.namespace.db import GroupRepository
+from renku_data_services.notebooks.config import _NotebooksConfig
 from renku_data_services.platform.db import PlatformRepository
 from renku_data_services.project.db import ProjectMemberRepository, ProjectRepository
 from renku_data_services.repositories.db import GitRepositoriesRepository
@@ -144,6 +145,7 @@ class Config:
     kc_api: IKeycloakAPI
     message_queue: IMessageQueue
     gitlab_url: str | None
+    nb_config: _NotebooksConfig
 
     secrets_service_public_key: rsa.RSAPublicKey
     """The public key of the secrets service, used to encrypt user secrets that only it can decrypt."""
@@ -208,6 +210,10 @@ def __post_init__(self) -> None:
         with open(spec_file) as f:
             repositories = safe_load(f)
 
+        spec_file = Path(renku_data_services.notebooks.__file__).resolve().parent / "api.spec.yaml"
+        with open(spec_file) as f:
+            repositories = safe_load(f)
+
         spec_file = Path(renku_data_services.platform.__file__).resolve().parent / "api.spec.yaml"
         with open(spec_file) as f:
             platform = safe_load(f)
@@ -408,8 +414,8 @@ def from_env(cls, prefix: str = "") -> "Config":
         gitlab_client: base_models.GitlabAPIProtocol
         user_preferences_config: UserPreferencesConfig
         version = os.environ.get(f"{prefix}VERSION", "0.0.1")
-        server_options_file = os.environ.get("SERVER_OPTIONS")
-        server_defaults_file = os.environ.get("SERVER_DEFAULTS")
+        server_options_file = os.environ.get("NB_SERVER_OPTIONS__UI_CHOICES_PATH")
+        server_defaults_file = os.environ.get("NB_SERVER_OPTIONS__DEFAULTS_PATH")
         k8s_namespace = os.environ.get("K8S_NAMESPACE", "default")
         max_pinned_projects = int(os.environ.get(f"{prefix}MAX_PINNED_PROJECTS", "10"))
         user_preferences_config = UserPreferencesConfig(max_pinned_projects=max_pinned_projects)
@@ -491,6 +497,7 @@ def from_env(cls, prefix: str = "") -> "Config":
         sentry = SentryConfig.from_env(prefix)
         trusted_proxies = TrustedProxiesConfig.from_env(prefix)
         message_queue = RedisQueue(redis)
+        nb_config = _NotebooksConfig.from_env(db)
 
         return cls(
             version=version,
@@ -511,4 +518,5 @@ def from_env(cls, prefix: str = "") -> "Config":
             encryption_key=encryption_key,
             secrets_service_public_key=secrets_service_public_key,
             gitlab_url=gitlab_url,
+            nb_config=nb_config,
         )

components/renku_data_services/authn/dummy.py

@@ -57,11 +57,12 @@ async def authenticate(access_token: str, request: Request) -> base_models.APIUs
         )
 
         return base_models.APIUser(
-            is_admin=user_props.get("is_admin", False),
+            is_admin_init=user_props.get("is_admin", False),
             id=user_props.get("id", "some-id") if is_set else None,
             access_token=access_token,
             first_name=user_props.get("first_name", "John") if is_set else None,
             last_name=user_props.get("last_name", "Doe") if is_set else None,
             email=user_props.get("email", "john.doe@gmail.com") if is_set else None,
             full_name=user_props.get("full_name", "John Doe") if is_set else None,
+            refresh_token=request.headers.get("Renku-Auth-Refresh-Token"),
         )