fix: cloning repos token urls (#1253)

Author: olevski

bases/renku_data_services/data_api/config.py

@@ -63,6 +63,7 @@ def from_env(cls, db: DBConfig | None = None) -> Self:
             else:
                 gitlab_url = None
 
+        nb_config = NotebooksConfig.from_env(db, enable_internal_gitlab=enable_internal_gitlab)
         return cls(
             enable_internal_gitlab=enable_internal_gitlab,
             version=os.environ.get("VERSION", "0.0.1"),
@@ -71,7 +72,7 @@ def from_env(cls, db: DBConfig | None = None) -> Self:
             k8s_config_root=os.environ.get("K8S_CONFIGS_ROOT", "/secrets/kube_configs"),
             db=db,
             builds=BuildsConfig.from_env(),
-            nb_config=NotebooksConfig.from_env(db, enable_internal_gitlab=enable_internal_gitlab),
+            nb_config=nb_config,
             secrets=PublicSecretsConfig.from_env(),
             sentry=SentryConfig.from_env(),
             posthog=PosthogConfig.from_env(),
@@ -83,5 +84,5 @@ def from_env(cls, db: DBConfig | None = None) -> Self:
             gitlab_url=gitlab_url,
             log_cfg=LoggingConfig.from_env(),
             alertmanager_webhook_role=os.environ.get("ALERTMANAGER_WEBHOOK_ROLE", "alertmanager-webhook"),
-            deposit_config=DepositConfig.from_env(),
+            deposit_config=DepositConfig.from_env(nb_config.sessions.renku_url),
         )

components/renku_data_services/data_connectors/config.py

@@ -29,7 +29,7 @@ class DepositConfig:
     cluster_id: Final[ClusterId] = DEFAULT_K8S_CLUSTER
 
     @classmethod
-    def from_env(cls) -> DepositConfig:
+    def from_env(cls, renku_url: str) -> DepositConfig:
         """Create a data deposit configuration from environment variables."""
         # NOTE: The deserialize method from the K8s API client needs the json payload to be in
         # a `data` property on the object passed into the deserializer.
@@ -57,8 +57,8 @@ def from_env(cls) -> DepositConfig:
                     "Could not validate DATA_DEPOSITS_NODE_TOLERATIONS. Will not use tolerations for data upload jobs."
                 )
         return cls(
-            image=os.environ["DATA_DEPOSITS_JOB_IMAGE"],
-            renku_url=os.environ["RENKU_URL"],
+            image=os.environ.get("DATA_DEPOSITS_JOB_IMAGE", "ghcr.io/swissdatasciencecenter/renku-cli"),
+            renku_url=renku_url,
             tolerations=tolerations,
             node_selector=node_selector,
             namespace=os.environ["KUBERNETES_NAMESPACE"],

components/renku_data_services/notebooks/api/classes/data_service.py

@@ -154,7 +154,7 @@ def create(cls, csr: ConnectedServicesRepository, enable_internal_gitlab: bool)
         return GitProviderHelper(
             connected_services_repo=csr,
             service_url=data_service_url,
-            renku_url=f"http://{sessions_config.ingress.host}",
+            renku_url=sessions_config.renku_url,
             internal_gitlab_url=git_config.url,
             enable_internal_gitlab=enable_internal_gitlab,
         )

components/renku_data_services/notebooks/config/dynamic.py

@@ -7,7 +7,7 @@
 from dataclasses import dataclass, field
 from enum import Enum
 from io import StringIO
-from typing import Any, ClassVar, Self, Union
+from typing import Any, ClassVar, Literal, Self, Union
 
 import yaml
 from kubernetes import client
@@ -256,6 +256,16 @@ def from_env(cls) -> Self:
             annotations=yaml.safe_load(StringIO(os.environ.get("NB_SESSIONS__INGRESS__ANNOTATIONS", "{}"))),
         )
 
+    @property
+    def scheme(self) -> Literal["https"] | Literal["http"]:
+        if self.tls_secret or self.use_default_cluster_tls_cert:
+            return "https"
+        return "http"
+
+    @property
+    def renku_url(self) -> str:
+        return f"{self.scheme}::{self.host}"
+
 
 @dataclass
 class _GenericCullingConfig:
@@ -356,6 +366,11 @@ def from_env(cls) -> Self:
         )
 
 
+def _get_renku_url_fallback(ingress_config: _SessionIngress) -> str:
+    scheme = "https" if ingress_config.tls_secret or ingress_config.use_default_cluster_tls_cert else "http"
+    return f"{scheme}::{ingress_config.host}"
+
+
 @dataclass
 class _SessionConfig:
     culling: _SessionCullingConfig
@@ -368,6 +383,7 @@ class _SessionConfig:
     storage: _SessionStorageConfig
     containers: _SessionContainers
     ssh: _SessionSshConfig
+    renku_url: str
     default_image: str = "renku/singleuser:latest"
     enforce_cpu_limits: CPUEnforcement = CPUEnforcement.OFF
     termination_warning_duration_seconds: int = 12 * 60 * 60
@@ -386,12 +402,13 @@ class _SessionConfig:
 
     @classmethod
     def from_env(cls) -> Self:
+        ingress = _SessionIngress.from_env()
         return cls(
             culling=_SessionCullingConfig.from_env(),
             git_proxy=_GitProxyConfig.from_env(),
             git_rpc_server=_GitRpcServerConfig.from_env(),
             git_clone=_GitCloneConfig.from_env(),
-            ingress=_SessionIngress.from_env(),
+            ingress=ingress,
             ca_certs=_CustomCaCertsConfig.from_env(),
             oidc=_SessionOidcConfig.from_env(),
             storage=_SessionStorageConfig.from_env(),
@@ -404,16 +421,18 @@ def from_env(cls) -> Self:
             node_selector=yaml.safe_load(StringIO(os.environ.get("NB_SESSIONS__NODE_SELECTOR", "{}"))),
             affinity=yaml.safe_load(StringIO(os.environ.get("NB_SESSIONS__AFFINITY", "{}"))),
             tolerations=yaml.safe_load(StringIO(os.environ.get("NB_SESSIONS__TOLERATIONS", "[]"))),
+            renku_url=os.environ.get("RENKU_URL") or _get_renku_url_fallback(ingress),
         )
 
     @classmethod
     def _for_testing(cls) -> Self:
+        ingress = _SessionIngress(host="localhost", tls_secret="some-secret")  # nosec: B106
         return cls(
             culling=_SessionCullingConfig.from_env(),
             git_proxy=_GitProxyConfig(renku_client_secret="not-defined"),  # nosec B106
             git_rpc_server=_GitRpcServerConfig.from_env(),
             git_clone=_GitCloneConfig.from_env(),
-            ingress=_SessionIngress(host="localhost", tls_secret="some-secret"),  # nosec: B106
+            ingress=ingress,
             ca_certs=_CustomCaCertsConfig.from_env(),
             oidc=_SessionOidcConfig(
                 client_id="not-defined",
@@ -432,6 +451,7 @@ def _for_testing(cls) -> Self:
             node_selector=yaml.safe_load(StringIO(os.environ.get("", "{}"))),
             affinity=yaml.safe_load(StringIO(os.environ.get("", "{}"))),
             tolerations=yaml.safe_load(StringIO(os.environ.get("", "[]"))),
+            renku_url=os.environ.get("RENKU_URL") or _get_renku_url_fallback(ingress),
         )
 
     @property