Skip to content

Commit 055f19d

Browse files
committed
Enable read-write storage for projects (#1378)
- first draw of enabling read-write storage for projects - admins must put projects into an allow list first - owners then can create one storage for such projects - the storage is mounted read-write for users who have write access to the project and read-only otherwise
1 parent 378fc59 commit 055f19d

29 files changed

Lines changed: 2862 additions & 21 deletions

File tree

bases/renku_data_services/data_api/app.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,7 @@ def register_all_handlers(app: Sanic, dm: DependencyManager) -> Sanic:
144144
session_repo=dm.session_repo,
145145
session_secret_repo=dm.project_session_secret_repo,
146146
metrics=dm.metrics,
147+
project_storage_k8s=dm.project_storage_k8s,
147148
)
148149
project_session_secrets = ProjectSessionSecretBP(
149150
name="project_session_secrets",
@@ -229,6 +230,7 @@ def register_all_handlers(app: Sanic, dm: DependencyManager) -> Sanic:
229230
internal_token_mint=dm.internal_token_mint,
230231
resource_usage_service=dm.resource_usage_service,
231232
resource_requests_repo=dm.resource_requests_repo,
233+
authz=dm.authz,
232234
)
233235
platform_config = PlatformConfigBP(
234236
name="platform_config",
@@ -278,6 +280,7 @@ def register_all_handlers(app: Sanic, dm: DependencyManager) -> Sanic:
278280
data_service_base_url=dm.config.nb_config.data_service_url,
279281
k8s_client=dm.k8s_client,
280282
deposit_config=dm.config.deposit_config,
283+
project_storage_k8s=dm.project_storage_k8s,
281284
)
282285
notifications = NotificationsBP(
283286
name="notifications",

bases/renku_data_services/data_api/config.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
)
1515
from renku_data_services.app_config.logging import Config as LoggingConfig
1616
from renku_data_services.authz.config import AuthzConfig
17-
from renku_data_services.data_connectors.config import DepositConfig
17+
from renku_data_services.data_connectors.config import DepositConfig, ProjectStorageConfig
1818
from renku_data_services.db_config.config import DBConfig
1919
from renku_data_services.notebooks.config import NotebooksConfig
2020
from renku_data_services.secrets.config import PublicSecretsConfig
@@ -48,6 +48,7 @@ class Config:
4848
version: str
4949
alertmanager_webhook_role: str
5050
deposit_config: DepositConfig
51+
project_storage_config: ProjectStorageConfig
5152

5253
@classmethod
5354
def from_env(cls, db: DBConfig | None = None) -> Self:
@@ -95,4 +96,5 @@ def from_env(cls, db: DBConfig | None = None) -> Self:
9596
log_cfg=LoggingConfig.from_env(),
9697
alertmanager_webhook_role=os.environ.get("ALERTMANAGER_WEBHOOK_ROLE", "alertmanager-webhook"),
9798
deposit_config=DepositConfig.from_env(nb_config.sessions.renku_url),
99+
project_storage_config=ProjectStorageConfig.from_env(),
98100
)

bases/renku_data_services/data_api/dependencies.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
)
4141
from renku_data_services.data_connectors.deposits.envidat import EnvidatClient
4242
from renku_data_services.data_connectors.deposits.zenodo import ZenodoAPIClient
43+
from renku_data_services.data_connectors.project_storage_k8s import ProjectStorageK8s
4344
from renku_data_services.git.gitlab import DummyGitlabAPI, EmptyGitlabAPI, GitlabAPI
4445
from renku_data_services.k8s.client_interfaces import K8sClient
4546
from renku_data_services.k8s.clients import (
@@ -175,6 +176,7 @@ class DependencyManager:
175176
secret_client: K8sSecretClient
176177
internal_token_mint: RenkuSelfTokenMint
177178
internal_scope_verifier: ScopeVerifier
179+
project_storage_k8s: ProjectStorageK8s
178180

179181
spec: dict[str, Any] = field(init=False, repr=False, default_factory=dict)
180182
app_name: str = "renku_data_services"
@@ -364,6 +366,7 @@ def from_env(cls) -> DependencyManager:
364366
resource_requests_repo=resource_requests_repo,
365367
member_repo=member_repo,
366368
)
369+
project_storage_k8s = ProjectStorageK8s(config.nb_config.k8s_v2_client)
367370
reprovisioning_repo = ReprovisioningRepository(session_maker=config.db.async_session_maker)
368371

369372
git_repositories_repo = GitRepositoriesRepository(
@@ -420,6 +423,7 @@ def from_env(cls) -> DependencyManager:
420423
project_repo=project_repo,
421424
group_repo=group_repo,
422425
search_updates_repo=search_updates_repo,
426+
project_storage_config=config.project_storage_config,
423427
)
424428
data_connector_secret_repo = DataConnectorSecretRepository(
425429
session_maker=config.db.async_session_maker,
@@ -513,4 +517,5 @@ def from_env(cls) -> DependencyManager:
513517
secret_client=secret_client,
514518
internal_token_mint=internal_token_mint,
515519
internal_scope_verifier=internal_scope_verifier,
520+
project_storage_k8s=project_storage_k8s,
516521
)
Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
"""Byte size model with unit conversions."""
2+
3+
from __future__ import annotations
4+
5+
from dataclasses import dataclass
6+
7+
8+
@dataclass(frozen=True, order=True)
9+
class ByteSize:
10+
"""Represents a size in bytes, with convenience conversions and formatting."""
11+
12+
value: int
13+
14+
# Binary (1024-based) unit thresholds
15+
KIBI = 1024
16+
MEBI = 1024**2
17+
GIBI = 1024**3
18+
TEBI = 1024**4
19+
20+
def __post_init__(self) -> None:
21+
if self.value < 0:
22+
raise ValueError(f"ByteSize cannot be negative: {self.value}")
23+
if not isinstance(self.value, int):
24+
raise TypeError(f"ByteSize value must be int, got {type(self.value).__name__}")
25+
26+
def to_bytes(self) -> int:
27+
"""Return the size in bytes."""
28+
return self.value
29+
30+
def to_kibi(self) -> float:
31+
"""Return the size in kibibytes (KiB)."""
32+
return self.value / self.KIBI
33+
34+
def to_mibi(self) -> float:
35+
"""Return the size in mebibytes (MiB)."""
36+
return self.value / self.MEBI
37+
38+
def to_gibi(self) -> float:
39+
"""Return the size in gibibytes (GiB)."""
40+
return self.value / self.GIBI
41+
42+
def to_tebi(self) -> float:
43+
"""Return the size in tebibytes (TiB)."""
44+
return self.value / self.TEBI
45+
46+
def to_human(self) -> str:
47+
"""Return a human-readable string with the appropriate binary unit."""
48+
if self.value < self.KIBI:
49+
return f"{self.value}B"
50+
elif self.value < self.MEBI:
51+
return f"{self.to_kibi():.2f}KiB"
52+
elif self.value < self.GIBI:
53+
return f"{self.to_mibi():.2f}MiB"
54+
elif self.value < self.TEBI:
55+
return f"{self.to_gibi():.2f}GiB"
56+
else:
57+
return f"{self.to_tebi():.2f}TiB"
58+
59+
def __str__(self) -> str:
60+
return self.to_human()
61+
62+
def __repr__(self) -> str:
63+
return f"ByteSize({self.value}B)"
64+
65+
def __add__(self, other: ByteSize) -> ByteSize:
66+
return ByteSize(self.value + other.value)
67+
68+
def __sub__(self, other: ByteSize) -> ByteSize:
69+
result = self.value - other.value
70+
if result < 0:
71+
raise ValueError("Subtraction would result in negative ByteSize")
72+
return ByteSize(result)
73+
74+
def __radd__(self, other: int) -> ByteSize:
75+
# allows sum([ByteSize(1), ByteSize(2)]) to work, since sum() starts with 0
76+
if other == 0:
77+
return self
78+
return NotImplemented
79+
80+
@classmethod
81+
def from_bytes(cls, bs: int) -> ByteSize:
82+
"""Create a ByteSize from a byte count."""
83+
return ByteSize(value=bs)
84+
85+
@classmethod
86+
def from_kibi(cls, kb: float) -> ByteSize:
87+
"""Create a ByteSize from a kibibyte value."""
88+
return ByteSize(value=int(kb * cls.KIBI))
89+
90+
@classmethod
91+
def from_mibi(cls, mb: float) -> ByteSize:
92+
"""Create a ByteSize from a mebibyte value."""
93+
return ByteSize(value=int(mb * cls.MEBI))
94+
95+
@classmethod
96+
def from_gibi(cls, gb: float) -> ByteSize:
97+
"""Create a ByteSize from a gibibyte value."""
98+
return ByteSize(value=int(gb * cls.GIBI))
99+
100+
@classmethod
101+
def from_tebi(cls, tib: float) -> ByteSize:
102+
"""Create a ByteSize from a tebibyte value."""
103+
return cls(value=round(tib * cls.TEBI))
104+
105+
@classmethod
106+
def zero(cls) -> ByteSize:
107+
"""Create a byte size with value 0."""
108+
return ByteSize(0)

components/renku_data_services/base_models/core.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -316,6 +316,12 @@ def from_strings(cls, *slugs: str) -> Self:
316316
raise errors.ValidationError(message=f"Two slug strings are needed to create a project path, got {slugs}.")
317317
return cls(NamespaceSlug(slugs[0]), ProjectSlug(slugs[1]))
318318

319+
@classmethod
320+
def parse(cls, slug: str) -> Self:
321+
"""Parses a single string into a ProjectPath."""
322+
namespace_split = slug.split("/")
323+
return cls.from_strings(*namespace_split)
324+
319325

320326
@dataclass(frozen=True, eq=True, repr=False)
321327
class DataConnectorPath(__NamespaceCommonMixin):

0 commit comments

Comments
 (0)