From 21efcf4b10ab94fca4b832eb0b15e44a9e6debcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E7=84=B6?= Date: Fri, 5 Jun 2026 17:44:47 +0800 Subject: [PATCH 1/6] feat(server): implement multi-tenancy provider with file and HTTP backends Wire TenantProvider interface through full request chain: - TenantProvider Protocol + FileTenantProvider + HTTPTenantProvider - Auth middleware multi-tenant/single-tenant mode branching - ContextVar-based tenant propagation (tenants/context.py) - Dynamic namespace resolution in KubernetesSandboxService - [tenants] config section in server.toml (provider=file|http) - Startup guards: docker+tenants fatal, api_key+tenants mutual exclusion - HTTP provider: per-key TTL cache, sync fetch, max_stale, 401 eviction - 27 e2e tests covering both providers end-to-end - Update OSEP-0012 with formal interface spec and HTTP contract Co-Authored-By: Claude Opus 4.6 (1M context) --- oseps/0012-multi-tenancy.md | 183 ++++-- server/opensandbox_server/config.py | 41 ++ server/opensandbox_server/main.py | 55 +- server/opensandbox_server/middleware/auth.py | 98 +-- .../services/k8s/kubernetes_service.py | 47 +- server/opensandbox_server/tenants/__init__.py | 41 ++ server/opensandbox_server/tenants/context.py | 32 + .../tenants/file_provider.py | 181 ++++++ .../tenants/http_provider.py | 187 ++++++ server/opensandbox_server/tenants/models.py | 25 + server/opensandbox_server/tenants/provider.py | 66 +++ server/tests/test_multi_tenancy.py | 558 ++++++++++++++++++ 12 files changed, 1400 insertions(+), 114 deletions(-) create mode 100644 server/opensandbox_server/tenants/__init__.py create mode 100644 server/opensandbox_server/tenants/context.py create mode 100644 server/opensandbox_server/tenants/file_provider.py create mode 100644 server/opensandbox_server/tenants/http_provider.py create mode 100644 server/opensandbox_server/tenants/models.py create mode 100644 server/opensandbox_server/tenants/provider.py create mode 100644 server/tests/test_multi_tenancy.py diff --git a/oseps/0012-multi-tenancy.md b/oseps/0012-multi-tenancy.md index 98d2d1b9e..dfd4380a5 100644 --- a/oseps/0012-multi-tenancy.md +++ b/oseps/0012-multi-tenancy.md @@ -20,7 +20,7 @@ status: draft - [Risks and Mitigations](#risks-and-mitigations) - [Design Details](#design-details) - [TenantProvider Abstraction](#tenantprovider-abstraction) - - [Config Model & Loading Flow (FileTenantProvider)](#config-model--loading-flow-filetenantprovider) + - [Tenants Config File Format (FileTenantProvider)](#tenants-config-file-format-filetenantprovider) - [Auth Middleware Flow](#auth-middleware-flow) - [Sandbox Service — Namespace Resolution](#sandbox-service--namespace-resolution) - [Startup Guards](#startup-guards) @@ -149,77 +149,162 @@ Implementation in 6 steps. No step blocks another except where noted. Tenant resolution is behind a `TenantProvider` interface, decoupling auth middleware from any specific config source. This lets the initial implementation ship with a simple file-based provider while leaving a clean extension point for enterprise deployments that already manage tenants in an external IAM or tenant management system. -**Interface (pseudocode):** +**Interface (`opensandbox_server/tenants/provider.py`):** + +```python +class TenantProvider(Protocol): + def lookup(self, api_key: str) -> Optional[TenantEntry]: + """Resolve API key → tenant. Returns None if not recognized. + Raises TenantProviderUnavailable if provider cannot serve.""" + ... + + def list_tenants(self) -> List[TenantEntry]: + """All known tenant entries (startup validation).""" + ... + + def ready(self) -> bool: + """True once provider can serve lookups.""" + ... + + def start(self) -> None: + """Start background resources (watchers, connections). Called at server startup.""" + ... + + def close(self) -> None: + """Release resources. Called on server shutdown.""" + ... + + def on_reload(self, callback: Callable[[List[TenantEntry]], None]) -> None: + """Register callback invoked on tenant data change. + Not all providers support this; those that don't may ignore.""" + ... ``` -TenantProvider (Protocol): - lookup(api_key: str) → TenantEntry | None - list_tenants() → list[TenantEntry] # for startup validation - ready() → bool # provider has loaded initial state - on_reload(callback) → None # notify consumers on config change (optional) + +**Exception:** +- `TenantProviderUnavailable` — raised when provider cannot serve lookups (e.g., HTTP endpoint unreachable + cache expired beyond `max_stale_seconds`) + +**Data model (`opensandbox_server/tenants/models.py`):** + +```python +@dataclass(frozen=True) +class TenantEntry: + name: str + namespace: str + api_keys: List[str] ``` -**Initial provider — FileTenantProvider:** -- Backed by `tenants.toml`, loaded at startup, hot-reloaded via fsnotify +--- + +#### Provider 1 — FileTenantProvider + +Backed by `tenants.toml`, loaded at startup, hot-reloaded via filesystem mtime polling. + - Implements full `TenantProvider` interface +- `start()` parses file and starts watcher thread (2s mtime poll) - `ready()` returns `True` after initial file parse succeeds -- `on_reload` triggers on fsnotify events; auth middleware picks up new key→tenant mappings without restart +- `on_reload` triggers on file change; auth middleware picks up new key→tenant mappings without restart +- File delete → all entries cleared (all tenant keys → 401) +- Parse error during reload → log warning, keep previous state (no downtime) +- Watcher monitors parent directory for ConfigMap atomic symlink swap -**Future providers (not in this OSEP, but the interface accommodates):** -- `HTTPTenantProvider` — polls or streams from an internal IAM API; tenant metadata, key rotation, enable/disable all managed in the external system -- `K8sConfigMapProvider` — watches a ConfigMap or Secret across namespaces -- Composite/chained providers for fallback (e.g., file + external API merge) +--- -**Startup wiring (pseudocode):** +#### Provider 2 — HTTPTenantProvider + +Per-key lookup against a remote HTTP endpoint with in-memory TTL cache. No background thread, no file persistence, no bulk fetch. Keys not looked up are not cached. + +**Endpoint contract:** + +``` +GET {endpoint} +Header: OPEN-SANDBOX-API-KEY: // 客户端原始 key 原封不动转发 + +200 OK: +{ + "namespace": "ns-a", // target K8s namespace for this key + "ttl": 60 // suggested cache duration in seconds +} + +401 Unauthorized: +{ + "code": "UNAUTHORIZED", + "message": "API key is invalid or revoked" +} ``` + +Server 将客户端的 `OPEN-SANDBOX-API-KEY` 原封不动转发给 HTTP provider 做校验。Provider 是权威方 — 决定 key 是否有效、映射到哪个 namespace。Server 只需要 `namespace` + `ttl`。 + +**Cache behavior:** + +| Scenario | Action | +|----------|--------| +| Cache hit + within server-suggested TTL | Return cached entry immediately | +| Cache hit + TTL expired | Sync GET → success: update cache with new TTL; failure + within `max_stale_seconds`: return stale; failure + beyond `max_stale_seconds`: raise `TenantProviderUnavailable` | +| Cache miss | Sync GET → 200: cache + return; 401: return `None`; network error: raise `TenantProviderUnavailable` | +| Remote returns 401 for previously cached key | Evict from cache + return `None` (key revoked) | + +**Configuration (`HTTPTenantProviderConfig`):** + +| Field | Default | Description | +|-------|---------|-------------| +| `endpoint` | (required) | Remote tenant lookup URL | +| `max_stale_seconds` | 300 | Maximum time to serve stale cache when endpoint unreachable | +| `timeout_seconds` | 5 | HTTP request timeout | +| `auth_header` | None | Optional header name for provider-level authentication | +| `auth_token` | None | Optional token value for provider-level authentication | + +**Security properties:** +- No persistent cache file → no disk attack surface, no stale file after long downtime +- Cold start (`start()`) only marks ready, does not bulk-fetch (per-key on demand) +- Revoked key (401) immediately evicted from cache +- Max stale bounds the window where unreachable endpoint + stale cache could allow a revoked key + +--- + +#### Provider Selection + +Provider type is determined at startup: + +```python +# Config field: tenant_provider_type = "file" | "http" +# Or auto-detect: if tenants.toml exists: provider = FileTenantProvider(path) - if not provider.ready(): - → SystemExit (parse error, duplicates, etc.) +elif http_tenant_endpoint configured: + provider = HTTPTenantProvider(config) else: provider = None # single-tenant mode + +provider.start() +if not provider.ready(): + → SystemExit ``` -Auth middleware depends only on `TenantProvider`, not on `FileTenantProvider` directly. Switching backends in the future does not touch auth code. +Auth middleware depends only on `TenantProvider`, not on any specific implementation. Switching backends does not touch auth code. --- -### Config Model & Loading Flow (FileTenantProvider) +### Tenants Config File Format (FileTenantProvider) -**New package:** `opensandbox_server/tenants/` +**Package:** `opensandbox_server/tenants/` -This is the initial `TenantProvider` implementation. It reads `tenants.toml` and hot-reloads on file changes. +**File:** `tenants.toml` (path resolved via `SANDBOX_TENANTS_CONFIG_PATH` env or default `~/.opensandbox/tenants.toml`) -**Data model (pseudocode):** -``` -TenantEntry: - - name: str - - namespace: str - - api_keys: list[str] - -TenantsConfig: - - entries: list[TenantEntry] - - validation: reject duplicate api_keys across tenants (on parse) -``` +```toml +[[tenants]] +name = "team-a" +namespace = "sandbox-team-a" +api_keys = ["sk-a-1", "sk-a-2"] -**Loading flow:** -``` -FileTenantProvider(path): - 1. resolve path: env SANDBOX_TENANTS_CONFIG_PATH || ~/.opensandbox/tenants.toml - 2. if file absent → ready() returns False → server stays in single-tenant mode - 3. parse TOML → TenantsConfig → build dict[api_key → TenantEntry] - 4. on parse error or duplicate keys → raise, server exits - 5. start fsnotify watcher thread for hot-reload +[[tenants]] +name = "team-b" +namespace = "sandbox-team-b" +api_keys = ["sk-b-1"] ``` -**Hot-reload behavior:** -``` - - maintains dict[api_key → TenantEntry] under threading.Lock - - on file change: reload atomically (swap dict under lock) - - on parse error during reload: log warning, keep old entries (no downtime) - - file delete → clear all entries (all tenant keys → 401) - - new key added → live immediately on next lookup -``` -Watcher monitors parent directory for ConfigMap atomic symlink swap. +**Validation rules (on parse):** +- Each tenant must have non-empty `name`, `namespace`, `api_keys` +- Duplicate `api_keys` across tenants → `ValueError`, server exits --- diff --git a/server/opensandbox_server/config.py b/server/opensandbox_server/config.py index 369c716d7..fe4299a1d 100644 --- a/server/opensandbox_server/config.py +++ b/server/opensandbox_server/config.py @@ -855,6 +855,43 @@ class StoreConfig(BaseModel): ) +class TenantsConfig(BaseModel): + """Multi-tenant provider configuration.""" + + provider: Literal["file", "http"] = Field( + default="file", + description="Tenant provider type: 'file' (tenants.toml) or 'http' (remote endpoint).", + ) + endpoint: Optional[str] = Field( + default=None, + description="HTTP tenant provider endpoint URL. Required when provider='http'.", + ) + max_stale_seconds: float = Field( + default=300.0, + ge=0, + description="Maximum seconds to serve stale cache when HTTP endpoint is unreachable.", + ) + timeout_seconds: float = Field( + default=5.0, + gt=0, + description="HTTP request timeout in seconds.", + ) + auth_header: Optional[str] = Field( + default=None, + description="Optional header name for provider-level authentication to HTTP endpoint.", + ) + auth_token: Optional[str] = Field( + default=None, + description="Optional token value for provider-level authentication to HTTP endpoint.", + ) + + @model_validator(mode="after") + def require_endpoint_for_http(self) -> "TenantsConfig": + if self.provider == "http" and not self.endpoint: + raise ValueError("[tenants] endpoint must be set when provider='http'.") + return self + + class AppConfig(BaseModel): """Root application configuration model.""" @@ -863,6 +900,10 @@ class AppConfig(BaseModel): default_factory=LogConfig, description="Logging configuration (level, file output, rotation).", ) + tenants: Optional[TenantsConfig] = Field( + default=None, + description="Multi-tenant configuration. When present, enables multi-tenant mode.", + ) renew_intent: RenewIntentConfig = Field( default_factory=RenewIntentConfig, description="Auto-renew sandbox expiration when reverse-proxy access is observed.", diff --git a/server/opensandbox_server/main.py b/server/opensandbox_server/main.py index f33633ed7..d1fbfa5ee 100644 --- a/server/opensandbox_server/main.py +++ b/server/opensandbox_server/main.py @@ -21,6 +21,7 @@ import logging import os +import sys from contextlib import asynccontextmanager from typing import Any @@ -34,11 +35,50 @@ from opensandbox_server.integrations.renew_intent import start_renew_intent_consumer from opensandbox_server.logging_config import configure_logging from opensandbox_server.startup_guard import api_key_confirm +from opensandbox_server.tenants import ( + FileTenantProvider, + HTTPTenantProvider, + HTTPTenantProviderConfig, + TenantProvider, + _resolve_tenants_path, +) # Load configuration before initializing routers/middleware app_config = load_config() _log_config = configure_logging(app_config.log) +# --- Multi-tenant provider initialization --- +_tenant_provider: TenantProvider | None = None + +if app_config.tenants is not None: + if app_config.runtime.type == "docker": + sys.exit( + "FATAL: [tenants] configured but runtime.type='docker'. " + "Multi-tenancy requires Kubernetes namespaces." + ) + if app_config.server.api_key and app_config.server.api_key.strip(): + sys.exit( + "FATAL: server.api_key must be removed from server.toml when using [tenants]. " + "Tenant API keys are managed by the tenant provider." + ) + + _tenants_cfg = app_config.tenants + if _tenants_cfg.provider == "file": + _tenants_path = _resolve_tenants_path() + _tenant_provider = FileTenantProvider(_tenants_path) + _tenant_provider.start() + elif _tenants_cfg.provider == "http": + _tenant_provider = HTTPTenantProvider( + HTTPTenantProviderConfig( + endpoint=_tenants_cfg.endpoint, + max_stale_seconds=_tenants_cfg.max_stale_seconds, + timeout_seconds=_tenants_cfg.timeout_seconds, + auth_header=_tenants_cfg.auth_header, + auth_token=_tenants_cfg.auth_token, + ) + ) + _tenant_provider.start() + from opensandbox_server.api.devops import router as devops_router # noqa: E402 from opensandbox_server.api.pool import router as pool_router # noqa: E402 from opensandbox_server.api.lifecycle import router, sandbox_service, snapshot_service # noqa: E402 @@ -55,11 +95,12 @@ @asynccontextmanager async def lifespan(app: FastAPI): - try: - api_key_confirm(configured_api_key=app_config.server.api_key) - except Exception as exc: - logger.error("API key startup confirmation failed: %s", exc) - os._exit(1) + if _tenant_provider is None: + try: + api_key_confirm(configured_api_key=app_config.server.api_key) + except Exception as exc: + logger.error("API key startup confirmation failed: %s", exc) + os._exit(1) from anyio.to_thread import current_default_thread_limiter @@ -114,6 +155,8 @@ async def lifespan(app: FastAPI): if consumer is not None: await consumer.stop() snapshot_service.close() + if _tenant_provider is not None: + _tenant_provider.close() await app.state.http_client.aclose() @@ -133,7 +176,7 @@ async def lifespan(app: FastAPI): # Middleware run in reverse order of addition: last added = first to run (outermost). # Add auth and CORS first so they run after RequestIdMiddleware. -app.add_middleware(AuthMiddleware, config=app_config) +app.add_middleware(AuthMiddleware, config=app_config, tenant_provider=_tenant_provider) app.add_middleware( CORSMiddleware, allow_origins=["*"], diff --git a/server/opensandbox_server/middleware/auth.py b/server/opensandbox_server/middleware/auth.py index 323bdc9d4..2817ab8d2 100644 --- a/server/opensandbox_server/middleware/auth.py +++ b/server/opensandbox_server/middleware/auth.py @@ -15,10 +15,12 @@ """ Authentication middleware for OpenSandbox Lifecycle API. -This module implements API Key authentication as specified in the OpenAPI spec. -API keys are configured via config.toml and validated against the OPEN-SANDBOX-API-KEY header. +Supports two modes: +- Single-tenant: validates against server.api_key (legacy) +- Multi-tenant: delegates to a TenantProvider for key→tenant resolution """ +import logging import re from typing import Callable, Optional @@ -27,6 +29,10 @@ from starlette.middleware.base import BaseHTTPMiddleware from opensandbox_server.config import AppConfig, get_config +from opensandbox_server.tenants.context import set_current_tenant +from opensandbox_server.tenants.provider import TenantProvider, TenantProviderUnavailable + +logger = logging.getLogger(__name__) SANDBOX_API_KEY_HEADER = "OPEN-SANDBOX-API-KEY" @@ -53,61 +59,40 @@ def _is_proxy_path(path: str) -> bool: return False return bool(AuthMiddleware._PROXY_PATH_RE.match(path)) - def __init__(self, app, config: Optional[AppConfig] = None): - """ - Initialize authentication middleware. - - Args: - app: FastAPI application instance - config: Optional application configuration (for dependency injection) - """ + def __init__( + self, + app, + config: Optional[AppConfig] = None, + tenant_provider: Optional[TenantProvider] = None, + ): super().__init__(app) self.config = config or get_config() - # Read the API key directly from config; suitable for dev/test usage + self.tenant_provider = tenant_provider self.valid_api_keys = self._load_api_keys() def _load_api_keys(self) -> set: - """ - Load valid API keys from configuration. - - Returns: - set: Set of valid API keys - """ - # Supports a single API key from config; extend later for secret managers api_key = self.config.server.api_key - # Treat empty string as no key configured if api_key and api_key.strip(): return {api_key} return set() - async def dispatch(self, request: Request, call_next: Callable) -> Response: - """ - Process each request and validate authentication. - - Args: - request: Incoming HTTP request - call_next: Next middleware or route handler + @property + def _is_multi_tenant(self) -> bool: + return self.tenant_provider is not None - Returns: - Response: HTTP response - """ - # Skip authentication for exempt paths + async def dispatch(self, request: Request, call_next: Callable) -> Response: if any(request.url.path.startswith(path) for path in self.EXEMPT_PATHS): return await call_next(request) - # Skip authentication only for the exact proxy-to-sandbox route shape - # (no path traversal, no loose substring match) if self._is_proxy_path(request.url.path): return await call_next(request) - # If no API keys are configured, skip authentication - if not self.valid_api_keys: + # If no API keys configured AND no tenant provider → skip auth + if not self._is_multi_tenant and not self.valid_api_keys: return await call_next(request) - # Extract API key from header api_key = request.headers.get(SANDBOX_API_KEY_HEADER) - # Validate API key if not api_key: return JSONResponse( status_code=status.HTTP_401_UNAUTHORIZED, @@ -118,7 +103,44 @@ async def dispatch(self, request: Request, call_next: Callable) -> Response: }, ) - # Enforce strict comparison whenever API keys are configured + if self._is_multi_tenant: + return await self._authenticate_multi_tenant(api_key, request, call_next) + else: + return await self._authenticate_single_tenant(api_key, request, call_next) + + async def _authenticate_multi_tenant( + self, api_key: str, request: Request, call_next: Callable + ) -> Response: + try: + tenant = self.tenant_provider.lookup(api_key) + except TenantProviderUnavailable as e: + logger.error("Tenant provider unavailable: %s", e) + return JSONResponse( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + content={ + "code": "TENANT_PROVIDER_UNAVAILABLE", + "message": "Tenant authentication service is temporarily unavailable.", + }, + ) + + if tenant is None: + return JSONResponse( + status_code=status.HTTP_401_UNAUTHORIZED, + content={ + "code": "INVALID_API_KEY", + "message": "Authentication credentials are invalid. " + "Check your API key and try again.", + }, + ) + + set_current_tenant(tenant) + request.state.tenant = tenant + response = await call_next(request) + return response + + async def _authenticate_single_tenant( + self, api_key: str, request: Request, call_next: Callable + ) -> Response: if self.valid_api_keys and api_key not in self.valid_api_keys: return JSONResponse( status_code=status.HTTP_401_UNAUTHORIZED, @@ -129,6 +151,6 @@ async def dispatch(self, request: Request, call_next: Callable) -> Response: }, ) - # Authentication successful, proceed to next middleware/handler + set_current_tenant(None) response = await call_next(request) return response diff --git a/server/opensandbox_server/services/k8s/kubernetes_service.py b/server/opensandbox_server/services/k8s/kubernetes_service.py index 4c674efc9..a9e1c12de 100644 --- a/server/opensandbox_server/services/k8s/kubernetes_service.py +++ b/server/opensandbox_server/services/k8s/kubernetes_service.py @@ -87,6 +87,7 @@ from opensandbox_server.services.k8s.client import K8sClient from opensandbox_server.services.k8s.provider_factory import create_workload_provider from opensandbox_server.services.snapshot_restore import resolve_sandbox_image_from_request +from opensandbox_server.tenants.context import get_current_tenant logger = logging.getLogger(__name__) @@ -160,7 +161,11 @@ def __init__(self, config: Optional[AppConfig] = None): self.namespace, self.execd_image, ) - + + def _resolve_namespace(self) -> str: + tenant = get_current_tenant() + return tenant.namespace if tenant else self.namespace + async def _wait_for_sandbox_ready( self, sandbox_id: str, @@ -194,7 +199,7 @@ async def _wait_for_sandbox_ready( workload = await asyncio.to_thread( self.workload_provider.get_workload, sandbox_id=sandbox_id, - namespace=self.namespace, + namespace=self._resolve_namespace(), ) if not workload: @@ -324,7 +329,7 @@ def _ensure_pvc_volumes(self, volumes: list) -> None: seen_claims.add(claim_name) try: - existing = self.k8s_client.get_pvc(self.namespace, claim_name) + existing = self.k8s_client.get_pvc(self._resolve_namespace(), claim_name) except ApiException as e: if e.status == 403: logger.warning( @@ -334,7 +339,7 @@ def _ensure_pvc_volumes(self, volumes: list) -> None: return # Skip all remaining PVCs — same SA, same permissions raise if existing is not None: - logger.debug(f"PVC '{claim_name}' already exists in namespace '{self.namespace}'") + logger.debug(f"PVC '{claim_name}' already exists in namespace '{self._resolve_namespace()}'") continue storage = vol.pvc.storage or default_size @@ -344,7 +349,7 @@ def _ensure_pvc_volumes(self, volumes: list) -> None: pvc_body = V1PersistentVolumeClaim( metadata=V1ObjectMeta( name=claim_name, - namespace=self.namespace, + namespace=self._resolve_namespace(), ), spec={ "accessModes": access_modes, @@ -355,10 +360,10 @@ def _ensure_pvc_volumes(self, volumes: list) -> None: pvc_body.spec["storageClassName"] = storage_class try: - self.k8s_client.create_pvc(self.namespace, pvc_body) + self.k8s_client.create_pvc(self._resolve_namespace(), pvc_body) logger.info( f"Auto-created PVC '{claim_name}' (size={storage}, class={storage_class or ''}) " - f"in namespace '{self.namespace}'" + f"in namespace '{self._resolve_namespace()}'" ) except ApiException as e: if e.status == 409: @@ -450,7 +455,7 @@ async def create_sandbox(self, request: CreateSandboxRequest) -> CreateSandboxRe workload_info = await asyncio.to_thread( self.workload_provider.create_workload, sandbox_id=sandbox_id, - namespace=self.namespace, + namespace=self._resolve_namespace(), image_spec=request.image, entrypoint=request.entrypoint, env=request.env or {}, @@ -507,7 +512,7 @@ async def create_sandbox(self, request: CreateSandboxRequest) -> CreateSandboxRe await asyncio.to_thread( self.workload_provider.delete_workload, sandbox_id, - self.namespace, + self._resolve_namespace(), ) except Exception as cleanup_ex: logger.error(f"Failed to cleanup sandbox {sandbox_id}", exc_info=cleanup_ex) @@ -550,11 +555,11 @@ def get_sandbox(self, sandbox_id: str) -> Sandbox: try: workload = _get_workload_or_404( self.workload_provider, - self.namespace, + self._resolve_namespace(), sandbox_id, ) return _build_sandbox_from_workload(workload, self.workload_provider) - + except HTTPException: raise except Exception as e: @@ -574,7 +579,7 @@ def list_sandboxes(self, request: ListSandboxesRequest) -> ListSandboxesResponse try: label_selector = SANDBOX_ID_LABEL workloads = self.workload_provider.list_workloads( - namespace=self.namespace, + namespace=self._resolve_namespace(), label_selector=label_selector, ) sandboxes = [ @@ -607,7 +612,7 @@ def delete_sandbox(self, sandbox_id: str) -> None: try: _delete_workload_or_404( self.workload_provider, - self.namespace, + self._resolve_namespace(), sandbox_id, ) logger.info(f"Deleted sandbox: {sandbox_id}") @@ -623,7 +628,7 @@ def pause_sandbox(self, sandbox_id: str) -> None: Pause sandbox by delegating to the workload provider. """ try: - self.workload_provider.pause_sandbox(sandbox_id, self.namespace) + self.workload_provider.pause_sandbox(sandbox_id, self._resolve_namespace()) except NotImplementedError: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, @@ -664,7 +669,7 @@ def resume_sandbox(self, sandbox_id: str) -> None: Resume sandbox by delegating to the workload provider. """ try: - self.workload_provider.resume_sandbox(sandbox_id, self.namespace) + self.workload_provider.resume_sandbox(sandbox_id, self._resolve_namespace()) except NotImplementedError: raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, @@ -703,7 +708,7 @@ def resume_sandbox(self, sandbox_id: str) -> None: def get_access_renew_extend_seconds(self, sandbox_id: str) -> Optional[int]: workload = self.workload_provider.get_workload( sandbox_id=sandbox_id, - namespace=self.namespace, + namespace=self._resolve_namespace(), ) if not workload: return None @@ -746,7 +751,7 @@ def renew_expiration( try: workload = _get_workload_or_404( self.workload_provider, - self.namespace, + self._resolve_namespace(), sandbox_id, ) @@ -762,7 +767,7 @@ def renew_expiration( self.workload_provider.update_expiration( sandbox_id=sandbox_id, - namespace=self.namespace, + namespace=self._resolve_namespace(), expires_at=new_expiration, ) @@ -784,7 +789,7 @@ def patch_sandbox_metadata(self, sandbox_id: str, patch: PatchSandboxMetadataReq """Patch sandbox metadata via JSON Merge Patch (RFC 7396). Does not restart the sandbox.""" workload = _get_workload_or_404( self.workload_provider, - self.namespace, + self._resolve_namespace(), sandbox_id, ) @@ -809,7 +814,7 @@ def patch_sandbox_metadata(self, sandbox_id: str, patch: PatchSandboxMetadataReq try: updated = self.workload_provider.patch_labels( name=name, - namespace=self.namespace, + namespace=self._resolve_namespace(), labels=label_patch, ) except Exception as e: @@ -873,7 +878,7 @@ def get_endpoint( try: workload = _get_workload_or_404( self.workload_provider, - self.namespace, + self._resolve_namespace(), sandbox_id, ) diff --git a/server/opensandbox_server/tenants/__init__.py b/server/opensandbox_server/tenants/__init__.py new file mode 100644 index 000000000..d44c72478 --- /dev/null +++ b/server/opensandbox_server/tenants/__init__.py @@ -0,0 +1,41 @@ +# Copyright 2025 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from opensandbox_server.tenants.context import get_current_tenant, set_current_tenant +from opensandbox_server.tenants.file_provider import ( + DEFAULT_TENANTS_CONFIG_PATH, + TENANTS_CONFIG_ENV_VAR, + FileTenantProvider, + _resolve_tenants_path, +) +from opensandbox_server.tenants.http_provider import ( + HTTPTenantProvider, + HTTPTenantProviderConfig, +) +from opensandbox_server.tenants.models import TenantEntry +from opensandbox_server.tenants.provider import TenantProvider, TenantProviderUnavailable + +__all__ = [ + "TenantEntry", + "TenantProvider", + "TenantProviderUnavailable", + "FileTenantProvider", + "HTTPTenantProvider", + "HTTPTenantProviderConfig", + "DEFAULT_TENANTS_CONFIG_PATH", + "TENANTS_CONFIG_ENV_VAR", + "get_current_tenant", + "set_current_tenant", + "_resolve_tenants_path", +] diff --git a/server/opensandbox_server/tenants/context.py b/server/opensandbox_server/tenants/context.py new file mode 100644 index 000000000..ca550205b --- /dev/null +++ b/server/opensandbox_server/tenants/context.py @@ -0,0 +1,32 @@ +# Copyright 2025 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from contextvars import ContextVar +from typing import Optional + +from opensandbox_server.tenants.models import TenantEntry + +_current_tenant: ContextVar[Optional[TenantEntry]] = ContextVar( + "current_tenant", default=None +) + + +def get_current_tenant() -> Optional[TenantEntry]: + return _current_tenant.get() + + +def set_current_tenant(tenant: Optional[TenantEntry]) -> None: + _current_tenant.set(tenant) diff --git a/server/opensandbox_server/tenants/file_provider.py b/server/opensandbox_server/tenants/file_provider.py new file mode 100644 index 000000000..9ce10af9b --- /dev/null +++ b/server/opensandbox_server/tenants/file_provider.py @@ -0,0 +1,181 @@ +# Copyright 2025 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import logging +import os +import threading +from pathlib import Path +from typing import Callable, Dict, List, Optional + +from opensandbox_server.tenants.models import TenantEntry +from opensandbox_server.tenants.provider import TenantProvider + +try: + import tomllib +except ModuleNotFoundError: + import tomli as tomllib # type: ignore[import] + +logger = logging.getLogger(__name__) + +TENANTS_CONFIG_ENV_VAR = "SANDBOX_TENANTS_CONFIG_PATH" +DEFAULT_TENANTS_CONFIG_PATH = Path.home() / ".opensandbox" / "tenants.toml" + + +def _resolve_tenants_path(path: Optional[str | Path] = None) -> Path: + if path: + return Path(path) + env = os.environ.get(TENANTS_CONFIG_ENV_VAR) + if env: + return Path(env) + return DEFAULT_TENANTS_CONFIG_PATH + + +def _parse_tenants_file(path: Path) -> List[TenantEntry]: + with open(path, "rb") as f: + data = tomllib.load(f) + + entries: List[TenantEntry] = [] + seen_keys: Dict[str, str] = {} + + for raw in data.get("tenants", []): + name = raw["name"] + namespace = raw["namespace"] + api_keys = raw["api_keys"] + + if not api_keys: + raise ValueError(f"Tenant '{name}' has no api_keys configured.") + + for key in api_keys: + if key in seen_keys: + raise ValueError( + f"Duplicate api_key across tenants: '{name}' and '{seen_keys[key]}'." + ) + seen_keys[key] = name + + entries.append(TenantEntry(name=name, namespace=namespace, api_keys=list(api_keys))) + + return entries + + +def _build_lookup_dict(entries: List[TenantEntry]) -> Dict[str, TenantEntry]: + result: Dict[str, TenantEntry] = {} + for entry in entries: + for key in entry.api_keys: + result[key] = entry + return result + + +class FileTenantProvider: + """TenantProvider backed by a local tenants.toml file with hot-reload via filesystem polling.""" + + def __init__(self, path: Optional[str | Path] = None) -> None: + self._path = _resolve_tenants_path(path) + self._lock = threading.Lock() + self._lookup: Dict[str, TenantEntry] = {} + self._entries: List[TenantEntry] = [] + self._ready = False + self._callbacks: List[Callable[[List[TenantEntry]], None]] = [] + self._watcher_stop = threading.Event() + self._watcher_thread: Optional[threading.Thread] = None + + @property + def path(self) -> Path: + return self._path + + def lookup(self, api_key: str) -> Optional[TenantEntry]: + with self._lock: + return self._lookup.get(api_key) + + def list_tenants(self) -> List[TenantEntry]: + with self._lock: + return list(self._entries) + + def ready(self) -> bool: + return self._ready + + def start(self) -> None: + self._load() + self._watcher_thread = threading.Thread( + target=self._watch_loop, daemon=True, name="tenant-file-watcher" + ) + self._watcher_thread.start() + + def close(self) -> None: + self._watcher_stop.set() + if self._watcher_thread and self._watcher_thread.is_alive(): + self._watcher_thread.join(timeout=5) + + def on_reload(self, callback: Callable[[List[TenantEntry]], None]) -> None: + self._callbacks.append(callback) + + def _load(self) -> None: + if not self._path.exists(): + raise FileNotFoundError(f"Tenants config not found: {self._path}") + + entries = _parse_tenants_file(self._path) + lookup = _build_lookup_dict(entries) + + with self._lock: + self._entries = entries + self._lookup = lookup + self._ready = True + + logger.info("Loaded %d tenant(s) from %s", len(entries), self._path) + + def _reload(self) -> None: + try: + entries = _parse_tenants_file(self._path) + lookup = _build_lookup_dict(entries) + + with self._lock: + self._entries = entries + self._lookup = lookup + + logger.info("Reloaded %d tenant(s) from %s", len(entries), self._path) + for cb in self._callbacks: + try: + cb(entries) + except Exception: + logger.exception("Tenant reload callback failed") + + except FileNotFoundError: + with self._lock: + self._entries = [] + self._lookup = {} + logger.warning("Tenants config deleted: %s — all tenant keys invalidated", self._path) + + except Exception: + logger.exception("Failed to reload tenants config — keeping previous state") + + def _watch_loop(self) -> None: + last_mtime: Optional[float] = None + try: + last_mtime = self._path.stat().st_mtime + except OSError: + pass + + while not self._watcher_stop.wait(timeout=2.0): + try: + current_mtime = self._path.stat().st_mtime + except OSError: + if last_mtime is not None: + self._reload() + last_mtime = None + continue + + if last_mtime is None or current_mtime != last_mtime: + last_mtime = current_mtime + self._reload() diff --git a/server/opensandbox_server/tenants/http_provider.py b/server/opensandbox_server/tenants/http_provider.py new file mode 100644 index 000000000..9ed677720 --- /dev/null +++ b/server/opensandbox_server/tenants/http_provider.py @@ -0,0 +1,187 @@ +# Copyright 2025 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""HTTP-based TenantProvider with per-key in-memory TTL cache. + +Endpoint contract: + GET {endpoint} + Header: OPEN-SANDBOX-API-KEY: + + 200 OK: + { + "namespace": "ns-a", + "ttl": 60 + } + - namespace: target K8s namespace for this key + - ttl: suggested cache duration in seconds + + 401 Unauthorized: + { + "code": "UNAUTHORIZED", + "message": "..." + } + +Cache strategy: + - Per-key cache entry with server-suggested TTL + - lookup hit + within TTL → return cached + - lookup hit + TTL expired → sync GET → refresh or serve stale within max_stale + - lookup miss → sync GET → 200: cache + return; 401: return None + - Network failure + beyond max_stale → raise TenantProviderUnavailable +""" + +from __future__ import annotations + +import logging +import threading +import time +from dataclasses import dataclass, field +from typing import Callable, Dict, List, Optional + +import httpx + +from opensandbox_server.tenants.models import TenantEntry +from opensandbox_server.tenants.provider import TenantProvider, TenantProviderUnavailable + +logger = logging.getLogger(__name__) + + +@dataclass +class HTTPTenantProviderConfig: + endpoint: str + max_stale_seconds: float = 300.0 + timeout_seconds: float = 5.0 + auth_header: Optional[str] = None + auth_token: Optional[str] = None + + +@dataclass +class _CacheEntry: + tenant: TenantEntry + fetched_at: float + ttl: float + + +class HTTPTenantProvider: + """TenantProvider backed by a remote HTTP endpoint with per-key TTL cache. + + Each lookup that misses or expires in cache triggers a sync GET to the + remote endpoint. The server response includes a suggested TTL for caching. + """ + + def __init__(self, config: HTTPTenantProviderConfig) -> None: + self._config = config + self._lock = threading.Lock() + self._cache: Dict[str, _CacheEntry] = {} + self._ready = False + self._callbacks: List[Callable[[List[TenantEntry]], None]] = [] + self._client: Optional[httpx.Client] = None + + def lookup(self, api_key: str) -> Optional[TenantEntry]: + now = time.monotonic() + + with self._lock: + cached = self._cache.get(api_key) + + if cached is not None: + age = now - cached.fetched_at + if age <= cached.ttl: + return cached.tenant + + # TTL expired — sync refresh + try: + return self._fetch_and_cache(api_key, now) + except _Unauthorized: + with self._lock: + self._cache.pop(api_key, None) + return None + except Exception: + if age > self._config.max_stale_seconds: + raise TenantProviderUnavailable( + f"HTTP tenant endpoint unreachable and cache stale " + f"beyond {self._config.max_stale_seconds}s" + ) + logger.warning( + "HTTP tenant fetch failed, serving stale entry (age=%.1fs)", age + ) + return cached.tenant + + # Cache miss — sync fetch + try: + return self._fetch_and_cache(api_key, now) + except _Unauthorized: + return None + except Exception as e: + raise TenantProviderUnavailable( + f"HTTP tenant endpoint unreachable: {e}" + ) from e + + def list_tenants(self) -> List[TenantEntry]: + with self._lock: + seen = {} + for entry in self._cache.values(): + seen[entry.tenant.name] = entry.tenant + return list(seen.values()) + + def ready(self) -> bool: + return self._ready + + def start(self) -> None: + self._client = httpx.Client(timeout=self._config.timeout_seconds) + self._ready = True + logger.info("HTTP tenant provider started, endpoint=%s", self._config.endpoint) + + def close(self) -> None: + with self._lock: + self._cache.clear() + self._ready = False + if self._client: + self._client.close() + self._client = None + + def on_reload(self, callback: Callable[[List[TenantEntry]], None]) -> None: + self._callbacks.append(callback) + + def _fetch_and_cache(self, api_key: str, now: float) -> Optional[TenantEntry]: + """GET the endpoint for a single api_key. Returns TenantEntry or raises.""" + assert self._client is not None + + headers: Dict[str, str] = {"OPEN-SANDBOX-API-KEY": api_key} + if self._config.auth_header and self._config.auth_token: + headers[self._config.auth_header] = self._config.auth_token + + resp = self._client.get(self._config.endpoint, headers=headers) + + if resp.status_code == 401: + raise _Unauthorized() + + resp.raise_for_status() + + data = resp.json() + namespace = data["namespace"] + ttl = float(data.get("ttl", 30)) + + entry = TenantEntry( + name=namespace, + namespace=namespace, + api_keys=[api_key], + ) + + with self._lock: + self._cache[api_key] = _CacheEntry(tenant=entry, fetched_at=now, ttl=ttl) + + return entry + + +class _Unauthorized(Exception): + pass diff --git a/server/opensandbox_server/tenants/models.py b/server/opensandbox_server/tenants/models.py new file mode 100644 index 000000000..f94b7e5b8 --- /dev/null +++ b/server/opensandbox_server/tenants/models.py @@ -0,0 +1,25 @@ +# Copyright 2025 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import List + + +@dataclass(frozen=True) +class TenantEntry: + name: str + namespace: str + api_keys: List[str] = field(default_factory=list) diff --git a/server/opensandbox_server/tenants/provider.py b/server/opensandbox_server/tenants/provider.py new file mode 100644 index 000000000..e02de1ae3 --- /dev/null +++ b/server/opensandbox_server/tenants/provider.py @@ -0,0 +1,66 @@ +# Copyright 2025 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from typing import Callable, List, Optional, Protocol, runtime_checkable + +from opensandbox_server.tenants.models import TenantEntry + + +@runtime_checkable +class TenantProvider(Protocol): + """Abstraction for tenant resolution. + + Auth middleware depends only on this interface, not on any specific + config source. Implementations may be backed by a local file, an + HTTP endpoint, a Kubernetes Secret, or any other tenant store. + """ + + def lookup(self, api_key: str) -> Optional[TenantEntry]: + """Resolve an API key to a tenant entry. + + Returns None if the key is not recognized. + Raises TenantProviderUnavailable if the provider cannot serve lookups. + """ + ... + + def list_tenants(self) -> List[TenantEntry]: + """Return all known tenant entries (used for startup validation).""" + ... + + def ready(self) -> bool: + """True once the provider has loaded initial state and can serve lookups.""" + ... + + def start(self) -> None: + """Start background resources (watchers, pollers). Called once at server startup.""" + ... + + def close(self) -> None: + """Release resources (threads, connections). Called on server shutdown.""" + ... + + def on_reload(self, callback: Callable[[List[TenantEntry]], None]) -> None: + """Register a callback invoked when tenant data changes. + + The callback receives the new full list of tenant entries. + Not all providers support change notification; those that don't + may silently ignore this call. + """ + ... + + +class TenantProviderUnavailable(Exception): + """Raised when a provider cannot serve lookups (e.g. remote unreachable + cache expired).""" diff --git a/server/tests/test_multi_tenancy.py b/server/tests/test_multi_tenancy.py new file mode 100644 index 000000000..9f781e2ff --- /dev/null +++ b/server/tests/test_multi_tenancy.py @@ -0,0 +1,558 @@ +# Copyright 2025 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Multi-tenancy e2e tests covering: +- FileTenantProvider: config parsing, hot-reload, auth integration +- HTTPTenantProvider: mock HTTP server, TTL cache, 401 handling +- Auth middleware: multi-tenant mode routing, namespace resolution +- Startup guards: docker + tenants fatal, api_key conflict +""" + +import json +import threading +import time +import textwrap +from http.server import HTTPServer, BaseHTTPRequestHandler +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from fastapi import FastAPI, Request +from fastapi.testclient import TestClient + +from opensandbox_server.config import AppConfig, IngressConfig, RuntimeConfig, ServerConfig, TenantsConfig +from opensandbox_server.middleware.auth import AuthMiddleware +from opensandbox_server.tenants import ( + FileTenantProvider, + HTTPTenantProvider, + HTTPTenantProviderConfig, + TenantEntry, + TenantProviderUnavailable, + get_current_tenant, + set_current_tenant, +) +from opensandbox_server.tenants.context import _current_tenant + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +@pytest.fixture +def tenants_toml(tmp_path): + """Write a valid tenants.toml and return its path.""" + content = textwrap.dedent("""\ + [[tenants]] + name = "team-alpha" + namespace = "ns-alpha" + api_keys = ["sk-alpha-1", "sk-alpha-2"] + + [[tenants]] + name = "team-beta" + namespace = "ns-beta" + api_keys = ["sk-beta-1"] + """) + path = tmp_path / "tenants.toml" + path.write_text(content) + return path + + +@pytest.fixture +def mock_http_tenant_server(): + """Start a mock HTTP tenant verification server. + + Recognizes: + - sk-http-valid → namespace=ns-http, ttl=60 + - sk-http-short-ttl → namespace=ns-short, ttl=1 + - anything else → 401 + """ + tenant_db = { + "sk-http-valid": {"namespace": "ns-http", "ttl": 60}, + "sk-http-short-ttl": {"namespace": "ns-short", "ttl": 1}, + } + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + api_key = self.headers.get("OPEN-SANDBOX-API-KEY", "") + if api_key in tenant_db: + data = tenant_db[api_key] + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(json.dumps(data).encode()) + else: + self.send_response(401) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(json.dumps({ + "code": "UNAUTHORIZED", + "message": "Invalid API key", + }).encode()) + + def log_message(self, format, *args): + pass # suppress logging + + server = HTTPServer(("127.0.0.1", 0), Handler) + port = server.server_address[1] + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + yield f"http://127.0.0.1:{port}" + server.shutdown() + + +# --------------------------------------------------------------------------- +# FileTenantProvider tests +# --------------------------------------------------------------------------- + +class TestFileTenantProvider: + + def test_load_valid_config(self, tenants_toml): + provider = FileTenantProvider(tenants_toml) + provider.start() + try: + assert provider.ready() + assert len(provider.list_tenants()) == 2 + + entry = provider.lookup("sk-alpha-1") + assert entry is not None + assert entry.name == "team-alpha" + assert entry.namespace == "ns-alpha" + + entry2 = provider.lookup("sk-alpha-2") + assert entry2 == entry + + beta = provider.lookup("sk-beta-1") + assert beta.namespace == "ns-beta" + + assert provider.lookup("nonexistent") is None + finally: + provider.close() + + def test_duplicate_keys_rejected(self, tmp_path): + content = textwrap.dedent("""\ + [[tenants]] + name = "a" + namespace = "ns-a" + api_keys = ["shared-key"] + + [[tenants]] + name = "b" + namespace = "ns-b" + api_keys = ["shared-key"] + """) + path = tmp_path / "tenants.toml" + path.write_text(content) + + provider = FileTenantProvider(path) + with pytest.raises(ValueError, match="Duplicate api_key"): + provider.start() + + def test_empty_api_keys_rejected(self, tmp_path): + content = textwrap.dedent("""\ + [[tenants]] + name = "empty" + namespace = "ns-empty" + api_keys = [] + """) + path = tmp_path / "tenants.toml" + path.write_text(content) + + provider = FileTenantProvider(path) + with pytest.raises(ValueError, match="no api_keys"): + provider.start() + + def test_file_not_found_raises(self, tmp_path): + provider = FileTenantProvider(tmp_path / "nonexistent.toml") + with pytest.raises(FileNotFoundError): + provider.start() + + def test_hot_reload_new_key(self, tenants_toml): + provider = FileTenantProvider(tenants_toml) + provider.start() + try: + assert provider.lookup("sk-new") is None + + # Add a new tenant + new_content = tenants_toml.read_text() + textwrap.dedent(""" + [[tenants]] + name = "team-new" + namespace = "ns-new" + api_keys = ["sk-new"] + """) + tenants_toml.write_text(new_content) + + # Wait for watcher to pick up (polls every 2s) + time.sleep(3) + + entry = provider.lookup("sk-new") + assert entry is not None + assert entry.namespace == "ns-new" + finally: + provider.close() + + def test_hot_reload_removed_key(self, tenants_toml): + provider = FileTenantProvider(tenants_toml) + provider.start() + try: + assert provider.lookup("sk-beta-1") is not None + + # Rewrite without team-beta + content = textwrap.dedent("""\ + [[tenants]] + name = "team-alpha" + namespace = "ns-alpha" + api_keys = ["sk-alpha-1", "sk-alpha-2"] + """) + tenants_toml.write_text(content) + time.sleep(3) + + assert provider.lookup("sk-beta-1") is None + assert provider.lookup("sk-alpha-1") is not None + finally: + provider.close() + + def test_hot_reload_parse_error_keeps_old(self, tenants_toml): + provider = FileTenantProvider(tenants_toml) + provider.start() + try: + assert provider.lookup("sk-alpha-1") is not None + + # Write invalid TOML + tenants_toml.write_text("[[[ invalid toml") + time.sleep(3) + + # Old entries still work + assert provider.lookup("sk-alpha-1") is not None + finally: + provider.close() + + def test_file_delete_clears_all(self, tenants_toml): + provider = FileTenantProvider(tenants_toml) + provider.start() + try: + assert provider.lookup("sk-alpha-1") is not None + tenants_toml.unlink() + time.sleep(3) + assert provider.lookup("sk-alpha-1") is None + finally: + provider.close() + + +# --------------------------------------------------------------------------- +# HTTPTenantProvider tests +# --------------------------------------------------------------------------- + +class TestHTTPTenantProvider: + + def test_lookup_valid_key(self, mock_http_tenant_server): + provider = HTTPTenantProvider(HTTPTenantProviderConfig( + endpoint=mock_http_tenant_server, + )) + provider.start() + try: + entry = provider.lookup("sk-http-valid") + assert entry is not None + assert entry.namespace == "ns-http" + finally: + provider.close() + + def test_lookup_invalid_key_returns_none(self, mock_http_tenant_server): + provider = HTTPTenantProvider(HTTPTenantProviderConfig( + endpoint=mock_http_tenant_server, + )) + provider.start() + try: + entry = provider.lookup("sk-invalid") + assert entry is None + finally: + provider.close() + + def test_cache_hit_within_ttl(self, mock_http_tenant_server): + provider = HTTPTenantProvider(HTTPTenantProviderConfig( + endpoint=mock_http_tenant_server, + )) + provider.start() + try: + # First call — fetches from server + entry1 = provider.lookup("sk-http-valid") + assert entry1 is not None + + # Second call — should be cached (no network) + entry2 = provider.lookup("sk-http-valid") + assert entry2 == entry1 + finally: + provider.close() + + def test_cache_expires_after_ttl(self, mock_http_tenant_server): + provider = HTTPTenantProvider(HTTPTenantProviderConfig( + endpoint=mock_http_tenant_server, + )) + provider.start() + try: + entry = provider.lookup("sk-http-short-ttl") + assert entry is not None + assert entry.namespace == "ns-short" + + # Wait for TTL to expire (ttl=1s) + time.sleep(1.5) + + # Should re-fetch (still valid on server) + entry2 = provider.lookup("sk-http-short-ttl") + assert entry2 is not None + assert entry2.namespace == "ns-short" + finally: + provider.close() + + def test_unreachable_endpoint_raises_unavailable(self): + provider = HTTPTenantProvider(HTTPTenantProviderConfig( + endpoint="http://127.0.0.1:1", # unlikely to be listening + timeout_seconds=0.5, + )) + provider.start() + try: + with pytest.raises(TenantProviderUnavailable): + provider.lookup("any-key") + finally: + provider.close() + + def test_stale_cache_served_when_endpoint_down(self, mock_http_tenant_server): + provider = HTTPTenantProvider(HTTPTenantProviderConfig( + endpoint=mock_http_tenant_server, + max_stale_seconds=60, + )) + provider.start() + try: + # Populate cache with short TTL + entry = provider.lookup("sk-http-short-ttl") + assert entry is not None + + # Now point to dead endpoint + provider._config = HTTPTenantProviderConfig( + endpoint="http://127.0.0.1:1", + timeout_seconds=0.5, + max_stale_seconds=60, + ) + provider._client.close() + import httpx + provider._client = httpx.Client(timeout=0.5) + + time.sleep(1.5) # TTL expires + + # Should serve stale (within max_stale) + entry2 = provider.lookup("sk-http-short-ttl") + assert entry2 is not None + assert entry2.namespace == "ns-short" + finally: + provider.close() + + def test_revoked_key_evicted_from_cache(self, mock_http_tenant_server): + provider = HTTPTenantProvider(HTTPTenantProviderConfig( + endpoint=mock_http_tenant_server, + )) + provider.start() + try: + # This key is valid + entry = provider.lookup("sk-http-valid") + assert entry is not None + + # Manually expire the cache entry to force re-fetch + with provider._lock: + cached = provider._cache["sk-http-valid"] + provider._cache["sk-http-valid"] = type(cached)( + tenant=cached.tenant, fetched_at=0, ttl=0 + ) + + # Now mock server doesn't know this key (simulate revocation by using unknown key) + # We'll test with a key that's cached but server returns 401 + provider._cache["sk-revoked"] = type(cached)( + tenant=TenantEntry(name="old", namespace="old-ns", api_keys=["sk-revoked"]), + fetched_at=0, + ttl=0, + ) + + result = provider.lookup("sk-revoked") + assert result is None + assert "sk-revoked" not in provider._cache + finally: + provider.close() + + +# --------------------------------------------------------------------------- +# Auth middleware multi-tenant integration +# --------------------------------------------------------------------------- + +class TestAuthMiddlewareMultiTenant: + + def _build_multi_tenant_app(self, provider): + app = FastAPI() + config = AppConfig( + server=ServerConfig(api_key=""), + runtime=RuntimeConfig(type="docker", execd_image="opensandbox/execd:latest"), + ingress=IngressConfig(mode="direct"), + ) + app.add_middleware(AuthMiddleware, config=config, tenant_provider=provider) + + @app.get("/secured") + def secured_endpoint(request: Request): + tenant = get_current_tenant() + return { + "tenant_name": tenant.name if tenant else None, + "tenant_namespace": tenant.namespace if tenant else None, + } + + return app + + def test_valid_key_resolves_tenant(self, tenants_toml): + provider = FileTenantProvider(tenants_toml) + provider.start() + try: + app = self._build_multi_tenant_app(provider) + client = TestClient(app) + + resp = client.get("/secured", headers={"OPEN-SANDBOX-API-KEY": "sk-alpha-1"}) + assert resp.status_code == 200 + data = resp.json() + assert data["tenant_name"] == "team-alpha" + assert data["tenant_namespace"] == "ns-alpha" + finally: + provider.close() + + def test_invalid_key_returns_401(self, tenants_toml): + provider = FileTenantProvider(tenants_toml) + provider.start() + try: + app = self._build_multi_tenant_app(provider) + client = TestClient(app) + + resp = client.get("/secured", headers={"OPEN-SANDBOX-API-KEY": "bad-key"}) + assert resp.status_code == 401 + assert resp.json()["code"] == "INVALID_API_KEY" + finally: + provider.close() + + def test_missing_key_returns_401(self, tenants_toml): + provider = FileTenantProvider(tenants_toml) + provider.start() + try: + app = self._build_multi_tenant_app(provider) + client = TestClient(app) + + resp = client.get("/secured") + assert resp.status_code == 401 + assert resp.json()["code"] == "MISSING_API_KEY" + finally: + provider.close() + + def test_provider_unavailable_returns_503(self, tenants_toml): + provider = FileTenantProvider(tenants_toml) + provider.start() + try: + app = self._build_multi_tenant_app(provider) + client = TestClient(app) + + # Monkey-patch lookup to raise + def broken_lookup(key): + raise TenantProviderUnavailable("test") + provider.lookup = broken_lookup + + resp = client.get("/secured", headers={"OPEN-SANDBOX-API-KEY": "sk-alpha-1"}) + assert resp.status_code == 503 + assert resp.json()["code"] == "TENANT_PROVIDER_UNAVAILABLE" + finally: + provider.close() + + def test_http_provider_auth_integration(self, mock_http_tenant_server): + provider = HTTPTenantProvider(HTTPTenantProviderConfig( + endpoint=mock_http_tenant_server, + )) + provider.start() + try: + app = self._build_multi_tenant_app(provider) + client = TestClient(app) + + resp = client.get("/secured", headers={"OPEN-SANDBOX-API-KEY": "sk-http-valid"}) + assert resp.status_code == 200 + assert resp.json()["tenant_namespace"] == "ns-http" + + resp2 = client.get("/secured", headers={"OPEN-SANDBOX-API-KEY": "bad"}) + assert resp2.status_code == 401 + finally: + provider.close() + + +# --------------------------------------------------------------------------- +# Namespace resolution +# --------------------------------------------------------------------------- + +class TestNamespaceResolution: + + def test_resolve_namespace_with_tenant(self): + from opensandbox_server.tenants.context import set_current_tenant, get_current_tenant + + tenant = TenantEntry(name="team-x", namespace="ns-x", api_keys=["k"]) + set_current_tenant(tenant) + try: + assert get_current_tenant().namespace == "ns-x" + finally: + set_current_tenant(None) + + def test_resolve_namespace_without_tenant(self): + set_current_tenant(None) + assert get_current_tenant() is None + + +# --------------------------------------------------------------------------- +# Startup guards +# --------------------------------------------------------------------------- + +class TestStartupGuards: + + def test_tenants_config_with_docker_runtime_rejected(self): + with pytest.raises(Exception): + # Simulating what main.py does — docker + tenants should be caught + # Either by AppConfig validator or main.py sys.exit + config = AppConfig( + server=ServerConfig(api_key=""), + runtime=RuntimeConfig(type="docker", execd_image="opensandbox/execd:latest"), + tenants=TenantsConfig(provider="file"), + ingress=IngressConfig(mode="direct"), + ) + # The actual guard is in main.py (sys.exit), not in config validation. + # So we test the logic directly: + if config.runtime.type == "docker" and config.tenants is not None: + raise RuntimeError("Docker + tenants not allowed") + + def test_tenants_config_with_api_key_rejected(self): + config = AppConfig( + server=ServerConfig(api_key="some-key"), + runtime=RuntimeConfig(type="docker", execd_image="opensandbox/execd:latest"), + tenants=TenantsConfig(provider="file"), + ingress=IngressConfig(mode="direct"), + ) + if config.tenants is not None and config.server.api_key: + with pytest.raises(RuntimeError): + raise RuntimeError("api_key + tenants not allowed") + + def test_http_provider_requires_endpoint(self): + with pytest.raises(Exception, match="endpoint must be set"): + TenantsConfig(provider="http") + + def test_http_provider_with_endpoint_valid(self): + cfg = TenantsConfig(provider="http", endpoint="http://localhost:8080/verify") + assert cfg.endpoint == "http://localhost:8080/verify" + + def test_file_provider_no_endpoint_needed(self): + cfg = TenantsConfig(provider="file") + assert cfg.endpoint is None From 827df7e5155582afcd4826155677a31cf8e1adad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E7=84=B6?= Date: Fri, 5 Jun 2026 17:48:32 +0800 Subject: [PATCH 2/6] docs(osep): mark OSEP-0012 multi-tenancy as implemented Co-Authored-By: Claude Opus 4.6 (1M context) --- oseps/0012-multi-tenancy.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/oseps/0012-multi-tenancy.md b/oseps/0012-multi-tenancy.md index dfd4380a5..e554120ef 100644 --- a/oseps/0012-multi-tenancy.md +++ b/oseps/0012-multi-tenancy.md @@ -3,8 +3,8 @@ title: Multi-Tenancy Support for Kubernetes Runtime authors: - "@Pangjiping" creation-date: 2026-04-29 -last-updated: 2026-05-07 -status: draft +last-updated: 2026-06-05 +status: implemented --- # OSEP-0012: Multi-Tenancy Support for Kubernetes Runtime From 7d51e00afb4611a89efce00377931013918d0d37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E7=84=B6?= Date: Fri, 5 Jun 2026 17:55:04 +0800 Subject: [PATCH 3/6] fix(server): remove unused imports flagged by ruff Co-Authored-By: Claude Opus 4.6 (1M context) --- server/opensandbox_server/tenants/file_provider.py | 1 - server/opensandbox_server/tenants/http_provider.py | 4 ++-- server/tests/test_multi_tenancy.py | 3 --- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/server/opensandbox_server/tenants/file_provider.py b/server/opensandbox_server/tenants/file_provider.py index 9ce10af9b..a41212857 100644 --- a/server/opensandbox_server/tenants/file_provider.py +++ b/server/opensandbox_server/tenants/file_provider.py @@ -21,7 +21,6 @@ from typing import Callable, Dict, List, Optional from opensandbox_server.tenants.models import TenantEntry -from opensandbox_server.tenants.provider import TenantProvider try: import tomllib diff --git a/server/opensandbox_server/tenants/http_provider.py b/server/opensandbox_server/tenants/http_provider.py index 9ed677720..bfbdd626a 100644 --- a/server/opensandbox_server/tenants/http_provider.py +++ b/server/opensandbox_server/tenants/http_provider.py @@ -45,13 +45,13 @@ import logging import threading import time -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import Callable, Dict, List, Optional import httpx from opensandbox_server.tenants.models import TenantEntry -from opensandbox_server.tenants.provider import TenantProvider, TenantProviderUnavailable +from opensandbox_server.tenants.provider import TenantProviderUnavailable logger = logging.getLogger(__name__) diff --git a/server/tests/test_multi_tenancy.py b/server/tests/test_multi_tenancy.py index 9f781e2ff..3fe6d8eca 100644 --- a/server/tests/test_multi_tenancy.py +++ b/server/tests/test_multi_tenancy.py @@ -25,8 +25,6 @@ import time import textwrap from http.server import HTTPServer, BaseHTTPRequestHandler -from pathlib import Path -from unittest.mock import MagicMock, patch import pytest from fastapi import FastAPI, Request @@ -43,7 +41,6 @@ get_current_tenant, set_current_tenant, ) -from opensandbox_server.tenants.context import _current_tenant # --------------------------------------------------------------------------- From cb11d2b25403c6464faf7930db38cf5f02965100 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E7=84=B6?= Date: Fri, 5 Jun 2026 18:21:17 +0800 Subject: [PATCH 4/6] test(e2e): add multi-tenant file and HTTP provider e2e tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - New script scripts/python-k8s-e2e-multi-tenant.sh with file/http variants - Add multi-tenant-file and multi-tenant-http matrix entries to nightly workflow - Helm chart: support tenantsToml value for mounting tenants.toml ConfigMap - HTTP variant: mock Python provider on host, accessible via Kind bridge IP - Both variants verify auth (valid key→200, invalid→401, missing→401) then run full SDK kubernetes-mini test suite with tenant key Co-Authored-By: Claude Opus 4.6 (1M context) --- .../workflows/kubernetes-nightly-build.yml | 10 + .../opensandbox-server/templates/server.yaml | 28 ++ .../charts/opensandbox-server/values.yaml | 9 + scripts/python-k8s-e2e-multi-tenant.sh | 270 ++++++++++++++++++ 4 files changed, 317 insertions(+) create mode 100755 scripts/python-k8s-e2e-multi-tenant.sh diff --git a/.github/workflows/kubernetes-nightly-build.yml b/.github/workflows/kubernetes-nightly-build.yml index a64c99cd7..de86eb902 100644 --- a/.github/workflows/kubernetes-nightly-build.yml +++ b/.github/workflows/kubernetes-nightly-build.yml @@ -13,6 +13,7 @@ on: - '.github/workflows/kubernetes-nightly-build.yml' - 'scripts/python-k8s-e2e.sh' - 'scripts/python-k8s-e2e-ingress.sh' + - 'scripts/python-k8s-e2e-multi-tenant.sh' - 'scripts/common/kubernetes-e2e.sh' - 'kubernetes/charts/**' @@ -37,6 +38,14 @@ jobs: - variant: ingress-uri script: scripts/python-k8s-e2e-ingress.sh e2e_gateway_route_mode: uri + - variant: multi-tenant-file + script: scripts/python-k8s-e2e-multi-tenant.sh + e2e_gateway_route_mode: "" + e2e_tenant_provider: file + - variant: multi-tenant-http + script: scripts/python-k8s-e2e-multi-tenant.sh + e2e_gateway_route_mode: "" + e2e_tenant_provider: http env: KIND_CLUSTER: opensandbox-e2e KIND_K8S_VERSION: v1.30.4 @@ -75,6 +84,7 @@ jobs: - name: Run Kubernetes runtime E2E env: E2E_GATEWAY_ROUTE_MODE: ${{ matrix.e2e_gateway_route_mode }} + TENANT_PROVIDER: ${{ matrix.e2e_tenant_provider }} run: bash "./${{ matrix.script }}" - name: Dump kind diagnostics diff --git a/kubernetes/charts/opensandbox-server/templates/server.yaml b/kubernetes/charts/opensandbox-server/templates/server.yaml index 2dcf1d7fa..292426a07 100644 --- a/kubernetes/charts/opensandbox-server/templates/server.yaml +++ b/kubernetes/charts/opensandbox-server/templates/server.yaml @@ -68,6 +68,19 @@ data: config.toml: | {{ .Values.configToml | indent 4 }} {{ include "opensandbox-server.ingressConfigToml" . | indent 4 }} +{{- if .Values.tenantsToml }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "opensandbox-server.fullname" . }}-tenants + namespace: {{ include "opensandbox-server.namespace" . }} + labels: + {{- include "opensandbox-server.labels" . | nindent 4 }} +data: + tenants.toml: | +{{ .Values.tenantsToml | indent 4 }} +{{- end }} --- apiVersion: apps/v1 kind: Deployment @@ -106,6 +119,10 @@ spec: env: - name: SANDBOX_CONFIG_PATH value: "/etc/opensandbox/config.toml" + {{- if .Values.tenantsToml }} + - name: SANDBOX_TENANTS_CONFIG_PATH + value: "/etc/opensandbox/tenants.toml" + {{- end }} {{- with .Values.server.env }} {{- toYaml . | nindent 12 }} {{- end }} @@ -114,6 +131,12 @@ spec: mountPath: /etc/opensandbox/config.toml subPath: config.toml readOnly: true + {{- if .Values.tenantsToml }} + - name: tenants + mountPath: /etc/opensandbox/tenants.toml + subPath: tenants.toml + readOnly: true + {{- end }} {{- with .Values.server.volumeMounts }} {{- toYaml . | nindent 12 }} {{- end }} @@ -137,6 +160,11 @@ spec: - name: config configMap: name: {{ include "opensandbox-server.fullname" . }}-config + {{- if .Values.tenantsToml }} + - name: tenants + configMap: + name: {{ include "opensandbox-server.fullname" . }}-tenants + {{- end }} {{- with .Values.server.volumes }} {{- toYaml . | nindent 8 }} {{- end }} diff --git a/kubernetes/charts/opensandbox-server/values.yaml b/kubernetes/charts/opensandbox-server/values.yaml index afdde64de..788ec97dc 100644 --- a/kubernetes/charts/opensandbox-server/values.yaml +++ b/kubernetes/charts/opensandbox-server/values.yaml @@ -100,3 +100,12 @@ configToml: | [egress] image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/egress:v1.0.12" mode = "dns+nft" + +# Optional: multi-tenant tenants.toml content. +# When set, a separate ConfigMap is created and mounted at /etc/opensandbox/tenants.toml. +# tenantsToml: | +# [[tenants]] +# name = "team-a" +# namespace = "sandbox-team-a" +# api_keys = ["sk-a-1"] +tenantsToml: "" diff --git a/scripts/python-k8s-e2e-multi-tenant.sh b/scripts/python-k8s-e2e-multi-tenant.sh new file mode 100755 index 000000000..133a6741d --- /dev/null +++ b/scripts/python-k8s-e2e-multi-tenant.sh @@ -0,0 +1,270 @@ +#!/bin/bash +# Multi-tenant Kubernetes E2E test +# Copyright 2026 Alibaba Group Holding Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euxo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=common/kubernetes-e2e.sh +source "${SCRIPT_DIR}/common/kubernetes-e2e.sh" + +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +KIND_CLUSTER="${KIND_CLUSTER:-opensandbox-e2e}" +KIND_K8S_VERSION="${KIND_K8S_VERSION:-v1.30.4}" +KUBECONFIG_PATH="${KUBECONFIG_PATH:-/tmp/opensandbox-kind-kubeconfig}" +E2E_NAMESPACE="${E2E_NAMESPACE:-opensandbox-e2e}" +SERVER_NAMESPACE="${SERVER_NAMESPACE:-opensandbox-system}" +PVC_NAME="${PVC_NAME:-opensandbox-e2e-pvc-test}" +PV_NAME="${PV_NAME:-opensandbox-e2e-pv-test}" +CONTROLLER_IMG="${CONTROLLER_IMG:-opensandbox/controller:e2e-local}" +SERVER_IMG="${SERVER_IMG:-opensandbox/server:e2e-local}" +EXECD_IMG="${EXECD_IMG:-opensandbox/execd:e2e-local}" +EGRESS_IMG="${EGRESS_IMG:-opensandbox/egress:e2e-local}" +SERVER_RELEASE="${SERVER_RELEASE:-opensandbox-server}" +SERVER_VALUES_FILE="${SERVER_VALUES_FILE:-/tmp/opensandbox-server-values.yaml}" +PORT_FORWARD_LOG="${PORT_FORWARD_LOG:-/tmp/opensandbox-server-port-forward.log}" +SANDBOX_TEST_IMAGE="${SANDBOX_TEST_IMAGE:-ubuntu:latest}" +LIFECYCLE_LOCAL_PORT="${LIFECYCLE_LOCAL_PORT:-8080}" + +# Multi-tenant specific +TENANT_PROVIDER="${TENANT_PROVIDER:-file}" # "file" or "http" +TENANT_API_KEY="mt-e2e-tenant-key" +TENANT_NAMESPACE="${E2E_NAMESPACE}" +HTTP_MOCK_PORT=9999 +HTTP_MOCK_PID="" + +SERVER_IMG_REPOSITORY="${SERVER_IMG%:*}" +SERVER_IMG_TAG="${SERVER_IMG##*:}" + +k8s_e2e_export_kubeconfig +k8s_e2e_setup_kind_and_controller +k8s_e2e_build_runtime_images +k8s_e2e_kind_load_runtime_images +k8s_e2e_apply_pvc_and_seed + +# --- Multi-tenant Helm values --- + +_write_multi_tenant_file_values() { + cat > "${SERVER_VALUES_FILE}" </dev/null || echo "172.18.0.1") + + cat > "${SERVER_VALUES_FILE}" </dev/null 2>&1 || true + fi +} + +# --- Main --- + +if [ "${TENANT_PROVIDER}" = "http" ]; then + _start_http_mock_provider + trap '_stop_http_mock_provider; kill "${PORT_FORWARD_PID}" >/dev/null 2>&1 || true' EXIT + _write_multi_tenant_http_values +else + trap 'kill "${PORT_FORWARD_PID}" >/dev/null 2>&1 || true' EXIT + _write_multi_tenant_file_values +fi + +k8s_e2e_helm_install_server + +kubectl port-forward -n "${SERVER_NAMESPACE}" svc/opensandbox-server "${LIFECYCLE_LOCAL_PORT}:80" >"${PORT_FORWARD_LOG}" 2>&1 & +PORT_FORWARD_PID=$! + +k8s_e2e_wait_http_ok "http://127.0.0.1:${LIFECYCLE_LOCAL_PORT}/health" + +# --- Verify multi-tenant auth --- + +echo "=== Verifying multi-tenant auth ===" + +# Valid tenant key should work +HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" \ + "http://127.0.0.1:${LIFECYCLE_LOCAL_PORT}/v1/sandboxes" \ + -H "OPEN-SANDBOX-API-KEY: ${TENANT_API_KEY}") +if [ "${HTTP_CODE}" != "200" ]; then + echo "FAIL: valid tenant key got HTTP ${HTTP_CODE}, expected 200" + exit 1 +fi +echo "PASS: valid tenant key → 200" + +# Invalid key should get 401 +HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" \ + "http://127.0.0.1:${LIFECYCLE_LOCAL_PORT}/v1/sandboxes" \ + -H "OPEN-SANDBOX-API-KEY: invalid-key") +if [ "${HTTP_CODE}" != "401" ]; then + echo "FAIL: invalid key got HTTP ${HTTP_CODE}, expected 401" + exit 1 +fi +echo "PASS: invalid key → 401" + +# No key should get 401 +HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" \ + "http://127.0.0.1:${LIFECYCLE_LOCAL_PORT}/v1/sandboxes") +if [ "${HTTP_CODE}" != "401" ]; then + echo "FAIL: no key got HTTP ${HTTP_CODE}, expected 401" + exit 1 +fi +echo "PASS: no key → 401" + +echo "=== Multi-tenant auth verification passed ===" + +# --- Run SDK mini E2E with tenant key --- + +export OPENSANDBOX_TEST_DOMAIN="localhost:${LIFECYCLE_LOCAL_PORT}" +export OPENSANDBOX_TEST_PROTOCOL="http" +export OPENSANDBOX_TEST_API_KEY="${TENANT_API_KEY}" +export OPENSANDBOX_SANDBOX_DEFAULT_IMAGE="${SANDBOX_TEST_IMAGE}" +export OPENSANDBOX_E2E_RUNTIME="kubernetes" +export OPENSANDBOX_TEST_USE_SERVER_PROXY="true" +export OPENSANDBOX_TEST_PVC_NAME="${PVC_NAME}" + +k8s_e2e_export_sandbox_resource_env + +k8s_e2e_generate_sdk_and_run_kubernetes_mini From bbdbc1c82292050e7544b3d37e9e239ad3e4c886 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E7=84=B6?= Date: Fri, 5 Jun 2026 18:59:58 +0800 Subject: [PATCH 5/6] fix(e2e): improve host IP detection for multi-tenant HTTP mock Use Kind control-plane container's gateway IP as primary detection, with fallbacks. Add pre-deploy reachability check for the mock endpoint. Co-Authored-By: Claude Opus 4.6 (1M context) --- scripts/python-k8s-e2e-multi-tenant.sh | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/scripts/python-k8s-e2e-multi-tenant.sh b/scripts/python-k8s-e2e-multi-tenant.sh index 133a6741d..bd675e488 100755 --- a/scripts/python-k8s-e2e-multi-tenant.sh +++ b/scripts/python-k8s-e2e-multi-tenant.sh @@ -107,9 +107,25 @@ EOF } _write_multi_tenant_http_values() { - # Get the Docker bridge IP (gateway from Kind container's perspective) + # Get the host IP reachable from Kind containers. + # On Linux CI, the Kind Docker bridge gateway is the host. + # On Mac, host.docker.internal works but not on Linux without extra setup. local host_ip - host_ip=$(docker network inspect kind -f '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null || echo "172.18.0.1") + host_ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.Gateway}}{{end}}' "${KIND_CLUSTER}-control-plane" 2>/dev/null | tr -d '[:space:]') + if [ -z "${host_ip}" ]; then + # Fallback: inspect the "kind" network + host_ip=$(docker network inspect kind -f '{{(index .IPAM.Config 0).Gateway}}' 2>/dev/null | tr -d '[:space:]') + fi + if [ -z "${host_ip}" ]; then + host_ip="172.18.0.1" + fi + echo "HTTP tenant mock endpoint: http://${host_ip}:${HTTP_MOCK_PORT}/verify" + # Verify the mock is reachable from host before deploying + curl -fsS "http://${host_ip}:${HTTP_MOCK_PORT}/verify" -H "OPEN-SANDBOX-API-KEY: ${TENANT_API_KEY}" || { + echo "WARN: mock not reachable at ${host_ip}:${HTTP_MOCK_PORT}, trying 127.0.0.1" + # On CI, host-to-host should work via 0.0.0.0 binding + curl -fsS "http://127.0.0.1:${HTTP_MOCK_PORT}/verify" -H "OPEN-SANDBOX-API-KEY: ${TENANT_API_KEY}" + } cat > "${SERVER_VALUES_FILE}" < Date: Sat, 6 Jun 2026 15:23:19 +0800 Subject: [PATCH 6/6] fix(server): address multi-tenancy PR review feedback - Fix sync HTTP blocking event loop: use asyncio.to_thread() in auth middleware - Add per-key singleflight pattern to prevent thundering herd on TTL expiry - Log warning at startup when HTTP tenant endpoint is not HTTPS - Rename _resolve_tenants_path to resolve_tenants_path (public API) - Use tuple for TenantEntry.api_keys for true frozen immutability - Extract validate_tenant_config() so startup guards are testable - Add [tenants] section to configuration.md, README, and example configs Co-Authored-By: Claude Opus 4.6 (1M context) --- server/README.md | 16 ++++++ server/configuration.md | 49 +++++++++++++++++-- .../examples/example.config.k8s.toml | 13 +++++ .../examples/example.config.k8s.zh.toml | 13 +++++ server/opensandbox_server/main.py | 19 +++---- server/opensandbox_server/middleware/auth.py | 4 +- server/opensandbox_server/tenants/__init__.py | 25 +++++++++- .../tenants/file_provider.py | 6 +-- .../tenants/http_provider.py | 36 +++++++++++++- server/opensandbox_server/tenants/models.py | 4 +- server/tests/test_multi_tenancy.py | 34 ++++--------- 11 files changed, 170 insertions(+), 49 deletions(-) diff --git a/server/README.md b/server/README.md index f764a859a..43ddd5e53 100644 --- a/server/README.md +++ b/server/README.md @@ -11,6 +11,7 @@ A production-grade, FastAPI-based service for managing the lifecycle of containe - **Kubernetes**: Production-ready (see [`../kubernetes/README.md`](../kubernetes/README.md) for deployment) - **Lifecycle cleanup modes**: Configurable TTL with renewal, or manual cleanup with explicit delete - **Access control**: API Key authentication (`OPEN-SANDBOX-API-KEY`); can be disabled for local/dev +- **Multi-tenancy**: Per-tenant Kubernetes namespace isolation with file or HTTP-based tenant providers ([OSEP-0012](../oseps/0012-multi-tenancy.md)) - **Networking modes**: - Host: shared host network, performance first - Bridge: isolated network with built-in HTTP routing @@ -103,6 +104,21 @@ All API endpoints (except `/health`, `/docs`, `/redoc`) require authentication v curl -H "OPEN-SANDBOX-API-KEY: your-secret-api-key" http://localhost:8080/v1/sandboxes ``` +### Multi-tenant mode + +When `[tenants]` is configured in the server TOML, the server switches to multi-tenant mode: + +- Each API key maps to a specific tenant and Kubernetes namespace +- Sandbox operations are scoped to the tenant's namespace +- `server.api_key` must be removed (keys are managed by the tenant provider) +- Requires `runtime.type = "kubernetes"` + +Two provider backends are available: +- **File provider** (`provider = "file"`): reads `tenants.toml` with hot-reload support +- **HTTP provider** (`provider = "http"`): per-key lookup against a remote endpoint with TTL cache + +See **[configuration.md](configuration.md)** for full `[tenants]` options and [OSEP-0012](../oseps/0012-multi-tenancy.md) for design. + ### Example usage **Create a Sandbox** diff --git a/server/configuration.md b/server/configuration.md index e2531bcf3..7c38073fe 100644 --- a/server/configuration.md +++ b/server/configuration.md @@ -31,9 +31,10 @@ Example files in this repository: 10. [`[storage]`](#storage) 11. [`[store]`](#store) 12. [`[secure_runtime]`](#secure_runtime) -13. [`[renew_intent]`](#renew_intent--experimental) -14. [Environment variables (outside TOML)](#environment-variables-outside-toml) -15. [Cross-field validation rules](#cross-field-validation-rules) +13. [`[tenants]`](#tenants--multi-tenant-mode) +14. [`[renew_intent]`](#renew_intent--experimental) +15. [Environment variables (outside TOML)](#environment-variables-outside-toml) +16. [Cross-field validation rules](#cross-field-validation-rules) --- @@ -52,6 +53,7 @@ Example files in this repository: | `[storage]` | No | Host bind mounts / OSSFS mount root | | `[store]` | No | Server-managed persistent metadata backend | | `[secure_runtime]` | No | gVisor / Kata / Firecracker | +| `[tenants]` | No | Multi-tenant mode (Kubernetes only) | | `[renew_intent]` | No | Experimental auto-renew on access | --- @@ -275,6 +277,39 @@ See [`docs/secure-container.md`](../docs/secure-container.md) for installation a --- +## `[tenants]` — multi-tenant mode + +Enables multi-tenant isolation with per-tenant Kubernetes namespaces. When present, `server.api_key` must be removed — API keys are managed by the tenant provider. Only supported with `runtime.type = "kubernetes"`. + +Design: [OSEP-0012](../oseps/0012-multi-tenancy.md). + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| `provider` | `"file"` \| `"http"` | `"file"` | Tenant provider type. `file` reads a local TOML file; `http` queries a remote endpoint per key. | +| `endpoint` | string \| omitted | `null` | Remote tenant lookup URL. **Required** when `provider = "http"`. Should use HTTPS in production. | +| `max_stale_seconds` | float | `300.0` | Maximum seconds to serve stale cache when HTTP endpoint is unreachable. | +| `timeout_seconds` | float | `5.0` | HTTP request timeout in seconds. | +| `auth_header` | string \| omitted | `null` | Optional header name for provider-level authentication to HTTP endpoint. | +| `auth_token` | string \| omitted | `null` | Optional token value for provider-level authentication to HTTP endpoint. | + +**File provider** (`provider = "file"`): reads a separate `tenants.toml` file. Path resolved via `SANDBOX_TENANTS_CONFIG_PATH` env or default `~/.opensandbox/tenants.toml`. Hot-reloaded on file change (2s mtime poll). Format: + +```toml +[[tenants]] +name = "team-a" +namespace = "sandbox-team-a" +api_keys = ["sk-a-1", "sk-a-2"] + +[[tenants]] +name = "team-b" +namespace = "sandbox-team-b" +api_keys = ["sk-b-1"] +``` + +**HTTP provider** (`provider = "http"`): per-key lookup with in-memory TTL cache. Endpoint contract: send `OPEN-SANDBOX-API-KEY` header, expect 200 `{"namespace": "...", "ttl": 60}` or 401. + +--- + ## `[renew_intent]` — **experimental** **🧪 Experimental:** auto-renew sandbox expiration when access is observed (lifecycle proxy and/or Redis queue). Off by default. Full design: [OSEP-0009](../oseps/0009-auto-renew-sandbox-on-ingress-access.md). @@ -301,6 +336,7 @@ These are read by the server or runtime code in addition to the TOML file: | Variable | Where used | Description | |----------|------------|-------------| | `SANDBOX_CONFIG_PATH` | `config.py`, CLI | Path to the TOML file. Overrides the default `~/.sandbox.toml` when set. | +| `SANDBOX_TENANTS_CONFIG_PATH` | `tenants/` | Path to the tenants TOML file (file provider). Overrides default `~/.opensandbox/tenants.toml`. | | `OPENSANDBOX_SERVER_API_KEY` | `config.py` | Overrides the API key from the TOML file. | | `DOCKER_HOST` | Docker service | Standard Docker daemon address (e.g. `unix:///var/run/docker.sock`). | | `PENDING_FAILURE_TTL` | Docker service | Seconds to retain **failed Pending** sandboxes before cleanup; default **`3600`**. | @@ -323,7 +359,12 @@ Rules enforced when the full `AppConfig` is parsed (see `AppConfig.validate_runt 3. **`ingress.mode = "gateway"`** - `[ingress.gateway]` is **required**; address and `route.mode` must satisfy the validators (wildcard domain for `wildcard` route mode, no URL scheme in `address`, etc.). -4. **`secure_runtime`** +4. **`[tenants]` present** + - `runtime.type` must be **`"kubernetes"`** (Docker + tenants is fatal). + - `server.api_key` must be empty or omitted (tenant provider manages keys). + - `provider = "http"` requires `endpoint` to be set. + +5. **`secure_runtime`** - See [Secure runtime](#secure_runtime) above. --- diff --git a/server/opensandbox_server/examples/example.config.k8s.toml b/server/opensandbox_server/examples/example.config.k8s.toml index 5f397e9f4..c14c39a0a 100644 --- a/server/opensandbox_server/examples/example.config.k8s.toml +++ b/server/opensandbox_server/examples/example.config.k8s.toml @@ -80,6 +80,19 @@ mode = "dns" # Default is true (recommended for dual-stack CNI). Set false only if you need IPv6 in the netns (see server/configuration.md). # disable_ipv6 = false +# Multi-tenant mode — isolate tenants into separate Kubernetes namespaces. +# When enabled, remove server.api_key above (tenant provider manages keys). +# Requires a separate tenants.toml file (see configuration.md). +# [tenants] +# provider = "file" +# +# For HTTP-based tenant lookup: +# [tenants] +# provider = "http" +# endpoint = "https://your-iam-service.internal/tenant/verify" +# timeout_seconds = 5 +# max_stale_seconds = 300 + # 🧪 [EXPERIMENTAL] Renew-on-access. Off by default — see server/README.md. [renew_intent] enabled = false diff --git a/server/opensandbox_server/examples/example.config.k8s.zh.toml b/server/opensandbox_server/examples/example.config.k8s.zh.toml index a1b7cd81c..254f090b2 100644 --- a/server/opensandbox_server/examples/example.config.k8s.zh.toml +++ b/server/opensandbox_server/examples/example.config.k8s.zh.toml @@ -81,6 +81,19 @@ mode = "dns" # Default is true (recommended for dual-stack CNI). Set false only if you need IPv6 in the netns (see server/configuration.md). # disable_ipv6 = false +# 多租户模式 — 将租户隔离到不同 Kubernetes 命名空间。 +# 启用时需移除上方 server.api_key（租户 API Key 由 provider 管理）。 +# 需要单独的 tenants.toml 文件（见 configuration.md）。 +# [tenants] +# provider = "file" +# +# HTTP 方式查询租户: +# [tenants] +# provider = "http" +# endpoint = "https://your-iam-service.internal/tenant/verify" +# timeout_seconds = 5 +# max_stale_seconds = 300 + # 🧪 [EXPERIMENTAL] 按访问续期。默认关闭 — 见 server/README_zh.md。 [renew_intent] enabled = false diff --git a/server/opensandbox_server/main.py b/server/opensandbox_server/main.py index d1fbfa5ee..45f1754f7 100644 --- a/server/opensandbox_server/main.py +++ b/server/opensandbox_server/main.py @@ -40,7 +40,8 @@ HTTPTenantProvider, HTTPTenantProviderConfig, TenantProvider, - _resolve_tenants_path, + resolve_tenants_path, + validate_tenant_config, ) # Load configuration before initializing routers/middleware @@ -51,20 +52,14 @@ _tenant_provider: TenantProvider | None = None if app_config.tenants is not None: - if app_config.runtime.type == "docker": - sys.exit( - "FATAL: [tenants] configured but runtime.type='docker'. " - "Multi-tenancy requires Kubernetes namespaces." - ) - if app_config.server.api_key and app_config.server.api_key.strip(): - sys.exit( - "FATAL: server.api_key must be removed from server.toml when using [tenants]. " - "Tenant API keys are managed by the tenant provider." - ) + try: + validate_tenant_config(app_config.runtime.type, app_config.server.api_key) + except ValueError as e: + sys.exit(f"FATAL: {e}") _tenants_cfg = app_config.tenants if _tenants_cfg.provider == "file": - _tenants_path = _resolve_tenants_path() + _tenants_path = resolve_tenants_path() _tenant_provider = FileTenantProvider(_tenants_path) _tenant_provider.start() elif _tenants_cfg.provider == "http": diff --git a/server/opensandbox_server/middleware/auth.py b/server/opensandbox_server/middleware/auth.py index 2817ab8d2..11bac2349 100644 --- a/server/opensandbox_server/middleware/auth.py +++ b/server/opensandbox_server/middleware/auth.py @@ -111,8 +111,10 @@ async def dispatch(self, request: Request, call_next: Callable) -> Response: async def _authenticate_multi_tenant( self, api_key: str, request: Request, call_next: Callable ) -> Response: + import asyncio + try: - tenant = self.tenant_provider.lookup(api_key) + tenant = await asyncio.to_thread(self.tenant_provider.lookup, api_key) except TenantProviderUnavailable as e: logger.error("Tenant provider unavailable: %s", e) return JSONResponse( diff --git a/server/opensandbox_server/tenants/__init__.py b/server/opensandbox_server/tenants/__init__.py index d44c72478..3857628fd 100644 --- a/server/opensandbox_server/tenants/__init__.py +++ b/server/opensandbox_server/tenants/__init__.py @@ -17,7 +17,7 @@ DEFAULT_TENANTS_CONFIG_PATH, TENANTS_CONFIG_ENV_VAR, FileTenantProvider, - _resolve_tenants_path, + resolve_tenants_path, ) from opensandbox_server.tenants.http_provider import ( HTTPTenantProvider, @@ -26,6 +26,26 @@ from opensandbox_server.tenants.models import TenantEntry from opensandbox_server.tenants.provider import TenantProvider, TenantProviderUnavailable + +def validate_tenant_config(runtime_type: str, api_key: str | None) -> None: + """Validate tenant configuration against runtime and auth settings. + + Raises ValueError if: + - runtime is docker (multi-tenancy requires Kubernetes namespaces) + - server.api_key is set (conflicts with tenant-managed keys) + """ + if runtime_type == "docker": + raise ValueError( + "[tenants] configured but runtime.type='docker'. " + "Multi-tenancy requires Kubernetes namespaces." + ) + if api_key and api_key.strip(): + raise ValueError( + "server.api_key must be removed from server.toml when using [tenants]. " + "Tenant API keys are managed by the tenant provider." + ) + + __all__ = [ "TenantEntry", "TenantProvider", @@ -37,5 +57,6 @@ "TENANTS_CONFIG_ENV_VAR", "get_current_tenant", "set_current_tenant", - "_resolve_tenants_path", + "resolve_tenants_path", + "validate_tenant_config", ] diff --git a/server/opensandbox_server/tenants/file_provider.py b/server/opensandbox_server/tenants/file_provider.py index a41212857..61fd32369 100644 --- a/server/opensandbox_server/tenants/file_provider.py +++ b/server/opensandbox_server/tenants/file_provider.py @@ -33,7 +33,7 @@ DEFAULT_TENANTS_CONFIG_PATH = Path.home() / ".opensandbox" / "tenants.toml" -def _resolve_tenants_path(path: Optional[str | Path] = None) -> Path: +def resolve_tenants_path(path: Optional[str | Path] = None) -> Path: if path: return Path(path) env = os.environ.get(TENANTS_CONFIG_ENV_VAR) @@ -64,7 +64,7 @@ def _parse_tenants_file(path: Path) -> List[TenantEntry]: ) seen_keys[key] = name - entries.append(TenantEntry(name=name, namespace=namespace, api_keys=list(api_keys))) + entries.append(TenantEntry(name=name, namespace=namespace, api_keys=tuple(api_keys))) return entries @@ -81,7 +81,7 @@ class FileTenantProvider: """TenantProvider backed by a local tenants.toml file with hot-reload via filesystem polling.""" def __init__(self, path: Optional[str | Path] = None) -> None: - self._path = _resolve_tenants_path(path) + self._path = resolve_tenants_path(path) self._lock = threading.Lock() self._lookup: Dict[str, TenantEntry] = {} self._entries: List[TenantEntry] = [] diff --git a/server/opensandbox_server/tenants/http_provider.py b/server/opensandbox_server/tenants/http_provider.py index bfbdd626a..2967384ba 100644 --- a/server/opensandbox_server/tenants/http_provider.py +++ b/server/opensandbox_server/tenants/http_provider.py @@ -77,12 +77,14 @@ class HTTPTenantProvider: Each lookup that misses or expires in cache triggers a sync GET to the remote endpoint. The server response includes a suggested TTL for caching. + Uses per-key locks to prevent thundering herd on TTL expiry. """ def __init__(self, config: HTTPTenantProviderConfig) -> None: self._config = config self._lock = threading.Lock() self._cache: Dict[str, _CacheEntry] = {} + self._inflight: Dict[str, threading.Event] = {} self._ready = False self._callbacks: List[Callable[[List[TenantEntry]], None]] = [] self._client: Optional[httpx.Client] = None @@ -137,6 +139,12 @@ def ready(self) -> bool: return self._ready def start(self) -> None: + if self._config.endpoint and not self._config.endpoint.startswith("https://"): + logger.warning( + "HTTP tenant endpoint is not HTTPS (%s). " + "API keys will be transmitted in cleartext.", + self._config.endpoint, + ) self._client = httpx.Client(timeout=self._config.timeout_seconds) self._ready = True logger.info("HTTP tenant provider started, endpoint=%s", self._config.endpoint) @@ -153,6 +161,32 @@ def on_reload(self, callback: Callable[[List[TenantEntry]], None]) -> None: self._callbacks.append(callback) def _fetch_and_cache(self, api_key: str, now: float) -> Optional[TenantEntry]: + """Singleflight GET: only one fetch per key at a time, others wait.""" + with self._lock: + event = self._inflight.get(api_key) + if event is not None: + is_leader = False + else: + event = threading.Event() + self._inflight[api_key] = event + is_leader = True + + if not is_leader: + event.wait(timeout=self._config.timeout_seconds) + with self._lock: + cached = self._cache.get(api_key) + if cached: + return cached.tenant + return None + + try: + return self._do_fetch(api_key, now) + finally: + with self._lock: + self._inflight.pop(api_key, None) + event.set() + + def _do_fetch(self, api_key: str, now: float) -> Optional[TenantEntry]: """GET the endpoint for a single api_key. Returns TenantEntry or raises.""" assert self._client is not None @@ -174,7 +208,7 @@ def _fetch_and_cache(self, api_key: str, now: float) -> Optional[TenantEntry]: entry = TenantEntry( name=namespace, namespace=namespace, - api_keys=[api_key], + api_keys=(api_key,), ) with self._lock: diff --git a/server/opensandbox_server/tenants/models.py b/server/opensandbox_server/tenants/models.py index f94b7e5b8..e63bd9f83 100644 --- a/server/opensandbox_server/tenants/models.py +++ b/server/opensandbox_server/tenants/models.py @@ -15,11 +15,11 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import List +from typing import Tuple @dataclass(frozen=True) class TenantEntry: name: str namespace: str - api_keys: List[str] = field(default_factory=list) + api_keys: Tuple[str, ...] = field(default_factory=tuple) diff --git a/server/tests/test_multi_tenancy.py b/server/tests/test_multi_tenancy.py index 3fe6d8eca..9ff7838d1 100644 --- a/server/tests/test_multi_tenancy.py +++ b/server/tests/test_multi_tenancy.py @@ -40,6 +40,7 @@ TenantProviderUnavailable, get_current_tenant, set_current_tenant, + validate_tenant_config, ) @@ -374,7 +375,7 @@ def test_revoked_key_evicted_from_cache(self, mock_http_tenant_server): # Now mock server doesn't know this key (simulate revocation by using unknown key) # We'll test with a key that's cached but server returns 401 provider._cache["sk-revoked"] = type(cached)( - tenant=TenantEntry(name="old", namespace="old-ns", api_keys=["sk-revoked"]), + tenant=TenantEntry(name="old", namespace="old-ns", api_keys=("sk-revoked",)), fetched_at=0, ttl=0, ) @@ -498,7 +499,7 @@ class TestNamespaceResolution: def test_resolve_namespace_with_tenant(self): from opensandbox_server.tenants.context import set_current_tenant, get_current_tenant - tenant = TenantEntry(name="team-x", namespace="ns-x", api_keys=["k"]) + tenant = TenantEntry(name="team-x", namespace="ns-x", api_keys=("k",)) set_current_tenant(tenant) try: assert get_current_tenant().namespace == "ns-x" @@ -517,30 +518,15 @@ def test_resolve_namespace_without_tenant(self): class TestStartupGuards: def test_tenants_config_with_docker_runtime_rejected(self): - with pytest.raises(Exception): - # Simulating what main.py does — docker + tenants should be caught - # Either by AppConfig validator or main.py sys.exit - config = AppConfig( - server=ServerConfig(api_key=""), - runtime=RuntimeConfig(type="docker", execd_image="opensandbox/execd:latest"), - tenants=TenantsConfig(provider="file"), - ingress=IngressConfig(mode="direct"), - ) - # The actual guard is in main.py (sys.exit), not in config validation. - # So we test the logic directly: - if config.runtime.type == "docker" and config.tenants is not None: - raise RuntimeError("Docker + tenants not allowed") + with pytest.raises(ValueError, match="runtime.type='docker'"): + validate_tenant_config(runtime_type="docker", api_key="") def test_tenants_config_with_api_key_rejected(self): - config = AppConfig( - server=ServerConfig(api_key="some-key"), - runtime=RuntimeConfig(type="docker", execd_image="opensandbox/execd:latest"), - tenants=TenantsConfig(provider="file"), - ingress=IngressConfig(mode="direct"), - ) - if config.tenants is not None and config.server.api_key: - with pytest.raises(RuntimeError): - raise RuntimeError("api_key + tenants not allowed") + with pytest.raises(ValueError, match="server.api_key must be removed"): + validate_tenant_config(runtime_type="kubernetes", api_key="some-key") + + def test_tenants_config_valid_kubernetes_no_api_key(self): + validate_tenant_config(runtime_type="kubernetes", api_key="") def test_http_provider_requires_endpoint(self): with pytest.raises(Exception, match="endpoint must be set"):