diff --git a/.github/workflows/code_style.yml b/.github/workflows/code_style.yml index 2d36207..b0053a9 100644 --- a/.github/workflows/code_style.yml +++ b/.github/workflows/code_style.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] + python-version: ['3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 850e765..6060541 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] + python-version: ['3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@v2 diff --git a/CHANGELOG.md b/CHANGELOG.md index c97ae6d..7c89be8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Added `SharedFileSystemMount` class for container sfs support +- Added `SecretMount` and `GeneralStorageMount` classes that inherit from base `VolumeMount` + +### Changed + +- Removed support for python 3.9 as it doesn't support `kw_only` and reaches EOS state in 2 months + ## [1.13.2] - 2025-06-04 ### Changed diff --git a/datacrunch/containers/__init__.py b/datacrunch/containers/__init__.py index 6a95373..dc7a4ab 100644 --- a/datacrunch/containers/__init__.py +++ b/datacrunch/containers/__init__.py @@ -7,6 +7,8 @@ EntrypointOverridesSettings, VolumeMount, SecretMount, + SharedFileSystemMount, + GeneralStorageMount, VolumeMountType, Container, ContainerRegistryCredentials, diff --git a/datacrunch/containers/containers.py b/datacrunch/containers/containers.py index d64c0ac..d614041 100644 --- a/datacrunch/containers/containers.py +++ b/datacrunch/containers/containers.py @@ -8,7 +8,7 @@ import os from dataclasses import dataclass, field from dataclasses_json import dataclass_json, Undefined # type: ignore -from typing import List, Optional, Dict, Any +from typing import List, Optional, Dict, Any, Union from enum import Enum from datacrunch.http_client.http_client import HTTPClient @@ -43,6 +43,7 @@ class VolumeMountType(str, Enum): SCRATCH = "scratch" SECRET = "secret" MEMORY = "memory" + SHARED = "shared" class ContainerRegistryType(str, Enum): @@ -119,25 +120,88 @@ class EnvVar: @dataclass_json(undefined=Undefined.EXCLUDE) @dataclass class VolumeMount: - """Volume mount configuration for containers. + """Base class for volume mount configurations. Attributes: type: Type of volume mount. mount_path: Path where the volume should be mounted in the container. - size_in_mb: Size of the memory volume in megabytes, only used for memory volume mounts + size_in_mb: Size of the volume in megabytes. Deprecated: use MemoryMount for memory volumes instead. """ type: VolumeMountType mount_path: str - size_in_mb: Optional[int] = None + # Deprecated: use MemoryMount for memory volumes instead. + size_in_mb: Optional[int] = field(default=None, kw_only=True) -@dataclass_json +@dataclass_json(undefined=Undefined.EXCLUDE) @dataclass -class SecretMount: - mount_path: str +class GeneralStorageMount(VolumeMount): + """General storage volume mount configuration. + """ + + def __init__(self, mount_path: str): + """Initialize a general scratch volume mount. + + Args: + mount_path: Path where the volume should be mounted in the container. + """ + super().__init__(type=VolumeMountType.SCRATCH, mount_path=mount_path) + + +@dataclass_json(undefined=Undefined.EXCLUDE) +@dataclass +class SecretMount(VolumeMount): + """Secret volume mount configuration. + + A secret volume mount allows mounting secret files into the container. + + Attributes: + secret_name: The name of the fileset secret to mount. This secret must be created in advance, for example using `create_fileset_secret_from_file_paths` + file_names: List of file names that are part of the fileset secret. + """ + secret_name: str - type: VolumeMountType = VolumeMountType.SECRET + file_names: Optional[List[str]] = None + + def __init__(self, mount_path: str, secret_name: str, file_names: Optional[List[str]] = None): + self.secret_name = secret_name + self.file_names = file_names + super().__init__(type=VolumeMountType.SECRET, mount_path=mount_path) + + +@dataclass_json(undefined=Undefined.EXCLUDE) +@dataclass +class MemoryMount(VolumeMount): + """Memory volume mount configuration. + + A memory volume mount provides high-speed, ephemeral in-memory storage inside your container. + The mount path is currently hardcoded to /dev/shm and cannot be changed. + + Attributes: + size_in_mb: Size of the memory volume in megabytes. + """ + + size_in_mb: int + + def __init__(self, size_in_mb: int): + super().__init__(type=VolumeMountType.MEMORY, mount_path='/dev/shm') + self.size_in_mb = size_in_mb + + +@dataclass_json(undefined=Undefined.EXCLUDE) +@dataclass +class SharedFileSystemMount(VolumeMount): + """Shared filesystem volume mount configuration. + + A shared filesystem volume mount allows mounting a shared filesystem into the container. + """ + + volume_id: str # The ID of the shared filesystem volume to mount, needs to be created first + + def __init__(self, mount_path: str, volume_id: str): + super().__init__(type=VolumeMountType.SHARED, mount_path=mount_path) + self.volume_id = volume_id @dataclass_json @@ -155,7 +219,7 @@ class Container: volume_mounts: Optional list of volume mounts. """ - image: str + image: Union[str, dict] exposed_port: int name: Optional[str] = None healthcheck: Optional[HealthcheckSettings] = None diff --git a/examples/containers/container_deployments_example.py b/examples/containers/container_deployments_example.py index 8001fb4..e40c4f2 100644 --- a/examples/containers/container_deployments_example.py +++ b/examples/containers/container_deployments_example.py @@ -20,11 +20,11 @@ QueueLoadScalingTrigger, UtilizationScalingTrigger, HealthcheckSettings, - VolumeMount, + GeneralStorageMount, SecretMount, + SharedFileSystemMount, ContainerRegistrySettings, Deployment, - VolumeMountType, ContainerDeploymentStatus, ) @@ -97,23 +97,24 @@ def main() -> None: path="/health" ), volume_mounts=[ - # Shared memory volume - VolumeMount( - type=VolumeMountType.SCRATCH, + GeneralStorageMount( mount_path="/data" ), - # Fileset secret + # Optional: Fileset secret SecretMount( mount_path="/path/to/mount", secret_name="my-fileset-secret" # This fileset secret must be created beforehand - ) + ), + # Optional: Mount an existing shared filesystem volume + SharedFileSystemMount( + mount_path="/sfs", volume_id=""), ], env=[ # Secret environment variables needed to be added beforehand EnvVar( name="HF_TOKEN", # This is a reference to a secret already created - value_or_reference_to_secret="hf_token", + value_or_reference_to_secret="hf-token", type=EnvVarType.SECRET ), # Plain environment variables can be added directly diff --git a/setup.py b/setup.py index 99d5608..8d1b925 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,6 @@ }, classifiers=[ "Programming Language :: Python :: 3", - 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', @@ -41,5 +40,5 @@ "Operating System :: OS Independent", "Natural Language :: English" ], - python_requires='>=3.9', + python_requires='>=3.10', )