Merge pull request #40 from DataCrunch-io/feature/filesecret-management

support for fileset management and example

Author: tamirse

CHANGELOG.rst

@@ -1,6 +1,8 @@
 Changelog
 =========
 
+* Added support for fileset secrets
+
 v1.11.0 (2025-04-28)
 --------------------
 

datacrunch/containers/containers.py

@@ -4,6 +4,8 @@
 creation, updates, deletion, and monitoring of containerized applications.
 """
 
+import base64
+import os
 from dataclasses import dataclass, field
 from dataclasses_json import dataclass_json, Undefined  # type: ignore
 from typing import List, Optional, Dict, Any
@@ -18,6 +20,7 @@
 SERVERLESS_COMPUTE_RESOURCES_ENDPOINT = '/serverless-compute-resources'
 CONTAINER_REGISTRY_CREDENTIALS_ENDPOINT = '/container-registry-credentials'
 SECRETS_ENDPOINT = '/secrets'
+FILESET_SECRETS_ENDPOINT = '/file-secrets'
 
 
 class EnvVarType(str, Enum):
@@ -27,6 +30,13 @@ class EnvVarType(str, Enum):
     SECRET = "secret"
 
 
+class SecretType(str, Enum):
+    """Types of secrets that can be set in containers."""
+
+    GENERIC = "generic"  # Regular secret, can be used in env vars
+    FILESET = "file-secret"  # A file secret that can be mounted into the container
+
+
 class VolumeMountType(str, Enum):
     """Types of volume mounts that can be configured for containers."""
 
@@ -446,10 +456,12 @@ class Secret:
     Attributes:
         name: Name of the secret.
         created_at: Timestamp when the secret was created.
+        secret_type: Type of the secret.
     """
 
     name: str
     created_at: str
+    secret_type: SecretType
 
 
 @dataclass_json
@@ -909,6 +921,7 @@ def get_secrets(self) -> List[Secret]:
             List[Secret]: List of all secrets.
         """
         response = self.client.get(SECRETS_ENDPOINT)
+        print(response.json())
         return [Secret.from_dict(secret) for secret in response.json()]
 
     def create_secret(self, name: str, value: str) -> None:
@@ -956,3 +969,42 @@ def delete_registry_credentials(self, credentials_name: str) -> None:
         """
         self.client.delete(
             f"{CONTAINER_REGISTRY_CREDENTIALS_ENDPOINT}/{credentials_name}")
+
+    def get_fileset_secrets(self) -> List[Secret]:
+        """Retrieves all fileset secrets.
+
+        Returns:
+           List of all fileset secrets.
+        """
+        response = self.client.get(FILESET_SECRETS_ENDPOINT)
+        return [Secret.from_dict(secret) for secret in response.json()]
+
+    def delete_fileset_secret(self, secret_name: str) -> None:
+        """Deletes a fileset secret.
+
+        Args:
+            secret_name: Name of the secret to delete.
+        """
+        self.client.delete(f"{FILESET_SECRETS_ENDPOINT}/{secret_name}")
+
+    def create_fileset_secret_from_file_paths(self, secret_name: str, file_paths: List[str]) -> None:
+        """Creates a new fileset secret.
+        A fileset secret is a secret that contains several files, 
+        and can be used to mount a directory with the files in a container.
+
+        Args:
+            secret_name: Name of the secret.
+            file_paths: List of file paths to include in the secret.
+        """
+        processed_files = []
+        for file_path in file_paths:
+            with open(file_path, "rb") as f:
+                base64_content = base64.b64encode(f.read()).decode("utf-8")
+                processed_files.append({
+                    "file_name": os.path.basename(file_path),
+                    "base64_content": base64_content
+                })
+        self.client.post(FILESET_SECRETS_ENDPOINT, {
+            "name": secret_name,
+            "files": processed_files
+        })

docs/source/examples/containers/fileset_secrets.rst

@@ -0,0 +1,9 @@
+Fileset Secrets 
+===============
+
+This example shows how to manage fileset secrets for container deployments in DataCrunch.
+Fileset secrets are a way to mount a directory with files into a container.
+
+.. literalinclude:: ../../../../examples/containers/fileset_secret_example.py
+   :language: python
+   :caption: Fileset Secrets 

docs/source/examples/containers/index.rst

@@ -12,6 +12,7 @@ This section contains examples demonstrating how to work with containers in Data
    environment_variables
    registry_credentials
    secrets
+   fileset_secrets
    sglang
    scaling 
    inference_async

examples/containers/container_deployments_example.py

@@ -96,9 +96,16 @@ def main() -> None:
                 path="/health"
             ),
             volume_mounts=[
+                # Shared memory volume
                 VolumeMount(
                     type=VolumeMountType.SCRATCH,
                     mount_path="/data"
+                ),
+                # Fileset secret
+                VolumeMount(
+                    type=VolumeMountType.SECRET,
+                    mount_path="/path/to/mount",
+                    name="my-fileset-secret"  # This fileset secret must be created beforehand
                 )
             ],
             env=[

examples/containers/fileset_secret_example.py

@@ -0,0 +1,28 @@
+import os
+from datacrunch import DataCrunchClient
+
+# Fileset secrets are a way to mount sensitive files like API keys, certs, and credentials securely inside a container, without hardcoding them in the image or env vars.
+# This example demonstrates how to create a fileset secret containing two files from your local filesystem
+
+# Get client secret and id from environment variables
+DATACRUNCH_CLIENT_ID = os.environ.get('DATACRUNCH_CLIENT_ID')
+DATACRUNCH_CLIENT_SECRET = os.environ.get('DATACRUNCH_CLIENT_SECRET')
+
+# Initialize the client with your credentials
+datacrunch = DataCrunchClient(DATACRUNCH_CLIENT_ID, DATACRUNCH_CLIENT_SECRET)
+
+# Define the secret name and the file paths from your local filesystem where this script is running
+SECRET_NAME = "my-fileset-secret"
+RELATIVE_FILE_PATH = "./relative-path/file1.txt"
+ABSOLUTE_FILE_PATH = "/home/username/absolute-path/file2.json"
+
+# Create the fileset secret that has 2 files
+fileset_secret = datacrunch.containers.create_fileset_secret_from_file_paths(
+    secret_name=SECRET_NAME, file_paths=[RELATIVE_FILE_PATH, ABSOLUTE_FILE_PATH])
+
+# Get the secret
+secrets = datacrunch.containers.get_fileset_secrets()
+print(secrets)
+
+# Delete the secret
+datacrunch.containers.delete_fileset_secret(secret_name=SECRET_NAME)

tests/unit_tests/containers/test_containers.py

@@ -134,7 +134,8 @@
 SECRETS_DATA = [
     {
         "name": SECRET_NAME,
-        "created_at": "2023-01-01T00:00:00+00:00"
+        "created_at": "2023-01-01T00:00:00+00:00",
+        "secret_type": "generic"
     }
 ]