Merge branch 'master' into feature/inference_functionality

Author: tamirse

CHANGELOG.rst

@@ -1,6 +1,10 @@
 Changelog
 =========
 
+* Added environment variables to container deployment example
+* Updated examples image from 'fastai' to 'ubuntu-24.04-cuda-12.8-open-docker'
+* Consistent naming and load of credentials from env variables in examples
+
 v1.8.4 (2025-03-25)
 -------------------
 

datacrunch/datacrunch.py

@@ -30,6 +30,11 @@ def __init__(self, client_id: str, client_secret: str, base_url: str = "https://
         :type inference_key: str, optional
         """
 
+        # Validate that client_id and client_secret are not empty
+        if not client_id or not client_secret:
+            raise ValueError(
+                "client_id and client_secret must be provided")
+
         # Constants
         self.constants: Constants = Constants(base_url, VERSION)
         """Constants"""

examples/advanced_create_instance.py

@@ -20,13 +20,14 @@
 # Arbitrary duration for the example
 DURATION = 24 * 7  # one week
 
-# Get client secret from environment variable
-CLIENT_SECRET = os.environ['DATACRUNCH_CLIENT_SECRET']
-CLIENT_ID = 'Ibk5bdxV64lKAWOqYnvSi'  # Replace with your client ID
+# Get client secret and id from environment variables
+DATACRUNCH_CLIENT_ID = os.environ.get('DATACRUNCH_CLIENT_ID')
+DATACRUNCH_CLIENT_SECRET = os.environ.get('DATACRUNCH_CLIENT_SECRET')
 
 try:
     # Create datcrunch client
-    datacrunch = DataCrunchClient(CLIENT_ID, CLIENT_SECRET)
+    datacrunch = DataCrunchClient(
+        DATACRUNCH_CLIENT_ID, DATACRUNCH_CLIENT_SECRET)
 
     # Create new SSH key
     public_key = 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI0qq2Qjt5GPi7DKdcnBHOkvk8xNsG9dA607tnWagOkHC test_key'

examples/containers/compute_resources_example.py

@@ -1,7 +1,12 @@
+import os
 from datacrunch import DataCrunchClient
 from typing import List
 from datacrunch.containers.containers import ComputeResource
 
+# Get client secret and id from environment variables
+DATACRUNCH_CLIENT_ID = os.environ.get('DATACRUNCH_CLIENT_ID')
+DATACRUNCH_CLIENT_SECRET = os.environ.get('DATACRUNCH_CLIENT_SECRET')
+
 
 def list_all_compute_resources(client: DataCrunchClient) -> List[ComputeResource]:
     """List all available compute resources.
@@ -44,27 +49,25 @@ def list_compute_resources_by_size(client: DataCrunchClient, size: int) -> List[
 
 def main():
     # Initialize the client with your credentials
-    client = DataCrunchClient(
-        client_id="your_client_id",
-        client_secret="your_client_secret"
-    )
+    datacrunch = DataCrunchClient(
+        DATACRUNCH_CLIENT_ID, DATACRUNCH_CLIENT_SECRET)
 
     # Example 1: List all compute resources
-    print("\nAll compute resources:")
-    all_resources = list_all_compute_resources(client)
+    print("All compute resources:")
+    all_resources = list_all_compute_resources(datacrunch)
     for resource in all_resources:
         print(
             f"Name: {resource.name}, Size: {resource.size}, Available: {resource.is_available}")
 
     # Example 2: List available compute resources
-    print("\nAvailable compute resources:")
-    available_resources = list_available_compute_resources(client)
+    print("Available compute resources:")
+    available_resources = list_available_compute_resources(datacrunch)
     for resource in available_resources:
         print(f"Name: {resource.name}, Size: {resource.size}")
 
     # Example 3: List compute resources of size 8
-    print("\nCompute resources with size 8:")
-    size_8_resources = list_compute_resources_by_size(client, 8)
+    print("Compute resources with size 8:")
+    size_8_resources = list_compute_resources_by_size(datacrunch, 8)
     for resource in size_8_resources:
         print(f"Name: {resource.name}, Available: {resource.is_available}")
 

examples/containers/container_deployments_example.py

@@ -12,6 +12,8 @@
 from datacrunch.containers.containers import (
     Container,
     ComputeResource,
+    EnvVar,
+    EnvVarType,
     ScalingOptions,
     ScalingPolicy,
     ScalingTriggers,
@@ -29,12 +31,12 @@
 DEPLOYMENT_NAME = "my-deployment"
 IMAGE_NAME = "your-image-name:version"
 
-# Environment variables
+# Get client secret and id from environment variables
 DATACRUNCH_CLIENT_ID = os.environ.get('DATACRUNCH_CLIENT_ID')
 DATACRUNCH_CLIENT_SECRET = os.environ.get('DATACRUNCH_CLIENT_SECRET')
 
 # DataCrunch client instance
-datacrunch_client = None
+datacrunch = None
 
 
 def wait_for_deployment_health(client: DataCrunchClient, deployment_name: str, max_attempts: int = 10, delay: int = 30) -> bool:
@@ -79,15 +81,9 @@ def cleanup_resources(client: DataCrunchClient) -> None:
 def main() -> None:
     """Main function demonstrating deployment lifecycle management."""
     try:
-        # Check required environment variables
-        if not DATACRUNCH_CLIENT_ID or not DATACRUNCH_CLIENT_SECRET:
-            print(
-                "Please set DATACRUNCH_CLIENT_ID and DATACRUNCH_CLIENT_SECRET environment variables")
-            return
-
         # Initialize client
-        global datacrunch_client
-        datacrunch_client = DataCrunchClient(
+        global datacrunch
+        datacrunch = DataCrunchClient(
             DATACRUNCH_CLIENT_ID, DATACRUNCH_CLIENT_SECRET)
 
         # Create container configuration
@@ -104,6 +100,21 @@ def main() -> None:
                     type=VolumeMountType.SCRATCH,
                     mount_path="/data"
                 )
+            ],
+            env=[
+                # Secret environment variables needed to be added beforehand
+                EnvVar(
+                    name="HF_TOKEN",
+                    # This is a reference to a secret already created
+                    value_or_reference_to_secret="hf_token",
+                    type=EnvVarType.SECRET
+                ),
+                # Plain environment variables can be added directly
+                EnvVar(
+                    name="VERSION",
+                    value_or_reference_to_secret="1.5.2",
+                    type=EnvVarType.PLAIN
+                )
             ]
         )
 
@@ -143,19 +154,19 @@ def main() -> None:
         )
 
         # Create the deployment
-        created_deployment = datacrunch_client.containers.create_deployment(
+        created_deployment = datacrunch.containers.create_deployment(
             deployment)
         print(f"Created deployment: {created_deployment.name}")
 
         # Wait for deployment to be healthy
-        if not wait_for_deployment_health(datacrunch_client, DEPLOYMENT_NAME):
+        if not wait_for_deployment_health(datacrunch, DEPLOYMENT_NAME):
             print("Deployment health check failed")
-            cleanup_resources(datacrunch_client)
+            cleanup_resources(datacrunch)
             return
 
         # Update scaling configuration
         try:
-            deployment = datacrunch_client.containers.get_deployment_by_name(
+            deployment = datacrunch.containers.get_deployment_by_name(
                 DEPLOYMENT_NAME)
             # Create new scaling options with increased replica counts
             deployment.scaling = ScalingOptions(
@@ -177,7 +188,7 @@ def main() -> None:
                     )
                 )
             )
-            updated_deployment = datacrunch_client.containers.update_deployment(
+            updated_deployment = datacrunch.containers.update_deployment(
                 DEPLOYMENT_NAME, deployment)
             print(f"Updated deployment scaling: {updated_deployment.name}")
         except APIException as e:
@@ -186,33 +197,33 @@ def main() -> None:
         # Demonstrate deployment operations
         try:
             # Pause deployment
-            datacrunch_client.containers.pause_deployment(DEPLOYMENT_NAME)
+            datacrunch.containers.pause_deployment(DEPLOYMENT_NAME)
             print("Deployment paused")
             time.sleep(60)
 
             # Resume deployment
-            datacrunch_client.containers.resume_deployment(DEPLOYMENT_NAME)
+            datacrunch.containers.resume_deployment(DEPLOYMENT_NAME)
             print("Deployment resumed")
 
             # Restart deployment
-            datacrunch_client.containers.restart_deployment(DEPLOYMENT_NAME)
+            datacrunch.containers.restart_deployment(DEPLOYMENT_NAME)
             print("Deployment restarted")
 
             # Purge queue
-            datacrunch_client.containers.purge_deployment_queue(
+            datacrunch.containers.purge_deployment_queue(
                 DEPLOYMENT_NAME)
             print("Queue purged")
         except APIException as e:
             print(f"Error in deployment operations: {e}")
 
         # Clean up
-        cleanup_resources(datacrunch_client)
+        cleanup_resources(datacrunch)
 
     except Exception as e:
         print(f"Unexpected error: {e}")
         # Attempt cleanup even if there was an error
         try:
-            cleanup_resources(datacrunch_client)
+            cleanup_resources(datacrunch)
         except Exception as cleanup_error:
             print(f"Error during cleanup after failure: {cleanup_error}")
 

examples/containers/environment_variables_example.py

@@ -12,12 +12,12 @@
 from datacrunch import DataCrunchClient
 from typing import Dict, List
 
+# Get client secret and id from environment variables
 DATACRUNCH_CLIENT_ID = os.environ.get('DATACRUNCH_CLIENT_ID')
 DATACRUNCH_CLIENT_SECRET = os.environ.get('DATACRUNCH_CLIENT_SECRET')
 
 # Initialize DataCrunch client
-datacrunch_client = DataCrunchClient(client_id=DATACRUNCH_CLIENT_ID,
-                                     client_secret=DATACRUNCH_CLIENT_SECRET)
+datacrunch = DataCrunchClient(DATACRUNCH_CLIENT_ID, DATACRUNCH_CLIENT_SECRET)
 
 # Example deployment and container names
 DEPLOYMENT_NAME = "my-deployment"
@@ -36,13 +36,13 @@ def print_env_vars(env_vars: Dict[str, List[EnvVar]]) -> None:
 def main():
     # First, let's get the current environment variables
     print("Getting current environment variables...")
-    env_vars = datacrunch_client.containers.get_deployment_environment_variables(
+    env_vars = datacrunch.containers.get_deployment_environment_variables(
         DEPLOYMENT_NAME)
     print_env_vars(env_vars)
 
     # Create a new secret
     secret_name = "my-secret-key"
-    datacrunch_client.containers.create_secret(
+    datacrunch.containers.create_secret(
         secret_name,
         "my-secret-value"
     )
@@ -62,7 +62,7 @@ def main():
         )
     ]
 
-    env_vars = datacrunch_client.containers.add_deployment_environment_variables(
+    env_vars = datacrunch.containers.add_deployment_environment_variables(
         deployment_name=DEPLOYMENT_NAME,
         container_name=CONTAINER_NAME,
         env_vars=new_env_vars
@@ -79,7 +79,7 @@ def main():
         ),
     ]
 
-    env_vars = datacrunch_client.containers.update_deployment_environment_variables(
+    env_vars = datacrunch.containers.update_deployment_environment_variables(
         deployment_name=DEPLOYMENT_NAME,
         container_name=CONTAINER_NAME,
         env_vars=updated_env_vars
@@ -88,7 +88,7 @@ def main():
 
     # Delete environment variables
     print("\nDeleting environment variables...")
-    env_vars = datacrunch_client.containers.delete_deployment_environment_variables(
+    env_vars = datacrunch.containers.delete_deployment_environment_variables(
         deployment_name=DEPLOYMENT_NAME,
         container_name=CONTAINER_NAME,
         env_var_names=["DEBUG"]

examples/containers/registry_credentials_example.py

@@ -8,21 +8,20 @@
     CustomRegistryCredentials
 )
 
-# Environment variables
+# Get client secret and id from environment variables
 DATACRUNCH_CLIENT_ID = os.environ.get('DATACRUNCH_CLIENT_ID')
 DATACRUNCH_CLIENT_SECRET = os.environ.get('DATACRUNCH_CLIENT_SECRET')
 
 # Initialize DataCrunch client
-datacrunch_client = DataCrunchClient(client_id=DATACRUNCH_CLIENT_ID,
-                                     client_secret=DATACRUNCH_CLIENT_SECRET)
+datacrunch = DataCrunchClient(DATACRUNCH_CLIENT_ID, DATACRUNCH_CLIENT_SECRET)
 
 # Example 1: DockerHub Credentials
 dockerhub_creds = DockerHubCredentials(
     name="my-dockerhub-creds",
     username="your-dockerhub-username",
     access_token="your-dockerhub-access-token"
 )
-datacrunch_client.containers.add_registry_credentials(dockerhub_creds)
+datacrunch.containers.add_registry_credentials(dockerhub_creds)
 print("Created DockerHub credentials")
 
 # Example 2: GitHub Container Registry Credentials
@@ -31,7 +30,7 @@
     username="your-github-username",
     access_token="your-github-token"
 )
-datacrunch_client.containers.add_registry_credentials(github_creds)
+datacrunch.containers.add_registry_credentials(github_creds)
 print("Created GitHub credentials")
 
 # Example 3: Google Container Registry (GCR) Credentials
@@ -53,7 +52,7 @@
     name="my-gcr-creds",
     service_account_key=gcr_service_account_key
 )
-datacrunch_client.containers.add_registry_credentials(gcr_creds)
+datacrunch.containers.add_registry_credentials(gcr_creds)
 print("Created GCR credentials")
 
 # Example 4: AWS ECR Credentials
@@ -64,7 +63,7 @@
     region="eu-north-1",
     ecr_repo="887841266746.dkr.ecr.eu-north-1.amazonaws.com"
 )
-datacrunch_client.containers.add_registry_credentials(aws_creds)
+datacrunch.containers.add_registry_credentials(aws_creds)
 print("Created AWS ECR credentials")
 
 # Example 5: Custom Registry Credentials
@@ -80,13 +79,13 @@
     name="my-custom-registry-creds",
     docker_config_json=custom_docker_config
 )
-datacrunch_client.containers.add_registry_credentials(custom_creds)
+datacrunch.containers.add_registry_credentials(custom_creds)
 print("Created Custom registry credentials")
 
 # Delete all registry credentials
-datacrunch_client.containers.delete_registry_credentials('my-dockerhub-creds')
-datacrunch_client.containers.delete_registry_credentials('my-github-creds')
-datacrunch_client.containers.delete_registry_credentials('my-gcr-creds')
-datacrunch_client.containers.delete_registry_credentials('my-aws-ecr-creds')
-datacrunch_client.containers.delete_registry_credentials(
+datacrunch.containers.delete_registry_credentials('my-dockerhub-creds')
+datacrunch.containers.delete_registry_credentials('my-github-creds')
+datacrunch.containers.delete_registry_credentials('my-gcr-creds')
+datacrunch.containers.delete_registry_credentials('my-aws-ecr-creds')
+datacrunch.containers.delete_registry_credentials(
     'my-custom-registry-creds')