updated slurm & config templates for different embedding use cases

Author: NetZissou

configs/config_embed_image_folder_template.yaml

@@ -0,0 +1,85 @@
+# =============================================================================
+# Configuration File for Batch Image Embedding from Image Folders
+# =============================================================================
+# This configuration is optimized for processing image files directly from
+# a directory structure.
+# -----------------------------------------------------------------------------
+
+# ---------------------------
+# Model Configurations
+# ---------------------------
+# List one or more models to use for embedding.
+# For OpenCLIP models, see: https://github.com/mlfoundations/open_clip#pretrained-models
+models:
+  vit_b_32:
+    name: ViT-B-32
+    pretrained: openai
+  # Uncomment to add more models for multi-model processing
+  # vit_l_14:
+  #   name: ViT-L-14
+  #   pretrained: laion2b_s32b_b82k
+  # bioclip:
+  #   name: hf-hub:imageomics/bioclip
+  #   pretrained: null
+
+# ---------------------------
+# DataLoader Configurations
+# ---------------------------
+batch_size: 32          # Number of images per batch (adjust based on GPU memory)
+num_workers: 28         # Number of worker processes for data loading
+prefetch_factor: 32     # Number of batches prefetched by each worker
+
+# ---------------------------
+# Image Processing Settings
+# ---------------------------
+validate_images: false  # Set to true to validate all images can be opened with PIL
+                        # Slower startup but catches corrupted files
+
+# How to generate unique IDs from image file paths
+uuid_mode: filename     # Options:
+                        # - "filename": image001.jpg
+                        # - "relative": subfolder/image001.jpg
+                        # - "fullpath": /full/path/to/image001.jpg
+                        # - "hash": MD5 hash of full path
+
+# ---------------------------
+# Distributed Processing
+# ---------------------------
+evenly_distribute: true  # Distribute files based on size for load balancing
+stagger: false           # Stagger worker start times to reduce file system load
+
+# ---------------------------
+# Output Configurations
+# ---------------------------
+max_rows_per_file: 50000      # Maximum number of embeddings per output file
+out_prefix: image_embeds      # Prefix for output files
+
+# =============================================================================
+# IMAGE DIRECTORY REQUIREMENTS:
+# =============================================================================
+# Your image directory can have any structure:
+#
+# Flat structure:
+# /images/
+#   ├── image001.jpg
+#   ├── image002.png
+#   └── image003.jpeg
+#
+# Nested structure:
+# /images/
+#   ├── category1/
+#   │   ├── img1.jpg
+#   │   └── img2.png
+#   └── category2/
+#       ├── img3.jpg
+#       └── img4.png
+#
+# Supported formats: .jpg, .jpeg, .png, .bmp, .tif, .tiff, .webp
+# All images are automatically converted to RGB mode for processing.
+#
+# UUID GENERATION MODES:
+# - filename: Good for flat directories with unique filenames
+# - relative: Good for nested directories where path info is important
+# - fullpath: Good when you need absolute path traceability
+# - hash: Good for very long paths or when you want anonymized IDs
+# =============================================================================

configs/config_embed_parquet_template.yaml

@@ -0,0 +1,89 @@
+# =============================================================================
+# Configuration File for Batch Image Embedding from Parquet Files
+# =============================================================================
+# This configuration is optimized for processing Parquet files containing
+# encoded image data with metadata.
+# -----------------------------------------------------------------------------
+
+# ---------------------------
+# Model Configurations
+# ---------------------------
+# List one or more models to use for embedding.
+# For OpenCLIP models, see: https://github.com/mlfoundations/open_clip#pretrained-models
+models:
+  bioclip:
+    name: hf-hub:imageomics/bioclip
+    pretrained: null
+  # Uncomment to add more models for multi-model processing
+  # vit_b_32:
+  #   name: ViT-B-32
+  #   pretrained: openai
+  # vit_l_14:
+  #   name: ViT-L-14
+  #   pretrained: laion2b_s32b_b82k
+
+# ---------------------------
+# DataLoader Configurations
+# ---------------------------
+batch_size: 32          # Number of images per batch (adjust based on GPU memory)
+num_workers: 28         # Number of worker processes for data loading
+prefetch_factor: 32     # Number of batches prefetched by each worker
+
+# ---------------------------
+# Parquet-Specific Settings
+# ---------------------------
+read_batch_size: 256    # Number of rows to read from Parquet at a time
+                        # Larger values = more memory usage but potentially faster I/O
+
+# Columns to read from Parquet files (must exist in your data)
+read_columns:
+  - uuid              # [REQUIRED] Unique identifier for each image
+  - image             # [REQUIRED] Encoded image bytes (JPEG, PNG, etc.)
+  - original_size     # [OPTIONAL] Original image dimensions
+  - resized_size      # [OPTIONAL] Resized image dimensions
+
+# ---------------------------
+# Distributed Processing
+# ---------------------------
+evenly_distribute: true  # Distribute files based on size for load balancing
+stagger: false           # Stagger worker start times to reduce I/O congestion
+
+# ---------------------------
+# Output Configurations
+# ---------------------------
+max_rows_per_file: 50000    # Maximum number of embeddings per output file
+out_prefix: parquet_embeds  # Prefix for output files
+
+# =============================================================================
+# PARQUET DATA REQUIREMENTS:
+# =============================================================================
+# Your Parquet files must contain:
+# 1. 'uuid' column: Unique string identifier for each image
+# 2. 'image' column: Image data encoded as bytes (from PIL Image.save() to BytesIO)
+# 3. Optional metadata columns as specified in read_columns
+#
+# Example of creating compatible Parquet data:
+# ```python
+# import io
+# from PIL import Image
+# import pandas as pd
+# import pyarrow.parquet as pq
+# 
+# # Encode image to bytes
+# img = Image.open('image.jpg')
+# img_bytes = io.BytesIO()
+# img.save(img_bytes, format='JPEG')
+# img_bytes = img_bytes.getvalue()
+# 
+# # Create DataFrame
+# df = pd.DataFrame({
+#     'uuid': ['img_001'],
+#     'image': [img_bytes],
+#     'original_size': [(1024, 768)],
+#     'resized_size': [(224, 224)]
+# })
+# 
+# # Save to Parquet
+# df.to_parquet('images.parquet')
+# ```
+# =============================================================================

configs/config_embed_template.yaml

@@ -0,0 +1,100 @@
+#!/bin/bash
+#SBATCH --job-name=embed_images_job            # [REQUIRED] Set a descriptive job name
+#SBATCH --nodes=NUM_NODES                      # [REQUIRED] Number of nodes to use
+#SBATCH --ntasks-per-node=TASKS_PER_NODE       # [RECOMMENDED] Number of tasks per node
+#SBATCH --gpus-per-task=1                      # [REQUIRED] Number of GPUs per task (set to 1)
+#SBATCH --cpus-per-task=CPUS_PER_TASK          # [RECOMMENDED] Number of CPU cores per task (e.g., 48)
+#SBATCH --partition=PARTITION_NAME             # [REQUIRED] Partition/queue name (e.g., gpu, gpu-exp)
+#SBATCH --time=HH:MM:SS                        # [REQUIRED] Walltime limit (e.g., 6:00:00)
+#SBATCH --output=logs/embed_images_%j.out      # [OPTIONAL] Stdout log file (%j = job ID)
+#SBATCH --error=logs/embed_images_%j.err       # [OPTIONAL] Stderr log file
+#SBATCH --account=ACCOUNT_NAME                 # [REQUIRED] Project account for allocation
+#SBATCH --mail-type=ALL                        # [OPTIONAL] Email notifications (BEGIN, END, FAIL, ALL)
+#SBATCH --mail-user=YOUR_EMAIL@domain.edu      # [OPTIONAL] Email address for notifications
+
+# === Load modules and activate environment ===
+module load cuda/VERSION                       # [REQUIRED] Load CUDA module (e.g., cuda/12.4.1)
+source /path/to/your/venv/bin/activate         # [REQUIRED] Activate your Python virtual environment
+
+# === Ensure package is installed ===
+# Make sure hpc-inference package is installed in your environment
+which python                                   # Print Python path for debugging
+
+# === Set data paths ===
+TARGET_DIR="/path/to/your/image_directory"     # [REQUIRED] Directory containing input images
+OUTPUT_DIR="/path/to/your/output_dir"          # [REQUIRED] Directory to save output embeddings
+
+# === Choose your configuration method ===
+# Option 1: Use config file (RECOMMENDED for production)
+CONFIG_FILE="/path/to/your/images_config.yaml" # Path to YAML config file
+
+srun python -m hpc_inference.inference.embed.open_clip_embed \
+    "${TARGET_DIR}" \
+    "${OUTPUT_DIR}" \
+    --input_type images \
+    --config "${CONFIG_FILE}"
+
+# Option 2: Use command line arguments (for quick testing)
+# Uncomment and modify the lines below, comment out the config version above
+#
+# srun python -m hpc_inference.inference.embed.open_clip_embed \
+#     "${TARGET_DIR}" \
+#     "${OUTPUT_DIR}" \
+#     --input_type images \
+#     --model_name "ViT-B-32" \
+#     --pretrained "openai" \
+#     --batch_size 32 \
+#     --num_workers 28 \
+#     --prefetch_factor 32 \
+#     --max_rows_per_file 10000 \
+#     --out_prefix "embed_results" \
+#     --uuid_mode filename \
+#     --evenly_distribute \
+#     --stagger \
+#     --validate_images
+
+
+# -------------------------------
+# IMAGE FOLDER-SPECIFIC PARAMETERS
+# -------------------------------
+# --input_type images:     Tells the script to process image files from a directory
+# --uuid_mode:             How to generate unique IDs from image paths:
+#                          - "filename": Use just the filename (image001.jpg)
+#                          - "relative": Use relative path from TARGET_DIR (subfolder/image001.jpg)
+#                          - "fullpath": Use full absolute path (/full/path/to/image001.jpg)
+#                          - "hash": Use MD5 hash of the full path (a1b2c3d4e5f6g7h8)
+# --validate_images:       [OPTIONAL] Validate that all images can be opened with PIL
+#                          Slower but safer - catches corrupted files before processing
+# --file_list:             NOT applicable for image folders (will cause error)
+
+# SUPPORTED IMAGE FORMATS:
+# .jpg, .jpeg, .png, .bmp, .tif, .tiff, .webp
+# Images are automatically converted to RGB mode for model processing
+
+# DIRECTORY STRUCTURE:
+# TARGET_DIR can contain:
+# - Flat structure: /images/img1.jpg, /images/img2.jpg, ...
+# - Nested structure: /images/class1/img1.jpg, /images/class2/img2.jpg, ...
+# All .jpg, .jpeg, .png, etc. files will be found recursively
+
+# -------------------------------
+# SLURM Template Field Explanations
+# -------------------------------
+# --job-name:          Name for your job in the queue/monitoring system.
+# --nodes:             Number of nodes to allocate for the job.
+# --gpus-per-task:     Number of GPUs per task (set to 1 unless using model parallelism).
+# --cpus-per-task:     Number of CPU cores per task (should match or exceed your data loader workers).
+# --ntasks-per-node:   Number of parallel tasks per node.
+#                      For image processing, balance between available GPUs and I/O capacity.
+# --partition:         Cluster partition/queue to submit to (e.g., gpu, gpu-exp).
+# --time:              Maximum walltime for the job (format: HH:MM:SS).
+# --output:            Path for standard output log file (use %j for job ID).
+# --error:             Path for standard error log file.
+# --account:           Your allocation/project account for resource usage.
+
+# PERFORMANCE TIPS FOR IMAGE FOLDERS:
+# - Use --evenly_distribute for better load balancing when file sizes vary
+# - Use --validate_images if you suspect corrupted files (adds startup time)
+# - Consider --uuid_mode based on your downstream analysis needs
+# - Use --stagger to reduce file system stress during startup
+# - For large datasets, consider converting to Parquet format first for better performance