sample.env

# =============================================================================
# Unstract Workers Environment Configuration
# =============================================================================
# Copy this file to .env and update the values for your environment

# =============================================================================
# Core Configuration (REQUIRED)
# =============================================================================

# Django Backend URL - REQUIRED
# Docker (default): http://unstract-backend:8000
# Local development: http://localhost:8000
DJANGO_APP_BACKEND_URL=http://unstract-backend:8000

# Internal API Base URL - REQUIRED
# This is the full URL with /internal suffix for worker→backend communication
# Docker: http://unstract-backend:8000/internal
# Local: http://localhost:8000/internal
INTERNAL_API_BASE_URL=http://unstract-backend:8000/internal

# Internal API Configuration
INTERNAL_API_PREFIX=/internal
INTERNAL_API_VERSION=v1

# Internal Service API Key - REQUIRED
INTERNAL_SERVICE_API_KEY=dev-internal-key-123

# Internal API Connection Settings
INTERNAL_API_TIMEOUT=120
INTERNAL_API_RETRY_ATTEMPTS=3
INTERNAL_API_RETRY_BACKOFF_FACTOR=1.0

# Internal API Endpoint Prefixes
INTERNAL_API_HEALTH_PREFIX=v1/health/
INTERNAL_API_TOOL_PREFIX=v1/tool-execution/
INTERNAL_API_EXECUTION_PREFIX=v1/execution/
INTERNAL_API_WEBHOOK_PREFIX=v1/webhook/
INTERNAL_API_FILE_HISTORY_PREFIX=v1/file-history/
INTERNAL_API_WORKFLOW_PREFIX=v1/workflow-execution/
INTERNAL_API_ORGANIZATION_PREFIX=v1/organization/

# =============================================================================
# Celery Configuration
# =============================================================================

# Celery Broker (RabbitMQ) - REQUIRED
# These credentials must match your RabbitMQ configuration
CELERY_BROKER_BASE_URL=amqp://unstract-rabbitmq:5672//
CELERY_BROKER_USER=admin
CELERY_BROKER_PASS=password

# =============================================================================
# Database Configuration (REQUIRED)
# =============================================================================

# PostgreSQL (for Celery result backend) - REQUIRED
# These credentials must match your PostgreSQL configuration
DB_HOST=unstract-db
DB_USER=unstract_dev
DB_PASSWORD=unstract_pass
DB_NAME=unstract_db
DB_PORT=5432
DB_SCHEMA=unstract

# Celery Backend Database Schema
CELERY_BACKEND_DB_SCHEMA=public

# Redis (for caching and queues) - REQUIRED
REDIS_HOST=unstract-redis
REDIS_PORT=6379
REDIS_PASSWORD=
REDIS_USER=default
REDIS_DB=0

# Redis Sentinel mode for SocketIO log consumer
# When True: point REDIS_HOST to Sentinel K8s service DNS, REDIS_PORT to 26379
# REDIS_PASSWORD is reused for Sentinel auth.
REDIS_SENTINEL_MODE=False
REDIS_SENTINEL_MASTER_NAME=mymaster

# Cache-Specific Redis Configuration
CACHE_REDIS_ENABLED=true
CACHE_REDIS_HOST=unstract-redis
CACHE_REDIS_PORT=6379
CACHE_REDIS_DB=0
CACHE_REDIS_PASSWORD=
CACHE_REDIS_USERNAME=
CACHE_REDIS_SSL=false
CACHE_REDIS_SSL_CERT_REQS=required

# Cache Redis Sentinel mode for worker execution cache backend
# When True: point CACHE_REDIS_HOST to Sentinel K8s service DNS, CACHE_REDIS_PORT to 26379
CACHE_REDIS_SENTINEL_MODE=False

# =============================================================================
# Distributed Barrier
# =============================================================================
# Selects the substrate that coordinates chord-style fan-out + callback:
#   chord (default) — celery.chord(header)(body); current production behaviour
#   redis           — Redis DECR remaining:{exec_id} + RPUSH results
#                     aggregation. Replaces only the chord aggregation
#                     primitive; dispatch / retries / monitoring still ride
#                     on Celery. Avoids the documented silent-drop class
#                     in Celery's chord backend at large scale.
# Default chord keeps existing behaviour exactly — unset / missing env
# is safe.
WORKER_BARRIER_BACKEND=chord

# Optional separate Redis connection for the barrier counter +
# results-aggregation keys. Falls back to REDIS_* (the canonical worker
# Redis instance) if unset. Set these only if you want the barrier on a
# different Redis from the cache layer.
# WORKER_BARRIER_REDIS_HOST=unstract-redis
# WORKER_BARRIER_REDIS_PORT=6379
# WORKER_BARRIER_REDIS_DB=0
# WORKER_BARRIER_REDIS_PASSWORD=
# WORKER_BARRIER_REDIS_SENTINEL_MODE=False

# Belt-and-suspenders cleanup TTL for orphaned barrier counter +
# results keys. On the happy path the keys are explicitly DEL'd when
# the last task completes — TTL only matters when the counter stalls
# (e.g. some header tasks fail and the link never fires for them).
#
# MUST be strictly > longest plausible execution wall-clock. If TTL
# expires while tasks are still in flight, the next successful task
# fires a spurious callback with just its own result.
#
# 6h default gives 2-3x margin over FILE_PROCESSING_TASK_TIME_LIMIT
# (this sample ships 2h; docker/sample.env up to 3h; process_file_batch
# has max_retries=0 so no retry amplification). Raise this if your
# workflows can plausibly span >6h.
WORKER_BARRIER_KEY_TTL_SECONDS=21600

# Database URL (for fallback usage)
DATABASE_URL=postgresql://unstract_dev:unstract_pass@unstract-db:5432/unstract_db

# =============================================================================
# Worker Infrastructure Settings
# =============================================================================

# Worker Singleton Infrastructure - Controls shared resource management
# Setting to "false" prevents SIGSEGV crashes in prefork workers by ensuring
# each API client creates its own HTTP session after fork.
#
# Only set to "true" if:
#   - Using thread pool (not prefork)
#   - Backend and workers are tightly coupled (same process)
#   - Performance optimization is critical AND fork-safety is not needed
#
ENABLE_API_CLIENT_SINGLETON=false
DEBUG_API_CLIENT_INIT=false
WORKER_INFRASTRUCTURE_HEALTH_CHECK=true

# API Client Connection Pool Size
# Controls pool_connections (number of host connection pools) and pool_maxsize (max connections per pool).
# pool_connections = API_CLIENT_POOL_SIZE, pool_maxsize = API_CLIENT_POOL_SIZE * 2
API_CLIENT_POOL_SIZE=10

# Session lifecycle safety valve - reset singleton after N tasks (0=disabled)
# Only effective when ENABLE_API_CLIENT_SINGLETON=true
WORKER_SINGLETON_RESET_THRESHOLD=1000

# Config Caching
ENABLE_CONFIG_CACHE=true
CONFIG_CACHE_TTL=300

# Debug Settings
ENABLE_DEBUG_LOGGING=false
DEBUG_ORGANIZATION_CONTEXT=false

# Worker Concurrency
MAX_CONCURRENT_TASKS=10

# =============================================================================
# Worker Performance Settings
# =============================================================================

CELERY_WORKER_PREFETCH_MULTIPLIER=1
CELERY_TASK_ACKS_LATE=true
CELERY_WORKER_MAX_TASKS_PER_CHILD=1000

# =============================================================================
# Task Timeout Configuration (Celery Standard Naming Convention)
# =============================================================================
# Uses format: {WORKER_TYPE}_TASK_TIME_LIMIT and {WORKER_TYPE}_TASK_SOFT_TIME_LIMIT
#
# Resolution hierarchy:
#   1. Worker-specific: FILE_PROCESSING_TASK_TIME_LIMIT (highest priority)
#   2. General: TASK_TIME_LIMIT (fallback for all workers)
#   3. Code defaults (lowest priority)

# General Task Timeouts - Applies to all workers without specific overrides
TASK_TIME_LIMIT=3600                          # 1 hour - General hard timeout
TASK_SOFT_TIME_LIMIT=3300                     # 55 minutes - General soft timeout

# Worker-Specific Timeouts - Overrides general timeouts for specific worker types
FILE_PROCESSING_TASK_TIME_LIMIT=7200         # 2 hours - File processing hard timeout
FILE_PROCESSING_TASK_SOFT_TIME_LIMIT=6300    # 1h 45m - File processing soft timeout
CALLBACK_TASK_TIME_LIMIT=3600                # 1 hour - Callback hard timeout
CALLBACK_TASK_SOFT_TIME_LIMIT=3300           # 55 minutes - Callback soft timeout

# Retry Configuration
CELERY_TASK_DEFAULT_RETRY_DELAY=60
CELERY_TASK_MAX_RETRIES=3
CELERY_TASK_REJECT_ON_WORKER_LOST=false

# Advanced Celery Configuration
CELERY_WORKER_POOL_RESTARTS=true
CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP=true
CELERY_RESULT_CHORD_RETRY_INTERVAL=3.0

# =============================================================================
# Worker-Specific Configuration
# =============================================================================

# API Deployment Worker
API_DEPLOYMENT_WORKER_NAME=api-deployment-worker
API_DEPLOYMENT_HEALTH_PORT=8080
API_DEPLOYMENT_AUTOSCALE=4,1

# General Worker
GENERAL_WORKER_NAME=general-worker
GENERAL_HEALTH_PORT=8081
GENERAL_AUTOSCALE=6,2

# File Processing Worker
FILE_PROCESSING_WORKER_NAME=file-processing-worker
FILE_PROCESSING_HEALTH_PORT=8082
FILE_PROCESSING_AUTOSCALE=8,2

# Callback Worker
CALLBACK_WORKER_NAME=callback-worker
CALLBACK_HEALTH_PORT=8083
CALLBACK_AUTOSCALE=4,1

# Scheduler Worker
SCHEDULER_WORKER_NAME=scheduler-worker
SCHEDULER_HEALTH_PORT=8087
SCHEDULER_AUTOSCALE=2,1

# Executor Worker
EXECUTOR_WORKER_NAME=executor-worker
EXECUTOR_HEALTH_PORT=8088
EXECUTOR_AUTOSCALE=2,1
EXECUTOR_RESULT_TIMEOUT=3600
EXECUTOR_TASK_TIME_LIMIT=3600
EXECUTOR_TASK_SOFT_TIME_LIMIT=3300

# Notification Worker
NOTIFICATION_WORKER_NAME=notification-worker
NOTIFICATION_HEALTH_PORT=8085
NOTIFICATION_AUTOSCALE=4,1

# Log Consumer Worker
LOG_CONSUMER_WORKER_NAME=log-consumer-worker
LOG_CONSUMER_HEALTH_PORT=8086
LOG_CONSUMER_AUTOSCALE=2,1

# =============================================================================
# Logging Configuration
# =============================================================================

LOG_LEVEL=INFO
# Note: LOG_FORMAT removed - format is now hardcoded (not configurable)
# All workers use a single standardized format matching Django backend
DEFAULT_LOG_LEVEL=INFO
WORKER_VERSION=1.0.0
WORKER_INSTANCE_ID=dev-01

# Log History Configuration
ENABLE_LOG_HISTORY=true
LOG_HISTORY_CONSUMER_INTERVAL=30
LOGS_BATCH_LIMIT=30
LOGS_EXPIRATION_TIME_IN_SECOND=86400
LOG_HISTORY_QUEUE_NAME=log_history_queue

# Log Queue Size Protection
# Maximum number of logs in Redis queue before dropping new logs
LOG_QUEUE_MAX_SIZE=10000

# =============================================================================
# Queue Configuration
# =============================================================================

# Notification Queue Name
NOTIFICATION_QUEUE_NAME=notifications

# =============================================================================
# Backend Services
# =============================================================================

# Platform Service
PLATFORM_SERVICE_HOST=http://unstract-platform-service
PLATFORM_SERVICE_PORT=3001

# Prompt Service
PROMPT_HOST=http://unstract-prompt-service
PROMPT_PORT=3003

# X2Text Service
X2TEXT_HOST=http://unstract-x2text-service
X2TEXT_PORT=3004

# Tool Runner
UNSTRACT_RUNNER_HOST=http://unstract-runner
UNSTRACT_RUNNER_PORT=5002
UNSTRACT_RUNNER_API_TIMEOUT=300
UNSTRACT_RUNNER_API_RETRY_COUNT=5
UNSTRACT_RUNNER_API_BACKOFF_FACTOR=3

# =============================================================================
# File Storage Configuration
# =============================================================================

# File Storage Credentials (MinIO)
WORKFLOW_EXECUTION_FILE_STORAGE_CREDENTIALS='{"provider": "minio", "credentials": {"endpoint_url": "http://unstract-minio:9000", "key": "minio", "secret": "minio123"}}'
API_FILE_STORAGE_CREDENTIALS='{"provider": "minio", "credentials": {"endpoint_url": "http://unstract-minio:9000", "key": "minio", "secret": "minio123"}}'

# Remote storage for Prompt Studio / IDE flows. Must match backend/sample.env.
# Required by executor and ide-callback workers; missing/empty values raise FileStorageError in EnvHelper.get_storage().
PERMANENT_REMOTE_STORAGE='{"provider": "minio", "credentials": {"endpoint_url": "http://unstract-minio:9000", "key": "minio", "secret": "minio123"}}'
TEMPORARY_REMOTE_STORAGE='{"provider": "minio", "credentials": {"endpoint_url": "http://unstract-minio:9000", "key": "minio", "secret": "minio123"}}'
REMOTE_PROMPT_STUDIO_FILE_PATH=unstract/prompt-studio-data

# File Execution Configuration
WORKFLOW_EXECUTION_DIR_PREFIX=unstract/execution
API_EXECUTION_DIR_PREFIX=unstract/api
MAX_PARALLEL_FILE_BATCHES=1

# File Execution TTL Configuration
FILE_EXECUTION_TRACKER_TTL_IN_SECOND=18000
FILE_EXECUTION_TRACKER_COMPLETED_TTL_IN_SECOND=300

# Destination Processing TTL Configuration
DESTINATION_PROCESSING_STAGE_TTL_IN_SECOND=600
# Actual Redis lock TTL
DESTINATION_PROCESSING_LOCK_TTL_IN_SECOND=10

EXECUTION_RESULT_TTL_SECONDS=86400
EXECUTION_CACHE_TTL_SECONDS=86400
INSTANT_WF_POLLING_TIMEOUT=300

# Active File Execution cache in seconds
ACTIVE_FILE_CACHE_TTL=300

# Polling Grace Period for NOT_FOUND Status
# How long the status poller tolerates NOT_FOUND before treating it as failure
POLL_NOT_FOUND_GRACE_PERIOD=40

# Redis Retry Configuration
# Controls automatic retry behavior for transient Redis connection failures
# These settings apply to all Redis read operations in execution status trackers
# Max retry attempts after initial attempt before giving up (default: 4, total attempts: 5)
REDIS_RETRY_MAX_ATTEMPTS=4
# Exponential backoff multiplier in seconds (default: 0.5)
# With defaults (4 retries after initial = 5 total attempts):
# Retry delays: 0.5s, 1s, 2s, 4s = 5 total attempts over 7.5s
# Handles: network blips, Redis restarts (3-5s), connection pool exhaustion
REDIS_RETRY_BACKOFF_FACTOR=0.5

# =============================================================================
# Development Settings
# =============================================================================

DEBUG=false
TESTING=false
ENABLE_METRICS=true
ENABLE_FILE_HISTORY=true
ENABLE_WEBHOOK_DELIVERY=true

# Tool Registry
TOOL_REGISTRY_CONFIG_PATH=../unstract/tool-registry/tool_registry_config
TOOL_REGISTRY_STORAGE_CREDENTIALS='{"provider":"local"}'

# =============================================================================
# Optional Advanced Settings
# =============================================================================

# Health Checks
HEALTH_CHECK_INTERVAL=30
HEALTH_CHECK_TIMEOUT=10
METRICS_PORT=8080

# Circuit Breaker
CIRCUIT_BREAKER_FAILURE_THRESHOLD=5
CIRCUIT_BREAKER_RECOVERY_TIMEOUT=60

# Notifications
NOTIFICATION_TIMEOUT=5

# Cache
CACHE_TTL_SEC=10800

# Connection Pooling
CONNECTION_POOL_SIZE=10
CONNECTION_POOL_MAX_OVERFLOW=20

# Task Routing and Backup
ENABLE_PRIORITY_ROUTING=false
HIGH_PRIORITY_QUEUE_SUFFIX=_high
LOW_PRIORITY_QUEUE_SUFFIX=_low
ENABLE_TASK_BACKUP=false
BACKUP_INTERVAL=3600

# Feature Flags
ENABLE_DESTINATION_CONNECTORS=true
ENABLE_CLEANUP_TASKS=true

# Security (for production)
SECURE_SSL_REDIRECT=false
SESSION_COOKIE_SECURE=false
CSRF_COOKIE_SECURE=false

# Monitoring
SENTRY_DSN=
SENTRY_ENVIRONMENT=development

# Google Drive
GDRIVE_GOOGLE_SERVICE_ACCOUNT=
GDRIVE_GOOGLE_PROJECT_ID=
GOOGLE_STORAGE_ACCESS_KEY_ID=
GOOGLE_STORAGE_SECRET_ACCESS_KEY=
GOOGLE_STORAGE_BASE_URL=https://storage.googleapis.com

# Connector OAuth
SOCIAL_AUTH_EXTRA_DATA_EXPIRATION_TIME_IN_SECOND=3600
GOOGLE_OAUTH2_KEY=
GOOGLE_OAUTH2_SECRET=



# =============================================================================
# Local Development Overrides
# =============================================================================
# For local development (all services on host), change Docker service names to localhost:
# DJANGO_APP_BACKEND_URL=http://localhost:8000
# INTERNAL_API_BASE_URL=http://localhost:8000/internal
# CELERY_BROKER_BASE_URL=amqp://localhost:5672//
# DB_HOST=localhost
# REDIS_HOST=localhost
# CACHE_REDIS_HOST=localhost
# PLATFORM_SERVICE_HOST=http://localhost
# PROMPT_HOST=http://localhost
# X2TEXT_HOST=http://localhost
# UNSTRACT_RUNNER_HOST=http://localhost
# WORKFLOW_EXECUTION_FILE_STORAGE_CREDENTIALS={"provider": "minio", "credentials": {"endpoint_url": "http://localhost:9000", "key": "minio", "secret": "minio123"}}
# API_FILE_STORAGE_CREDENTIALS={"provider": "minio", "credentials": {"endpoint_url": "http://localhost:9000", "key": "minio", "secret": "minio123"}}

# Flipt Service
FLIPT_SERVICE_AVAILABLE=False
EVALUATION_SERVER_IP=unstract-flipt
EVALUATION_SERVER_PORT=9005
PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python