add high velocity ai pipeline

Author: ShaliniAnandaPhD

high_velocity_AI_pipeline/.env.example

@@ -0,0 +1,189 @@
+# High-Velocity AI Pipeline - Environment Configuration Template
+# Copy this file to .env and fill in your actual values
+
+# =============================================================================
+# API KEYS (REQUIRED)
+# =============================================================================
+
+# OpenAI API Key for GPT-4 Standard Agent
+# Get from: https://platform.openai.com/api-keys
+OPENAI_API_KEY=sk-your-openai-key-here
+
+# GROQ API Key for Ultra-Fast Agent  
+# Get from: https://console.groq.com/keys
+GROQ_API_KEY=gsk_your-groq-key-here
+
+# Weights & Biases API Key for Weave Tracing (Optional)
+# Get from: https://wandb.ai/authorize
+WANDB_API_KEY=your-wandb-key-here
+
+# =============================================================================
+# PERFORMANCE THRESHOLDS (Tuned from successful experiment)
+# =============================================================================
+
+# Latency threshold to trigger swap to ultra-fast agent (ms)
+PIPELINE_LATENCY_THRESHOLD_MS=100
+
+# Safe latency threshold for swap back to standard agent (ms)
+PIPELINE_SAFE_LATENCY_THRESHOLD_MS=70
+
+# Safe throughput threshold for swap back (msg/sec)
+PIPELINE_SAFE_THROUGHPUT_THRESHOLD=600
+
+# Cooldown period between swaps (seconds)
+PIPELINE_COOLDOWN_PERIOD_SECONDS=20
+
+# =============================================================================
+# PIPELINE CONFIGURATION
+# =============================================================================
+
+# Target throughput (messages per second)
+PIPELINE_TARGET_THROUGHPUT=800
+
+# Message batch size
+PIPELINE_MESSAGE_BATCH_SIZE=100
+
+# Batch processing interval (seconds)
+PIPELINE_BATCH_INTERVAL_SECONDS=0.125
+
+# Default agent type (standard or ultra_fast)
+PIPELINE_DEFAULT_AGENT=standard
+
+# Maximum agent swaps per session
+PIPELINE_MAX_AGENT_SWAPS=50
+
+# =============================================================================
+# SYSTEM LIMITS
+# =============================================================================
+
+# Maximum message queue size
+PIPELINE_MAX_MESSAGE_QUEUE_SIZE=10000
+
+# Maximum memory usage (MB)
+PIPELINE_MAX_MEMORY_USAGE_MB=2048
+
+# Maximum CPU usage (%)
+PIPELINE_MAX_CPU_USAGE_PERCENT=80.0
+
+# =============================================================================
+# MONITORING CONFIGURATION
+# =============================================================================
+
+# Enable CSV export for analysis
+PIPELINE_ENABLE_CSV_EXPORT=true
+
+# Enable Weave tracing for observability
+PIPELINE_ENABLE_WEAVE_TRACING=true
+
+# Metrics update interval (seconds)
+PIPELINE_METRICS_UPDATE_INTERVAL_SECONDS=1.0
+
+# Export directory for CSV files and reports
+PIPELINE_EXPORT_DIRECTORY=exports
+
+# Custom CSV filename (optional)
+PIPELINE_EXPORT_CSV_FILENAME=
+
+# Enable JSON report generation
+PIPELINE_EXPORT_JSON_REPORTS=true
+
+# =============================================================================
+# CIRCUIT BREAKER SETTINGS
+# =============================================================================
+
+# Number of failures to open circuit breaker
+PIPELINE_CIRCUIT_BREAKER_FAILURE_THRESHOLD=5
+
+# Timeout before testing recovery (seconds)
+PIPELINE_CIRCUIT_BREAKER_TIMEOUT_SECONDS=30.0
+
+# Consecutive successes needed to close circuit
+PIPELINE_CIRCUIT_BREAKER_SUCCESS_THRESHOLD=3
+
+# =============================================================================
+# MARKET DATA GENERATION
+# =============================================================================
+
+# Enable market volatility patterns
+PIPELINE_MARKET_VOLATILITY_ENABLED=true
+
+# Number of market symbols to simulate
+PIPELINE_MARKET_SYMBOLS_COUNT=12
+
+# Market condition change interval (seconds)
+PIPELINE_MARKET_CONDITION_CHANGE_INTERVAL_SECONDS=30.0
+
+# =============================================================================
+# LOGGING CONFIGURATION
+# =============================================================================
+
+# Log level (DEBUG, INFO, WARNING, ERROR)
+PIPELINE_LOG_LEVEL=INFO
+
+# Log file path (optional, will log to console if not set)
+PIPELINE_LOG_FILE=logs/high_velocity_pipeline.log
+
+# Log format string
+PIPELINE_LOG_FORMAT=%(asctime)s - %(name)s - %(levelname)s - %(message)s
+
+# =============================================================================
+# AI MODEL CONFIGURATION
+# =============================================================================
+
+# OpenAI model for standard agent
+OPENAI_MODEL=gpt-4-turbo-preview
+
+# GROQ model for ultra-fast agent
+GROQ_MODEL=llama3-70b-8192
+
+# API timeout (seconds)
+API_TIMEOUT_SECONDS=30.0
+
+# Maximum API retries
+API_MAX_RETRIES=3
+
+# =============================================================================
+# DEVELOPMENT/TESTING OVERRIDES
+# =============================================================================
+
+# Set to 'true' to enable development mode
+DEVELOPMENT_MODE=false
+
+# Set to 'true' to enable debug logging
+DEBUG_MODE=false
+
+# Set to 'true' to disable external API calls (for testing)
+MOCK_API_CALLS=false
+
+# =============================================================================
+# DEPLOYMENT CONFIGURATION
+# =============================================================================
+
+# Deployment environment (development, staging, production)
+DEPLOYMENT_ENV=production
+
+# Instance identifier for multi-instance deployments
+INSTANCE_ID=hvp-001
+
+# Health check port
+HEALTH_CHECK_PORT=8080
+
+# Metrics export port
+METRICS_PORT=8081
+
+# =============================================================================
+# SECURITY SETTINGS
+# =============================================================================
+
+# Enable API key rotation
+ENABLE_API_KEY_ROTATION=false
+
+# API key rotation interval (hours)
+API_KEY_ROTATION_INTERVAL_HOURS=24
+
+# Enable request rate limiting
+ENABLE_RATE_LIMITING=true
+
+# Maximum requests per minute
+MAX_REQUESTS_PER_MINUTE=1000
+