1- # Firecrawl Production Environment Configuration
2- # Copy this file to .env and update the values for your deployment
3-
4- # =============================================================================
5- # CORE CONFIGURATION
6- # =============================================================================
7-
8- # API Configuration
9- FIRECRAWL_API_KEY = 7877e105e5f7b9ec3edf4a8eec5059ab9914efef1b30fe232f59ff31cb8e6fcf
10- BULL_AUTH_KEY = firecrawl-admin-2024
11-
12- # Port Configuration
13- FIRECRAWL_PORT = 3002
14- OLLAMA_PORT = 11434
15- NGINX_PORT = 80
16- NGINX_SSL_PORT = 443
17-
18- # =============================================================================
19- # AI/LLM CONFIGURATION - GEMINI 2.5 FLASH + OLLAMA EMBEDDINGS
20- # =============================================================================
21-
22- # Gemini 2.5 Flash for main processing (300k requests/month limit)
1+ NODE_ENV = production
2+ PORT = 3002
3+ HOST = 0.0.0.0
4+ USE_DB_AUTHENTICATION = false
235GEMINI_API_KEY = AIzaSyAJWyUi0s9DHtRorkuDVnpoxDXqDGTqNdw
24- MODEL_NAME = gemini-2.5-flash
25- LLM_API_KEY = AIzaSyAJWyUi0s9DHtRorkuDVnpoxDXqDGTqNdw
26-
27- # Ollama for local embeddings (cost-effective)
28- OLLAMA_BASE_URL = http://ollama:11434
6+ OPENAI_API_KEY = AIzaSyAJWyUi0s9DHtRorkuDVnpoxDXqDGTqNdw
7+ OPENAI_BASE_URL = https://generativelanguage.googleapis.com/v1beta
8+ MODEL_NAME = gemini-2.5-flash-latest
9+ OLLAMA_BASE_URL = http://host.docker.internal:11434
2910MODEL_EMBEDDING_NAME = nomic-embed-text
30- OLLAMA_MODEL = nomic-embed-text
31-
32- # =============================================================================
33- # WEBSHARE PROXY CONFIGURATION - 100 PROXIES FOR 300K REQUESTS/MONTH
34- # =============================================================================
35-
36- # Update these with your Webshare.io credentials
37- WEBSHARE_USERNAME = your-webshare-username-here
38- WEBSHARE_PASSWORD = your-webshare-password-here
39-
40- # Proxy settings for high-volume scraping
41- PROXY_SERVER = p.webshare.io:80
42- USE_PROXY = true
43- ROTATING_PROXY = true
44- PROXY_ROTATION_COUNT = 100
45-
46- # =============================================================================
47- # PERFORMANCE CONFIGURATION FOR 300K REQUESTS/MONTH
48- # =============================================================================
49-
50- # Worker Configuration (optimized for high throughput)
51- NUM_WORKERS_PER_QUEUE = 8
52- SCRAPE_RATE_LIMIT_CRAWL_PARALLEL = 4
53- SCRAPE_RATE_LIMIT_CRAWL_PARALLEL_SCALE = 20
54-
55- # Rate Limiting
56- RATE_LIMIT_TEST_MODE = false
57-
58- # =============================================================================
59- # SECURITY & MONITORING (OPTIONAL)
60- # =============================================================================
61-
62- # Leave empty for self-hosted deployment
63- SLACK_WEBHOOK_URL =
64- POSTHOG_API_KEY =
65- POSTHOG_HOST =
66- SENTRY_DSN =
67- LOGTAIL_KEY =
68-
69- # Database (Optional - uses memory if not configured)
70- USE_DB_AUTHENTICATION = false
71- SUPABASE_ANON_TOKEN =
72- SUPABASE_URL =
73- SUPABASE_SERVICE_TOKEN =
74-
75- # Search APIs (Optional)
76- SERPER_API_KEY =
77- SEARCHAPI_API_KEY =
78-
79- # =============================================================================
80- # DOCKER CONFIGURATION
81- # =============================================================================
82-
83- # Redis Configuration
8411REDIS_URL = redis://redis:6379
85- REDIS_RATE_LIMIT_URL = redis://redis:6379/1
86-
87- # Service URLs
88- PLAYWRIGHT_MICROSERVICE_URL = http://playwright-service:3000
89- FIRE_ENGINE_BETA_URL = http://fire-engine:8080
90-
91- # =============================================================================
92- # ENVIRONMENT SETTINGS
93- # =============================================================================
94-
95- NODE_ENV = production
96- HOST = 0.0.0.0
97- LOGGING_LEVEL = info
98- BLOCK_MEDIA = true
99-
100- # =============================================================================
101- # CONFIGURATION NOTES
102- # =============================================================================
103-
104- # 1. Gemini 2.5 Flash Configuration:
105- # - Optimized for speed and cost-effectiveness
106- # - 300k requests/month should handle your target volume
107- # - Replace GEMINI_API_KEY with your actual key
108-
109- # 2. Ollama Embeddings:
110- # - Uses nomic-embed-text model for local embedding generation
111- # - Significantly reduces costs compared to cloud embedding APIs
112- # - Model will be auto-downloaded on first startup
113-
114- # 3. Webshare Proxy Setup:
115- # - 100 rotating proxies for high-volume scraping
116- # - Update WEBSHARE_USERNAME and WEBSHARE_PASSWORD
117- # - Supports 300k+ requests per month with rotation
118-
119- # 4. Performance Tuning:
120- # - 8 workers per queue for high concurrency
121- # - Optimized rate limiting for sustained throughput
122- # - Can scale horizontally by adding more worker containers
123-
124- # 5. Security:
125- # - All optional monitoring services can be left empty
126- # - FIRECRAWL_API_KEY should be regenerated for production
127- # - Consider setting up proper SSL certificates
12+ REDIS_RATE_LIMIT_URL = redis://redis:6379
13+ PLAYWRIGHT_MICROSERVICE_URL = http://playwright-service:3000/scrape
14+ PROXY_SERVER = http://xcuuwcfa:yccpzo0b9nth@45.61.100.172:6440
15+ PROXY_USERNAME = xcuuwcfa
16+ PROXY_PASSWORD = yccpzo0b9nth
17+ WEBSHARE_API_KEY = 7tk8w3pjogiwpwpf84y0m8wc6zj6yrn8ycvhrrz8
18+ TEST_API_KEY = 7877e105e5f7b9ec3edf4a8eec5059ab9914efef1b30fe232f59ff31cb8e6fcf
19+ BULL_AUTH_KEY = firecrawl-bull-2024-secure
20+ MAX_CPU = 0.8
21+ MAX_RAM = 0.8
0 commit comments