CrawlGuard/config.example.yaml at main · gitstq/CrawlGuard · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
# CrawlGuard - AI Crawler Intelligent Protection Engine
# Example configuration file
# Copy this to crawlguard.yaml and customize as needed

# Server settings
server:
  listen_host: "0.0.0.0"          # Address to bind to
  listen_port: 8080                # Proxy listen port
  target_host: "http://localhost:3000"  # Upstream server URL
  workers: 1                       # Number of worker threads

# Dashboard settings
dashboard:
  enabled: true                    # Enable web dashboard
  port: 8081                       # Dashboard port (must differ from proxy port)
  host: "0.0.0.0"                  # Dashboard bind address

# Challenge settings
challenges:
  js_challenge: true               # Enable JavaScript challenge
  rate_limit: true                 # Enable rate limiting
  honeypot: true                   # Enable honeypot links
  user_agent_check: true           # Enable User-Agent blacklist/whitelist
  captcha_fallback: false          # Enable math CAPTCHA fallback
  js_difficulty: 3                 # JS puzzle difficulty (1-10)
  challenge_timeout: 300           # Challenge expiration in seconds

# Rate limiting
rate_limit:
  enabled: true
  requests_per_minute: 60          # Max requests per minute per IP
  requests_per_hour: 1000          # Max requests per hour per IP
  burst_size: 10                   # Token bucket burst size

# IP reputation system
ip_reputation:
  enabled: true
  max_score: 100                   # Maximum reputation score
  block_threshold: 80              # Score threshold to block an IP
  decay_interval: 3600             # Score decay interval in seconds
  request_penalty: 5               # Score penalty per request (high volume)
  block_penalty: 20                # Score penalty for being blocked
  challenge_penalty: 10            # Score penalty for being challenged

# Whitelist - bots that should ALWAYS be allowed
whitelist:
  bots:
    - Googlebot                    # Google search crawler
    - Bingbot                      # Bing search crawler
    - Baiduspider                  # Baidu search crawler
    - YandexBot                    # Yandex search crawler
    - DuckDuckBot                  # DuckDuckGo search crawler
    - Slurp                        # Yahoo search crawler
  ips: []                          # Whitelisted IP addresses
  user_agents: []                  # Additional whitelisted UA patterns

# Blacklist - bots that should ALWAYS be blocked
blacklist:
  bots:
    - GPTBot                       # OpenAI GPT crawler
    - ChatGPT-User                 # OpenAI ChatGPT user agent
    - ClaudeBot                    # Anthropic Claude crawler
    - Claude-Web                   # Anthropic Claude web
    - Google-Extended              # Google AI training crawler
    - Bytespider                   # ByteDance crawler
    - FacebookBot                  # Meta/Facebook crawler
    - Applebot-Extended            # Apple AI training crawler
    - CCBot                        # Common Crawl crawler
    - anthropic-ai                 # Anthropic AI
    - PerplexityBot                # Perplexity AI crawler
    - Perplexity-User              # Perplexity AI user
    - cohere-ai                    # Cohere AI
    - ai2bot                       # Allen AI
    - Amazonbot                    # Amazon crawler
    - OMGILI                       # Webhose crawler
    - facebookexternalhit          # Facebook link preview
    - ImagesiftBot                 # ImageSift crawler
    - Timpibot                     # Timpi search crawler
  ips: []                          # Blacklisted IP addresses
  user_agents: []                  # Additional blacklisted UA patterns

# Logging settings
logging:
  level: "INFO"                    # Log level: DEBUG, INFO, WARN, ERROR
  log_file: null                   # Log file path (null = no file logging)
  log_dir: "./logs"                # Directory for log files
  access_log: true                 # Enable access logging
  max_log_size_mb: 100             # Maximum log file size in MB
  log_rotation: true               # Enable daily log rotation
  color_output: true               # Enable colored terminal output

# Security settings
security:
  secret_key: "change-me-to-a-random-secret-string"  # Secret for token generation
  allowed_methods:                 # Allowed HTTP methods
    - GET
    - POST
    - HEAD
    - OPTIONS
  max_request_size_mb: 10          # Maximum request body size
  block_missing_referer: false     # Block requests without Referer header
  block_empty_accept: false        # Block requests without Accept header