Linux-Server-Training/robots.txt at main · SchoolOfFreelancing/Linux-Server-Training · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
# School of Freelancing - Robots.txt
# Search Engine & AI Crawler Directives

# Default rules for all search engines
User-agent: *
Allow: /
Allow: *.css$
Allow: *.js$
Allow: *.jpg$
Allow: *.jpeg$
Allow: *.png$
Allow: *.gif$
Allow: *.svg$
Allow: *.webp$

# Block private/admin areas
Disallow: /admin/
Disallow: /private/
Disallow: /cgi-bin/
Disallow: /.git/
Disallow: /.gitignore

# Optimize crawling parameters
Crawl-delay: 5
Request-rate: 10/1s

# Google specific
User-agent: Googlebot
Crawl-delay: 1

# Bing specific
User-agent: Bingbot
Crawl-delay: 2

# Yandex specific
User-agent: YandexBot
Crawl-delay: 2

##################################################
# AI search / answer engines & LLM crawlers
# This is a marketing/lead-gen site — we want
# maximum visibility in AI-generated answers and
# citations, so every major AI crawler is explicitly
# allowed with no crawl-delay. Revisit if that
# stance ever changes (e.g. to opt out of training
# use while still allowing live retrieval).
##################################################

# OpenAI — ChatGPT search/browsing and training crawlers
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: GPTBot
Allow: /

# Anthropic — Claude search/browsing and training crawlers
User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Google's AI features (separate from classic Googlebot above)
User-agent: Google-Extended
Allow: /

# Apple Intelligence
User-agent: Applebot-Extended
Allow: /

# Meta AI
User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

# ByteDance / TikTok
User-agent: Bytespider
Allow: /

# Amazon (Alexa+, Rufus)
User-agent: Amazonbot
Allow: /

# You.com
User-agent: YouBot
Allow: /

# Common Crawl (widely used as LLM pretraining data)
User-agent: CCBot
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# Diffbot
User-agent: Diffbot
Allow: /

# Sitemap location for all search engines and crawlers
Sitemap: https://www.schooloffreelancing.com/sitemap.xml

# LLM-oriented site index (see https://llmstxt.org)
# https://www.schooloffreelancing.com/llms.txt
# https://www.schooloffreelancing.com/llms-full.txt

# Additional crawling instructions
# Cache static assets for better performance
# Cache-Control: max-age=2592000

# Clean URL parameters (comment out if not applicable)
# Clean-param: utm_source remove
# Clean-param: utm_medium remove
# Clean-param: utm_campaign remove