dhaneshbb.github.io/robots.txt at main · dhaneshbb/dhaneshbb.github.io · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
# robots.txt for Dhanesh B.B. Portfolio Website
# Professional Data Science Portfolio

# Global directives
User-agent: *
Allow: /

# Allow all major search engines
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: YandexBot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: LinkedInBot
Allow: /

# Disallow specific directories
Disallow: /scripts/
Disallow: /styles/
Disallow: /components/
Disallow: /.git/
Disallow: /node_modules/
Disallow: /.github/
Disallow: /.bkup/
Disallow: /.backup/
Disallow: /.scripts-build/

# Allow specific important files
Allow: /media/
Allow: /sitemap.xml
Allow: /robots.txt
Allow: /cert-viewer.html
Allow: /resume-viewer.html

# Block specific file types
Disallow: /*.json$
Disallow: /*.log$
Disallow: /*.tmp$
Disallow: /*.backup$
Disallow: /*.md$

# Block development files
Disallow: /package.json
Disallow: /package-lock.json
Disallow: /.gitignore
Disallow: /.editorconfig
Disallow: /.prettierrc
Disallow: /.eslintrc.json
Disallow: /.stylelintrc.json
Disallow: /jsconfig.json
Disallow: /LICENSE

# Block backup and documentation
Disallow: /DESIGN_IMPROVEMENTS.md
Disallow: /COPYRIGHT_*.md
Disallow: /DEPENDENCIES.md
Disallow: /TERMS.md

# Crawl delay (be respectful)
Crawl-delay: 1

# Sitemap location
Sitemap: https://dhaneshbb.github.io/sitemap.xml