Skip to content

Commit d62b9a7

Browse files
mythzmythz
committed
Block AI Bots
1 parent e7a1cc6 commit d62b9a7

3 files changed

Lines changed: 217 additions & 1 deletion

File tree

‎MyApp/Configure.UserAgentBlocking.cs‎

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Threading.Tasks;
4+
using Microsoft.AspNetCore.Http;
5+
using Microsoft.Extensions.Logging;
6+
using Microsoft.Extensions.Options;
7+
8+
namespace MyApp;
9+
10+
/// <summary>
11+
/// Options for configuring the UserAgentBlockingMiddleware
12+
/// </summary>
13+
public class UserAgentBlockingOptions
14+
{
15+
/// <summary>
16+
/// List of user agents to block (supports exact matches or substring matches)
17+
/// </summary>
18+
public List<string> BlockedUserAgents { get; set; } = new();
19+
20+
/// <summary>
21+
/// HTTP status code to return when a user agent is blocked (defaults to 403 Forbidden)
22+
/// </summary>
23+
public int BlockedStatusCode { get; set; } = StatusCodes.Status403Forbidden;
24+
25+
/// <summary>
26+
/// Optional message to return in the response body when a user agent is blocked
27+
/// </summary>
28+
public string BlockedMessage { get; set; } = "Access denied based on your user agent";
29+
30+
/// <summary>
31+
/// If true, will perform case-insensitive matching (defaults to true)
32+
/// </summary>
33+
public bool IgnoreCase { get; set; } = true;
34+
35+
/// <summary>
36+
/// If true, blocked requests will be logged (defaults to true)
37+
/// </summary>
38+
public bool LogBlockedRequests { get; set; } = true;
39+
}
40+
41+
/// <summary>
42+
/// Middleware that blocks requests from specific user agents
43+
/// </summary>
44+
public class UserAgentBlockingMiddleware
45+
{
46+
private readonly RequestDelegate _next;
47+
private readonly UserAgentBlockingOptions _options;
48+
private readonly ILogger<UserAgentBlockingMiddleware> _logger;
49+
50+
public UserAgentBlockingMiddleware(
51+
RequestDelegate next,
52+
IOptions<UserAgentBlockingOptions> options,
53+
ILogger<UserAgentBlockingMiddleware> logger)
54+
{
55+
_next = next ?? throw new ArgumentNullException(nameof(next));
56+
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
57+
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
58+
}
59+
60+
public async Task InvokeAsync(HttpContext context)
61+
{
62+
if (context == null)
63+
{
64+
throw new ArgumentNullException(nameof(context));
65+
}
66+
67+
// Get the User-Agent header
68+
string userAgent = context.Request.Headers["User-Agent"].ToString();
69+
70+
// Check if the user agent should be blocked
71+
if (ShouldBlockUserAgent(userAgent))
72+
{
73+
// Log the blocked request if enabled
74+
if (_options.LogBlockedRequests)
75+
{
76+
_logger.LogInformation(
77+
"Request blocked from user agent: {UserAgent}, IP: {IPAddress}, Path: {Path}",
78+
userAgent,
79+
context.Connection.RemoteIpAddress,
80+
context.Request.Path);
81+
}
82+
83+
// Set the response status code
84+
context.Response.StatusCode = _options.BlockedStatusCode;
85+
context.Response.ContentType = "text/plain";
86+
87+
// Write the blocked message to the response
88+
await context.Response.WriteAsync(_options.BlockedMessage);
89+
return;
90+
}
91+
92+
// If not blocked, continue to the next middleware
93+
await _next(context);
94+
}
95+
96+
private bool ShouldBlockUserAgent(string userAgent)
97+
{
98+
if (string.IsNullOrEmpty(userAgent))
99+
{
100+
// You might want to block requests with no user agent
101+
// Return true here if you want to block empty user agents
102+
return false;
103+
}
104+
105+
foreach (var blockedAgent in _options.BlockedUserAgents)
106+
{
107+
var comparison = _options.IgnoreCase
108+
? StringComparison.OrdinalIgnoreCase
109+
: StringComparison.Ordinal;
110+
111+
if (userAgent.Contains(blockedAgent, comparison))
112+
return true;
113+
if (blockedAgent.Contains(' ') && userAgent.Contains(blockedAgent.Replace(" ",""), comparison))
114+
return true;
115+
}
116+
117+
return false;
118+
}
119+
}
120+
121+
/// <summary>
122+
/// Extension methods for registering the UserAgentBlockingMiddleware
123+
/// </summary>
124+
public static class UserAgentBlockingMiddlewareExtensions
125+
{
126+
/// <summary>
127+
/// Adds the UserAgentBlockingMiddleware to the application pipeline
128+
/// </summary>
129+
public static IApplicationBuilder UseUserAgentBlocking(
130+
this IApplicationBuilder builder)
131+
{
132+
return builder.UseMiddleware<UserAgentBlockingMiddleware>();
133+
}
134+
135+
/// <summary>
136+
/// Adds the UserAgentBlockingMiddleware to the application pipeline with custom options
137+
/// </summary>
138+
public static IApplicationBuilder UseUserAgentBlocking(
139+
this IApplicationBuilder builder,
140+
Action<UserAgentBlockingOptions> configureOptions)
141+
{
142+
// Create a new options instance
143+
var options = new UserAgentBlockingOptions();
144+
145+
// Apply the configuration
146+
configureOptions(options);
147+
148+
// Use the middleware with the configured options
149+
return builder.UseMiddleware<UserAgentBlockingMiddleware>(Options.Create(options));
150+
}
151+
}

‎MyApp/Program.cs‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,33 @@
6262
services.AddBlazorServerIdentityApiClient(baseUrl);
6363
services.AddLocalStorage();
6464

65+
// Register the options in the dependency injection container
66+
services.Configure<UserAgentBlockingOptions>(options =>
67+
{
68+
// Add user agents to block
69+
options.BlockedUserAgents.AddRange([
70+
"bytespider",
71+
"gptbot",
72+
"gptbot",
73+
"claudebot",
74+
"amazonbot",
75+
"imagesiftbot",
76+
"semrushbot",
77+
"dotbot",
78+
"semrushbot",
79+
"dataforseobot",
80+
"WhatsApp Bot",
81+
"HeadlessChrome",
82+
"PetalBot",
83+
]);
84+
85+
// Optional: Customize the response status code
86+
// options.BlockedStatusCode = StatusCodes.Status429TooManyRequests;
87+
88+
// Optional: Customize the blocked message
89+
options.BlockedMessage = "This bot is not allowed to access our website";
90+
});
91+
6592
services.AddServiceStack(typeof(MyServices).Assembly);
6693

6794
services.AddOutputCache(options =>
@@ -84,6 +111,9 @@
84111
app.UseHsts();
85112
}
86113

114+
// Add the middleware early in the pipeline (before routing and endpoints)
115+
app.UseUserAgentBlocking();
116+
87117
//TODO: Remove after bing search no longer includes these links
88118
string[] redirectPosts = [
89119
"net8-blazor-template",

‎MyApp/wwwroot/robots.txt‎

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,41 @@
11
User-agent: Googlebot
2+
Allow: /
3+
User-agent: YandexBot
4+
Allow: /
5+
User-agent: Bingbot
6+
Allow: /
7+
User-agent: ahrefsbot
8+
Allow: /
9+
User-agent: Baidu Spider
10+
Allow: /
11+
User-agent: coccocbot
12+
Allow: /
13+
User-agent: yacybot
14+
Allow: /
15+
User-agent: oai-searchbot
16+
Allow: /
217

18+
User-agent: bytespider
19+
Disallow: /
20+
User-agent: gptbot
21+
Disallow: /
22+
User-agent: claudebot
23+
Disallow: /
24+
User-agent: amazonbot
25+
Disallow: /
26+
User-agent: imagesiftbot
27+
Disallow: /
28+
User-agent: mj12bot
29+
Disallow: /
30+
User-agent: semrushbot
31+
Disallow: /
32+
User-agent: dotbot
33+
Disallow: /
34+
User-agent: dataforseobot
35+
Disallow: /
36+
User-agent: WhatsApp Bot
37+
Disallow: /
338
User-agent: *
4-
Allow: /
39+
Disallow: /
540

641
Sitemap: https://pvq.app/sitemap.xml

0 commit comments

Comments
 (0)