Add more ranges to google's allowed bots

Author: joecorall

internal/helper/google.go

@@ -7,10 +7,19 @@ import (
 	"log/slog"
 	"net"
 	"net/http"
+	"net/netip"
+	"slices"
 	"sync"
 	"time"
 )
 
+var GoogleCrawlerIPRangeURLs = []string{
+	"https://developers.google.com/static/search/apis/ipranges/googlebot.json",
+	"https://developers.google.com/static/crawling/ipranges/common-crawlers.json",
+	"https://developers.google.com/static/crawling/ipranges/special-crawlers.json",
+	"https://developers.google.com/static/crawling/ipranges/user-triggered-fetchers-google.json",
+}
+
 // GooglebotIPs holds the list of Googlebot IP ranges, providing a thread-safe way to check if an IP is a Googlebot.
 type GooglebotIPs struct {
 	cidrs []*net.IPNet
@@ -108,3 +117,76 @@ func FetchGooglebotIPs(log *slog.Logger, httpClient *http.Client, url string) ([
 
 	return cidrs, nil
 }
+
+// FetchGoogleCrawlerIPs fetches crawler IP ranges from multiple Google-managed endpoints,
+// then returns a canonical, unique list where broader prefixes replace narrower prefixes.
+func FetchGoogleCrawlerIPs(log *slog.Logger, httpClient *http.Client, urls []string) ([]string, error) {
+	if len(urls) == 0 {
+		return nil, nil
+	}
+
+	allCIDRs := make([]string, 0)
+	for _, url := range urls {
+		cidrs, err := FetchGooglebotIPs(log, httpClient, url)
+		if err != nil {
+			return nil, err
+		}
+		allCIDRs = append(allCIDRs, cidrs...)
+	}
+
+	return ReduceCIDRs(allCIDRs, log), nil
+}
+
+// ReduceCIDRs canonicalizes CIDRs, removes exact duplicates, and removes narrower
+// ranges when they are fully covered by broader ranges.
+func ReduceCIDRs(cidrs []string, log *slog.Logger) []string {
+	prefixes := make([]netip.Prefix, 0, len(cidrs))
+	for _, cidr := range cidrs {
+		prefix, err := netip.ParsePrefix(cidr)
+		if err != nil {
+			if log != nil {
+				log.Error("error parsing CIDR", "cidr", cidr, "err", err)
+			}
+			continue
+		}
+		prefixes = append(prefixes, prefix.Masked())
+	}
+
+	slices.SortFunc(prefixes, func(a, b netip.Prefix) int {
+		aIs4 := a.Addr().Is4()
+		bIs4 := b.Addr().Is4()
+		if aIs4 && !bIs4 {
+			return -1
+		}
+		if !aIs4 && bIs4 {
+			return 1
+		}
+
+		if a.Bits() != b.Bits() {
+			return a.Bits() - b.Bits()
+		}
+
+		return a.Addr().Compare(b.Addr())
+	})
+
+	reduced := make([]netip.Prefix, 0, len(prefixes))
+	for _, candidate := range prefixes {
+		covered := false
+		for _, existing := range reduced {
+			if existing.Bits() <= candidate.Bits() && existing.Contains(candidate.Addr()) {
+				covered = true
+				break
+			}
+		}
+		if !covered {
+			reduced = append(reduced, candidate)
+		}
+	}
+
+	result := make([]string, 0, len(reduced))
+	for _, prefix := range reduced {
+		result = append(result, prefix.String())
+	}
+
+	return result
+}

internal/helper/google_test.go

@@ -6,6 +6,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"reflect"
 	"testing"
 )
 
@@ -77,3 +78,49 @@ func TestFetchGooglebotIPs(t *testing.T) {
 		}
 	}
 }
+
+func TestReduceCIDRs(t *testing.T) {
+	log := slog.New(slog.NewTextHandler(os.Stdout, nil))
+
+	input := []string{
+		"8.8.8.0/24",
+		"8.8.8.0/25",
+		"8.8.8.128/25",
+		"8.8.8.0/24", // duplicate
+		"2001:4860::/32",
+		"2001:4860:1234::/48",
+	}
+
+	got := ReduceCIDRs(input, log)
+	want := []string{"8.8.8.0/24", "2001:4860::/32"}
+
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("unexpected reduced CIDRs: got %v want %v", got, want)
+	}
+}
+
+func TestFetchGoogleCrawlerIPs(t *testing.T) {
+	log := slog.New(slog.NewTextHandler(os.Stdout, nil))
+
+	serverA := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(`{"prefixes":[{"ipv4Prefix":"8.8.8.0/24"}]}`))
+	}))
+	defer serverA.Close()
+
+	serverB := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		_, _ = w.Write([]byte(`{"prefixes":[{"ipv4Prefix":"8.8.8.0/25"},{"ipv6Prefix":"2001:4860::/32"}]}`))
+	}))
+	defer serverB.Close()
+
+	got, err := FetchGoogleCrawlerIPs(log, serverA.Client(), []string{serverA.URL, serverB.URL})
+	if err != nil {
+		t.Fatalf("FetchGoogleCrawlerIPs failed: %v", err)
+	}
+
+	want := []string{"8.8.8.0/24", "2001:4860::/32"}
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("unexpected CIDRs: got %v want %v", got, want)
+	}
+}

main.go

@@ -351,7 +351,7 @@ func (bc *CaptchaProtect) googlebotIPCheckLoop(ctx context.Context) {
 	defer ticker.Stop()
 
 	// Initial fetch
-	cidrs, err := helper.FetchGooglebotIPs(bc.log, bc.httpClient, "https://developers.google.com/static/search/apis/ipranges/googlebot.json")
+	cidrs, err := helper.FetchGoogleCrawlerIPs(bc.log, bc.httpClient, helper.GoogleCrawlerIPRangeURLs)
 	if err != nil {
 		bc.log.Error("failed to fetch googlebot ips", "err", err)
 	} else {
@@ -362,7 +362,7 @@ func (bc *CaptchaProtect) googlebotIPCheckLoop(ctx context.Context) {
 	for {
 		select {
 		case <-ticker.C:
-			cidrs, err := helper.FetchGooglebotIPs(bc.log, bc.httpClient, "https://developers.google.com/static/search/apis/ipranges/googlebot.json")
+			cidrs, err := helper.FetchGoogleCrawlerIPs(bc.log, bc.httpClient, helper.GoogleCrawlerIPRangeURLs)
 			if err != nil {
 				bc.log.Error("failed to fetch googlebot ips", "err", err)
 				continue