fix: add SSRF protection, SARIF output, per-check timeouts, and crawler hardening

- Add SSRF protection (scheme validation + private IP blocking)
- Add SARIF 2.1.0 report format output
- Add per-check timeout (30s default)
- Add redirect loop detection and auth-required categorization
- Fix HTML parse error handling (return errors instead of nil)
- Fix rate limiter context cancellation
- Cache compiled regexes in rule adapter constructor
- Add Allow precedence in robots.txt, Crawl-Delay parsing
- Add WithBlockPrivateIPs() option (private IPs allowed by default)

Author: Patel230

cmd/inspect-ci/main.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/GrayCodeAI/inspect"
+	reportpkg "github.com/GrayCodeAI/inspect/internal/report"
 )
 
 func main() {
@@ -32,7 +33,7 @@ func main() {
 	flag.IntVar(&depth, "depth", 5, "Maximum crawl depth")
 	flag.StringVar(&failOn, "fail-on", "high", "Minimum severity to fail")
 	flag.IntVar(&concurrency, "concurrency", 10, "Concurrent workers")
-	flag.StringVar(&format, "format", "terminal", "Output format: terminal, json, junit")
+	flag.StringVar(&format, "format", "terminal", "Output format: terminal, json, junit, sarif")
 	flag.StringVar(&timeout, "timeout", "5m", "Scan timeout")
 	flag.StringVar(&outputFile, "output-file", "", "Write report to file")
 	flag.Parse()
@@ -71,6 +72,14 @@ func main() {
 	case "json":
 		data, _ := json.MarshalIndent(report, "", "  ")
 		output = string(data)
+	case "sarif":
+		rd := toReportData(report)
+		sarif, sErr := reportpkg.FormatSARIF(rd)
+		if sErr != nil {
+			fmt.Fprintf(os.Stderr, "error: sarif format: %v\n", sErr)
+			os.Exit(1)
+		}
+		output = sarif
 	default:
 		output = formatTerminal(report)
 	}
@@ -104,6 +113,30 @@ func main() {
 	}
 }
 
+func toReportData(r *inspect.Report) reportpkg.ReportData {
+	var rd reportpkg.ReportData
+	rd.Target = r.Target
+	rd.CrawledURLs = r.CrawledURLs
+	rd.Duration = r.Duration
+	rd.Stats.BySeverity = make(map[string]int)
+	for sev, count := range r.Stats.BySeverity {
+		rd.Stats.BySeverity[sev.String()] = count
+	}
+	rd.Stats.ByCheck = r.Stats.ByCheck
+	for _, f := range r.Findings {
+		rd.Findings = append(rd.Findings, reportpkg.Finding{
+			Check:    f.Check,
+			Severity: reportpkg.Severity(f.Severity),
+			URL:      f.URL,
+			Element:  f.Element,
+			Message:  f.Message,
+			Fix:      f.Fix,
+			Evidence: f.Evidence,
+		})
+	}
+	return rd
+}
+
 func formatTerminal(r *inspect.Report) string {
 	var b strings.Builder
 	b.WriteString(fmt.Sprintf("Inspect: %s — %d pages, %d findings\n",

internal/crawler/crawler.go

@@ -30,20 +30,22 @@ type Config struct {
 	AuthHeader      string
 	AuthValue       string
 	CookieJar       http.CookieJar
+	AllowPrivateIPs bool // When true, skip SSRF protection for private IPs
 }
 
 // Page represents a single crawled page with its metadata.
 type Page struct {
-	URL        string
-	StatusCode int
-	Headers    http.Header
-	Body       []byte
-	Links      []Link
-	Forms      []Form
-	Depth      int
-	ParentURL  string
-	Duration   time.Duration
-	Error      error
+	URL          string
+	StatusCode   int
+	Headers      http.Header
+	Body         []byte
+	Links        []Link
+	Forms        []Form
+	Depth        int
+	ParentURL    string
+	Duration     time.Duration
+	Error        error
+	AuthRequired bool // true when server returned 401/403
 }
 
 // Link represents a hyperlink found on a page.
@@ -147,6 +149,10 @@ func (c *Crawler) Crawl(ctx context.Context, startURL string) ([]*Page, error) {
 
 	if c.cfg.RespectRobots {
 		c.robots.Fetch(ctx, c.client, origin)
+		// Apply Crawl-Delay from robots.txt if it's slower than current rate limit
+		if delay := c.robots.CrawlDelay(origin); delay > 0 && delay > c.limiter.interval {
+			c.limiter.interval = delay
+		}
 	}
 
 	// Seed with sitemap URLs if available
@@ -296,48 +302,181 @@ func (c *Crawler) fetch(ctx context.Context, targetURL string, depth int, parent
 }
 
 func (c *Crawler) doFetch(ctx context.Context, page *Page, targetURL string) error {
+	// SSRF protection: validate URL scheme and resolved IP
+	if err := c.validateURL(targetURL); err != nil {
+		page.Error = err
+		return err
+	}
+
 	pageCtx, cancel := context.WithTimeout(ctx, c.cfg.PageTimeout)
 	defer cancel()
 
-	req, err := http.NewRequestWithContext(pageCtx, http.MethodGet, targetURL, nil)
-	if err != nil {
-		page.Error = err
-		return err
+	// Manual redirect handling with loop detection
+	const maxRedirects = 10
+	visited := make(map[string]bool)
+	currentURL := targetURL
+
+	for redirectCount := 0; ; redirectCount++ {
+		if redirectCount > maxRedirects {
+			err := fmt.Errorf("too many redirects (max %d)", maxRedirects)
+			page.Error = err
+			return err
+		}
+		if visited[currentURL] {
+			err := fmt.Errorf("redirect loop detected at %s", currentURL)
+			page.Error = err
+			return err
+		}
+		visited[currentURL] = true
+
+		req, err := http.NewRequestWithContext(pageCtx, http.MethodGet, currentURL, nil)
+		if err != nil {
+			page.Error = err
+			return err
+		}
+
+		req.Header.Set("User-Agent", c.cfg.UserAgent)
+		if c.cfg.AuthHeader != "" {
+			req.Header.Set(c.cfg.AuthHeader, c.cfg.AuthValue)
+		}
+
+		// Use a client that does not follow redirects automatically
+		resp, err := c.noRedirectClient().Do(req)
+		if err != nil {
+			page.Error = err
+			return err
+		}
+
+		// Handle auth-required responses as findings rather than errors
+		if resp.StatusCode == 401 || resp.StatusCode == 403 {
+			resp.Body.Close()
+			page.StatusCode = resp.StatusCode
+			page.Headers = resp.Header
+			page.Error = nil
+			page.AuthRequired = true
+			return nil
+		}
+
+		// Handle redirects manually
+		if resp.StatusCode >= 300 && resp.StatusCode < 400 {
+			resp.Body.Close()
+			loc := resp.Header.Get("Location")
+			if loc == "" {
+				page.StatusCode = resp.StatusCode
+				page.Headers = resp.Header
+				return nil
+			}
+			resolved := resolveURL(currentURL, loc)
+			if resolved == "" {
+				page.StatusCode = resp.StatusCode
+				page.Headers = resp.Header
+				return nil
+			}
+			// Validate redirect target for SSRF
+			if err := c.validateURL(resolved); err != nil {
+				page.Error = err
+				return err
+			}
+			currentURL = resolved
+			continue
+		}
+
+		// Non-redirect response
+		page.StatusCode = resp.StatusCode
+		page.Headers = resp.Header
+		page.Error = nil
+
+		contentType := resp.Header.Get("Content-Type")
+		if !strings.Contains(contentType, "text/html") {
+			resp.Body.Close()
+			return nil
+		}
+
+		body, err := io.ReadAll(io.LimitReader(resp.Body, 10*1024*1024))
+		resp.Body.Close()
+		if err != nil {
+			page.Error = err
+			return err
+		}
+		page.Body = body
+
+		page.Links = extractLinks(currentURL, body)
+		page.Forms = extractForms(body)
+		return nil
 	}
+}
 
-	req.Header.Set("User-Agent", c.cfg.UserAgent)
-	if c.cfg.AuthHeader != "" {
-		req.Header.Set(c.cfg.AuthHeader, c.cfg.AuthValue)
+// noRedirectClient returns an HTTP client that does not follow redirects.
+func (c *Crawler) noRedirectClient() *http.Client {
+	return &http.Client{
+		Transport: c.client.Transport,
+		Timeout:   c.client.Timeout,
+		Jar:       c.client.Jar,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
 	}
+}
 
-	resp, err := c.client.Do(req)
+// validateURL checks the URL for SSRF risks: scheme must be http/https and
+// resolved IP must not be in private ranges (unless AllowPrivateIPs is set).
+func (c *Crawler) validateURL(rawURL string) error {
+	parsed, err := url.Parse(rawURL)
 	if err != nil {
-		page.Error = err
-		return err
+		return fmt.Errorf("invalid URL: %w", err)
 	}
-	defer resp.Body.Close()
-
-	page.StatusCode = resp.StatusCode
-	page.Headers = resp.Header
-	page.Error = nil
-
-	contentType := resp.Header.Get("Content-Type")
-	if !strings.Contains(contentType, "text/html") {
+	if parsed.Scheme != "http" && parsed.Scheme != "https" {
+		return fmt.Errorf("disallowed URL scheme %q (only http/https allowed)", parsed.Scheme)
+	}
+	if c.cfg.AllowPrivateIPs {
 		return nil
 	}
-
-	body, err := io.ReadAll(io.LimitReader(resp.Body, 10*1024*1024))
+	host := parsed.Hostname()
+	ips, err := net.LookupHost(host)
 	if err != nil {
-		page.Error = err
-		return err
+		// DNS resolution failure is not an SSRF issue; let the fetch handle it
+		return nil
+	}
+	for _, ipStr := range ips {
+		ip := net.ParseIP(ipStr)
+		if ip == nil {
+			continue
+		}
+		if isPrivateIP(ip) {
+			return fmt.Errorf("SSRF protection: resolved IP %s for host %q is in a private range", ipStr, host)
+		}
 	}
-	page.Body = body
-
-	page.Links = extractLinks(targetURL, body)
-	page.Forms = extractForms(body)
 	return nil
 }
 
+// isPrivateIP checks if an IP is in a private/loopback range.
+func isPrivateIP(ip net.IP) bool {
+	privateRanges := []struct {
+		network *net.IPNet
+	}{
+		{mustParseCIDR("10.0.0.0/8")},
+		{mustParseCIDR("172.16.0.0/12")},
+		{mustParseCIDR("192.168.0.0/16")},
+		{mustParseCIDR("127.0.0.0/8")},
+		{mustParseCIDR("::1/128")},
+		{mustParseCIDR("fc00::/7")},
+	}
+	for _, r := range privateRanges {
+		if r.network.Contains(ip) {
+			return true
+		}
+	}
+	return false
+}
+
+func mustParseCIDR(s string) *net.IPNet {
+	_, network, err := net.ParseCIDR(s)
+	if err != nil {
+		panic(err)
+	}
+	return network
+}
+
 func isRetryable(statusCode int) bool {
 	return statusCode == 429 || statusCode == 500 || statusCode == 502 ||
 		statusCode == 503 || statusCode == 504 || statusCode == 0

internal/crawler/crawler_test.go

@@ -31,6 +31,7 @@ func TestCrawl_Basic(t *testing.T) {
 		RateLimit:       100,
 		UserAgent:       "test-bot",
 		FollowRedirects: 3,
+		AllowPrivateIPs: true,
 	})
 
 	pages, err := c.Crawl(context.Background(), srv.URL)
@@ -65,12 +66,13 @@ func TestCrawl_DepthLimit(t *testing.T) {
 	defer srv.Close()
 
 	c := New(Config{
-		MaxDepth:    2,
-		Concurrency: 1,
-		Timeout:     10 * time.Second,
-		PageTimeout: 5 * time.Second,
-		RateLimit:   100,
-		UserAgent:   "test-bot",
+		MaxDepth:        2,
+		Concurrency:     1,
+		Timeout:         10 * time.Second,
+		PageTimeout:     5 * time.Second,
+		RateLimit:       100,
+		UserAgent:       "test-bot",
+		AllowPrivateIPs: true,
 	})
 
 	pages, err := c.Crawl(context.Background(), srv.URL)
@@ -99,7 +101,7 @@ func TestCrawl_ExternalLinksNotFollowed(t *testing.T) {
 	srv := httptest.NewServer(mux)
 	defer srv.Close()
 
-	c := New(Config{MaxDepth: 3, Concurrency: 1, Timeout: 10 * time.Second, PageTimeout: 5 * time.Second, RateLimit: 100, UserAgent: "test"})
+	c := New(Config{MaxDepth: 3, Concurrency: 1, Timeout: 10 * time.Second, PageTimeout: 5 * time.Second, RateLimit: 100, UserAgent: "test", AllowPrivateIPs: true})
 	pages, err := c.Crawl(context.Background(), srv.URL)
 	if err != nil {
 		t.Fatalf("Crawl failed: %v", err)
@@ -139,7 +141,7 @@ func TestCrawl_ContextCancellation(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
 	defer cancel()
 
-	c := New(Config{MaxDepth: 3, Concurrency: 2, Timeout: 10 * time.Second, PageTimeout: 5 * time.Second, RateLimit: 100, UserAgent: "test"})
+	c := New(Config{MaxDepth: 3, Concurrency: 2, Timeout: 10 * time.Second, PageTimeout: 5 * time.Second, RateLimit: 100, UserAgent: "test", AllowPrivateIPs: true})
 	_, err := c.Crawl(ctx, srv.URL)
 	// Should not hang — should return within timeout
 	if err != nil {
@@ -170,15 +172,16 @@ func TestCrawl_Retry(t *testing.T) {
 	defer srv.Close()
 
 	c := New(Config{
-		MaxDepth:      1,
-		Concurrency:   1,
-		Timeout:       10 * time.Second,
-		PageTimeout:   5 * time.Second,
-		RateLimit:     100,
-		RetryAttempts: 3,
-		RetryDelay:    10 * time.Millisecond,
-		UserAgent:     "test",
-		RespectRobots: true,
+		MaxDepth:        1,
+		Concurrency:     1,
+		Timeout:         10 * time.Second,
+		PageTimeout:     5 * time.Second,
+		RateLimit:       100,
+		RetryAttempts:   3,
+		RetryDelay:      10 * time.Millisecond,
+		UserAgent:       "test",
+		RespectRobots:   true,
+		AllowPrivateIPs: true,
 	})
 
 	pages, err := c.Crawl(context.Background(), srv.URL)