feat: add per-rule and global Cloudflare routing, 429 retries

HTTP retriever stays the default. Cloudflare is now opt-in at two levels:

- per-rule: new Rule.UseCloudflare field (checkbox in rule editor UI)
  routes requests for that domain through Cloudflare Browser Rendering
- global: --cf-route-all / CF_ROUTE_ALL flag (default false) routes every
  request through Cloudflare

UReadability.pickRetriever(rule) picks: CFRouteAll > rule.UseCloudflare >
default HTTP. extractWithRules now resolves the rule once upfront and
shares it between routing and getContent (was looked up twice).

CloudflareRetriever retries on HTTP 429 with exponential backoff (base 11s,
max 2 retries by default → worst-case 33s of backoff), honours Retry-After
header, and aborts immediately on caller context cancel. MaxRetries=-1
disables retries.

Added WriteTimeout=150s on the HTTP server — was previously unset, allowing
handlers to run forever. 150s covers the worst-case CF path (up to ~123s).

Author: paskal

datastore/rules.go

@@ -18,16 +18,17 @@ type RulesDAO struct {
 
 // Rule record, entry in mongo
 type Rule struct {
-	ID        bson.ObjectID `json:"id" bson:"_id,omitempty"`
-	Domain    string        `json:"domain"`
-	MatchURLs []string      `json:"match_url,omitempty" bson:"match_urls,omitempty"`
-	Content   string        `json:"content"`
-	Author    string        `json:"author,omitempty" bson:"author,omitempty"`
-	TS        string        `json:"ts,omitempty" bson:"ts,omitempty"` // ts of original article
-	Excludes  []string      `json:"excludes,omitempty" bson:"excludes,omitempty"`
-	TestURLs  []string      `json:"test_urls,omitempty" bson:"test_urls"`
-	User      string        `json:"user"`
-	Enabled   bool          `json:"enabled"`
+	ID            bson.ObjectID `json:"id" bson:"_id,omitempty"`
+	Domain        string        `json:"domain"`
+	MatchURLs     []string      `json:"match_url,omitempty" bson:"match_urls,omitempty"`
+	Content       string        `json:"content"`
+	Author        string        `json:"author,omitempty" bson:"author,omitempty"`
+	TS            string        `json:"ts,omitempty" bson:"ts,omitempty"` // ts of original article
+	Excludes      []string      `json:"excludes,omitempty" bson:"excludes,omitempty"`
+	TestURLs      []string      `json:"test_urls,omitempty" bson:"test_urls"`
+	User          string        `json:"user"`
+	Enabled       bool          `json:"enabled"`
+	UseCloudflare bool          `json:"use_cloudflare,omitempty" bson:"use_cloudflare,omitempty"` // route fetch via Cloudflare Browser Rendering
 }
 
 // Get rule by url. Checks if found in mongo, matching by domain

extractor/readability.go

@@ -34,13 +34,15 @@ type UReadability struct {
 	TimeOut     time.Duration
 	SnippetSize int
 	Rules       Rules
-	Retriever   Retriever
+	Retriever   Retriever // default retriever; when nil a cached HTTPRetriever is used
+	CFRetriever Retriever // optional Cloudflare Browser Rendering retriever; when set, enables routing
+	CFRouteAll  bool      // route every request through CFRetriever (requires CFRetriever != nil)
 
 	defaultRetrieverOnce sync.Once
 	defaultRetriever     Retriever
 }
 
-// retriever returns the configured Retriever, defaulting to a cached HTTPRetriever if nil
+// retriever returns the configured default Retriever, creating a cached HTTPRetriever if nil
 func (f *UReadability) retriever() Retriever {
 	if f.Retriever != nil {
 		return f.Retriever
@@ -51,6 +53,22 @@ func (f *UReadability) retriever() Retriever {
 	return f.defaultRetriever
 }
 
+// pickRetriever decides which retriever should fetch the given URL based on routing config and an
+// optional pre-resolved rule. Falls back to the default retriever unless CFRetriever is set AND
+// either CFRouteAll is true or the rule explicitly asks for Cloudflare.
+func (f *UReadability) pickRetriever(rule *datastore.Rule) Retriever {
+	if f.CFRetriever == nil {
+		return f.retriever()
+	}
+	if f.CFRouteAll {
+		return f.CFRetriever
+	}
+	if rule != nil && rule.UseCloudflare {
+		return f.CFRetriever
+	}
+	return f.retriever()
+}
+
 // Response from api calls
 type Response struct {
 	Content     string   `json:"content"`
@@ -91,7 +109,15 @@ func (f *UReadability) extractWithRules(ctx context.Context, reqURL string, rule
 	log.Printf("[INFO] extract %s", reqURL)
 	rb := &Response{}
 
-	result, err := f.retriever().Retrieve(ctx, reqURL)
+	// look up a rule by domain once up front (unless one was explicitly passed) so retriever
+	// selection and getContent share the same lookup instead of paying for two round-trips.
+	if rule == nil && f.Rules != nil {
+		if r, found := f.Rules.Get(ctx, reqURL); found {
+			rule = &r
+		}
+	}
+
+	result, err := f.pickRetriever(rule).Retrieve(ctx, reqURL)
 	if err != nil {
 		return nil, err
 	}
@@ -136,10 +162,10 @@ func (f *UReadability) extractWithRules(ctx context.Context, reqURL string, rule
 	return rb, nil
 }
 
-// getContent retrieves content from raw body string, both content (text only) and rich (with html tags)
-// if rule is provided, it uses custom rule, otherwise tries to retrieve one from the storage,
-// and at last tries to use general readability parser
-func (f *UReadability) getContent(ctx context.Context, body, reqURL string, rule *datastore.Rule) (content, rich string, err error) {
+// getContent retrieves content from raw body string, both content (text only) and rich (with html tags).
+// if rule is provided, it tries the custom rule first and falls back to the general parser on failure.
+// rule lookup for a given URL is done upstream in extractWithRules.
+func (f *UReadability) getContent(_ context.Context, body, reqURL string, rule *datastore.Rule) (content, rich string, err error) {
 	// general parser
 	genParser := func(body, _ string) (content, rich string, err error) {
 		doc, err := readability.NewDocument(body)
@@ -172,19 +198,10 @@ func (f *UReadability) getContent(ctx context.Context, body, reqURL string, rule
 
 	if rule != nil {
 		log.Printf("[DEBUG] custom rule provided for %s: %v", reqURL, rule)
-		return customParser(body, reqURL, *rule)
-	}
-
-	if f.Rules != nil {
-		r := f.Rules
-		if rule, found := r.Get(ctx, reqURL); found {
-			if content, rich, err = customParser(body, reqURL, rule); err == nil {
-				return content, rich, nil
-			}
-			log.Printf("[WARN] custom extractor failed for %s, error=%v", reqURL, err) // back to general parser
+		if content, rich, err = customParser(body, reqURL, *rule); err == nil {
+			return content, rich, nil
 		}
-	} else {
-		log.Print("[DEBUG] no rules defined!")
+		log.Printf("[WARN] custom extractor failed for %s, error=%v", reqURL, err) // back to general parser
 	}
 
 	return genParser(body, reqURL)

extractor/readability_test.go

@@ -257,6 +257,92 @@ func TestExtractWithCustomRetriever(t *testing.T) {
 	assert.Equal(t, "https://example.com/test-page", calls[0].URL)
 }
 
+func TestPickRetriever(t *testing.T) {
+	mkRetriever := func(tag string) *RetrieverMock {
+		return &RetrieverMock{
+			RetrieveFunc: func(_ context.Context, reqURL string) (*RetrieveResult, error) {
+				h := make(http.Header)
+				h.Set("Content-Type", "text/html; charset=utf-8")
+				return &RetrieveResult{
+					Body:   []byte("<html><head><title>" + tag + "</title></head><body><p>body-" + tag + "</p></body></html>"),
+					URL:    reqURL,
+					Header: h,
+				}, nil
+			},
+		}
+	}
+
+	tests := []struct {
+		name          string
+		cfRouteAll    bool
+		useCloudflare bool
+		cfConfigured  bool
+		wantTag       string
+	}{
+		{name: "no CF configured uses HTTP", cfConfigured: false, wantTag: "http"},
+		{name: "CF configured, no flag uses HTTP", cfConfigured: true, wantTag: "http"},
+		{name: "CF configured, rule asks for CF uses CF", cfConfigured: true, useCloudflare: true, wantTag: "cf"},
+		{name: "CF configured, route-all uses CF", cfConfigured: true, cfRouteAll: true, wantTag: "cf"},
+		{name: "route-all overrides rule flag", cfConfigured: true, cfRouteAll: true, useCloudflare: false, wantTag: "cf"},
+		{name: "route-all without CF configured falls back to HTTP", cfConfigured: false, cfRouteAll: true, wantTag: "http"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			httpR := mkRetriever("http")
+			var cfR *RetrieverMock
+			lr := UReadability{
+				TimeOut:     time.Second,
+				SnippetSize: 200,
+				Retriever:   httpR,
+				CFRouteAll:  tt.cfRouteAll,
+				Rules: &mocks.RulesMock{
+					GetFunc: func(_ context.Context, _ string) (datastore.Rule, bool) {
+						return datastore.Rule{Domain: "example.com", UseCloudflare: tt.useCloudflare}, true
+					},
+				},
+			}
+			if tt.cfConfigured {
+				cfR = mkRetriever("cf")
+				lr.CFRetriever = cfR
+			}
+
+			_, err := lr.Extract(context.Background(), "https://example.com/page")
+			require.NoError(t, err)
+
+			switch tt.wantTag {
+			case "http":
+				assert.Len(t, httpR.RetrieveCalls(), 1, "http retriever should have been called")
+				if cfR != nil {
+					assert.Empty(t, cfR.RetrieveCalls(), "cf retriever should not have been called")
+				}
+			case "cf":
+				require.NotNil(t, cfR)
+				assert.Len(t, cfR.RetrieveCalls(), 1, "cf retriever should have been called")
+				assert.Empty(t, httpR.RetrieveCalls(), "http retriever should not have been called")
+			}
+		})
+	}
+}
+
+func TestPickRetrieverNoRules(t *testing.T) {
+	httpR := &RetrieverMock{RetrieveFunc: func(_ context.Context, reqURL string) (*RetrieveResult, error) {
+		h := make(http.Header)
+		h.Set("Content-Type", "text/html; charset=utf-8")
+		return &RetrieveResult{Body: []byte("<html><head><title>t</title></head><body>x</body></html>"), URL: reqURL, Header: h}, nil
+	}}
+	cfR := &RetrieverMock{RetrieveFunc: func(_ context.Context, reqURL string) (*RetrieveResult, error) {
+		h := make(http.Header)
+		h.Set("Content-Type", "text/html; charset=utf-8")
+		return &RetrieveResult{Body: []byte("<html><head><title>t</title></head><body>x</body></html>"), URL: reqURL, Header: h}, nil
+	}}
+	lr := UReadability{TimeOut: time.Second, SnippetSize: 200, Retriever: httpR, CFRetriever: cfR} // no Rules
+	_, err := lr.Extract(context.Background(), "https://example.com/page")
+	require.NoError(t, err)
+	assert.Len(t, httpR.RetrieveCalls(), 1, "no rules → HTTP path")
+	assert.Empty(t, cfR.RetrieveCalls())
+}
+
 func TestGetContentCustom(t *testing.T) {
 	rulesMock := &mocks.RulesMock{
 		GetFunc: func(_ context.Context, _ string) (datastore.Rule, bool) {

extractor/retriever.go

@@ -4,9 +4,11 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
+	"strconv"
 	"sync"
 	"time"
 
@@ -84,15 +86,27 @@ const (
 	cfDefaultBaseURL   = "https://api.cloudflare.com/client/v4"
 	cfDefaultWaitUntil = "networkidle0"
 	cfDefaultTimeout   = 60 * time.Second
+	// cfDefaultMaxRetries=2 keeps worst-case backoff at ~33s (11s + 22s) so total handler time
+	// stays under common upstream timeouts (nginx proxy_read_timeout default is 60s).
+	cfDefaultMaxRetries = 2
+	cfDefaultRetryDelay = 11 * time.Second // free tier: 1 req / 10s — add a little headroom
+	cfMaxRetryDelay     = 30 * time.Second
 )
 
+// errCFRateLimited is returned by the single-attempt inner retrieve when the CF API signals rate limiting;
+// the outer Retrieve uses it to decide whether to back off and retry.
+var errCFRateLimited = errors.New("cloudflare rate limited")
+
 // CloudflareRetriever fetches pages using Cloudflare Browser Rendering API.
 // it sends a POST to the /content endpoint which returns fully rendered HTML after JS execution.
+// on HTTP 429 it retries with backoff (respecting Retry-After) up to MaxRetries times.
 type CloudflareRetriever struct {
-	AccountID string
-	APIToken  string
-	BaseURL   string        // override for testing; defaults to Cloudflare API
-	Timeout   time.Duration // defaults to 60s (browser rendering can be slow)
+	AccountID  string
+	APIToken   string
+	BaseURL    string        // override for testing; defaults to Cloudflare API
+	Timeout    time.Duration // per-request HTTP client timeout; defaults to 60s
+	MaxRetries int           // number of retries on 429; defaults to 3. set to -1 to disable retries
+	RetryDelay time.Duration // base delay between 429 retries; defaults to 11s (CF free tier is 1 req/10s)
 
 	once   sync.Once
 	client *http.Client
@@ -123,8 +137,55 @@ type cfResponse struct {
 	Result  string `json:"result"`
 }
 
-// Retrieve fetches the URL via Cloudflare Browser Rendering /content endpoint
+// Retrieve fetches the URL via Cloudflare Browser Rendering /content endpoint.
+// on HTTP 429 it backs off and retries up to MaxRetries times, holding the caller's
+// connection open in the meantime. aborts early if the caller's context is canceled.
 func (c *CloudflareRetriever) Retrieve(ctx context.Context, reqURL string) (*RetrieveResult, error) {
+	maxRetries := c.MaxRetries
+	if maxRetries == 0 {
+		maxRetries = cfDefaultMaxRetries
+	}
+	if maxRetries < 0 {
+		maxRetries = 0
+	}
+	baseDelay := c.RetryDelay
+	if baseDelay <= 0 {
+		baseDelay = cfDefaultRetryDelay
+	}
+
+	var lastErr error
+	for attempt := 0; attempt <= maxRetries; attempt++ {
+		result, retryAfter, err := c.doRetrieve(ctx, reqURL)
+		if err == nil {
+			return result, nil
+		}
+		lastErr = err
+		if !errors.Is(err, errCFRateLimited) {
+			return nil, err
+		}
+		if attempt == maxRetries {
+			break
+		}
+		delay := retryAfter
+		if delay <= 0 {
+			delay = baseDelay << attempt // 11s, 22s, 44s, ...
+		}
+		if delay > cfMaxRetryDelay {
+			delay = cfMaxRetryDelay
+		}
+		log.Printf("[INFO] cloudflare rate limited for %s, retry %d/%d after %s", reqURL, attempt+1, maxRetries, delay)
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		case <-time.After(delay):
+		}
+	}
+	return nil, lastErr
+}
+
+// doRetrieve performs a single Browser Rendering request. on 429 it returns errCFRateLimited
+// (possibly wrapped) and the parsed Retry-After duration (0 if absent or unparseable).
+func (c *CloudflareRetriever) doRetrieve(ctx context.Context, reqURL string) (*RetrieveResult, time.Duration, error) {
 	baseURL := c.BaseURL
 	if baseURL == "" {
 		baseURL = cfDefaultBaseURL
@@ -137,20 +198,20 @@ func (c *CloudflareRetriever) Retrieve(ctx context.Context, reqURL string) (*Ret
 	}
 	reqBody, err := json.Marshal(cfReq)
 	if err != nil {
-		return nil, fmt.Errorf("marshal cf request: %w", err)
+		return nil, 0, fmt.Errorf("marshal cf request: %w", err)
 	}
 
 	httpReq, err := http.NewRequestWithContext(ctx, "POST", endpoint, bytes.NewReader(reqBody))
 	if err != nil {
-		return nil, fmt.Errorf("create cf request: %w", err)
+		return nil, 0, fmt.Errorf("create cf request: %w", err)
 	}
 	httpReq.Header.Set("Authorization", "Bearer "+c.APIToken)
 	httpReq.Header.Set("Content-Type", "application/json")
 
 	resp, err := c.httpClient().Do(httpReq)
 	if err != nil {
 		log.Printf("[WARN] cloudflare request failed for %s, error=%v", reqURL, err)
-		return nil, err
+		return nil, 0, err
 	}
 	defer func() {
 		if closeErr := resp.Body.Close(); closeErr != nil {
@@ -161,15 +222,19 @@ func (c *CloudflareRetriever) Retrieve(ctx context.Context, reqURL string) (*Ret
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
 		log.Printf("[WARN] failed to read cf response for %s, error=%v", reqURL, err)
-		return nil, err
+		return nil, 0, err
 	}
 
+	if resp.StatusCode == http.StatusTooManyRequests {
+		retryAfter := parseRetryAfter(resp.Header.Get("Retry-After"))
+		return nil, retryAfter, fmt.Errorf("%w: status 429", errCFRateLimited)
+	}
 	if resp.StatusCode != http.StatusOK {
 		bodySnippet := body
 		if len(bodySnippet) > 512 {
 			bodySnippet = bodySnippet[:512]
 		}
-		return nil, fmt.Errorf("cloudflare API error: status %d, body: %s", resp.StatusCode, string(bodySnippet))
+		return nil, 0, fmt.Errorf("cloudflare API error: status %d, body: %s", resp.StatusCode, string(bodySnippet))
 	}
 
 	// try JSON response format first: {"success": true, "result": "<html>"}
@@ -179,9 +244,9 @@ func (c *CloudflareRetriever) Retrieve(ctx context.Context, reqURL string) (*Ret
 		case cfResp.Success && cfResp.Result != "":
 			body = []byte(cfResp.Result)
 		case cfResp.Success && cfResp.Result == "":
-			return nil, fmt.Errorf("cloudflare returned empty content for %s", reqURL)
+			return nil, 0, fmt.Errorf("cloudflare returned empty content for %s", reqURL)
 		default: // !cfResp.Success
-			return nil, fmt.Errorf("cloudflare API returned success=false for %s", reqURL)
+			return nil, 0, fmt.Errorf("cloudflare API returned success=false for %s", reqURL)
 		}
 	}
 	// if unmarshal fails, use the raw body as-is (raw HTML response)
@@ -195,5 +260,22 @@ func (c *CloudflareRetriever) Retrieve(ctx context.Context, reqURL string) (*Ret
 		Body:   body,
 		URL:    reqURL,
 		Header: header,
-	}, nil
+	}, 0, nil
+}
+
+// parseRetryAfter parses an HTTP Retry-After header value as either delta-seconds
+// or an HTTP date. returns 0 if empty or unparseable.
+func parseRetryAfter(value string) time.Duration {
+	if value == "" {
+		return 0
+	}
+	if secs, err := strconv.Atoi(value); err == nil && secs >= 0 {
+		return time.Duration(secs) * time.Second
+	}
+	if t, err := http.ParseTime(value); err == nil {
+		if delta := time.Until(t); delta > 0 {
+			return delta
+		}
+	}
+	return 0
 }