fix(http): send default User-Agent; harden flaky 06 http/ssrf e2e (#270)

The durable HTTP client (reqwest) previously sent no User-Agent header.
Some hosts reject anonymous requests: httpbingo.org (fly.io) returns HTTP
402 to any request without a User-Agent, which made the redirect test
assert 3xx but receive 402, and silently let the other httpbingo tests
"pass" on a 402 instead of a real success response.

Product fix:
- build_client() now sets a default User-Agent of "pg_durable/<version>".
  Nodes may still override it via an explicit User-Agent header.

Test hardening (06_http_and_ssrf.sql):
- Move the redirect test (Test 8) off httpbin.org (frequently unavailable;
  the dominant CI flake) onto httpbingo.org/status/302, inheriting the
  default 30s timeout.
- Remove the residual-PUBLIC-grant permission test: it only exercised stock
  PostgreSQL grant semantics (a PUBLIC grant survives an additive per-role
  grant) and asserted nothing pg_durable-specific. Renumber the superuser
  bypass test accordingly.

Harness fix (scripts/test-e2e-local.sh):
- The repeat-run re-grant path (grant_e2e_df_usage) used the default
  include_http => false, so on runs >= 2 (after the extension is dropped
  and recreated) df_e2e_user lost HTTP access and 06 failed at the first
  df.http() call. Pass include_http => true to mirror 00_setup_playground,
  fixing `./scripts/test-e2e-local.sh 06_http_and_ssrf N`.

Verified: 06_http_and_ssrf now passes 3/3 in repeat mode against a reused
database. A separate, pre-existing BGW cold-start "pending" timeout flake
on the first instance remains out of scope (tracked for a retry harness).

Author: pinodeca

scripts/test-e2e-local.sh

@@ -363,7 +363,7 @@ show_version_once() {
 
 grant_e2e_df_usage() {
     "$PSQL" -h localhost -p "$PG_PORT" -U "$PG_USER" -d "$PG_DB" \
-        -c "SELECT df.grant_usage('$E2E_USER');" >/dev/null 2>&1
+        -c "SELECT df.grant_usage('$E2E_USER', include_http => true);" >/dev/null 2>&1
 }
 
 ensure_e2e_role() {

src/activities/execute_http.rs

@@ -49,13 +49,18 @@ async fn check_http_privilege(pool: &PgPool, submitted_by: &str) -> Result<(), S
 
 /// Build a reqwest Client with optional SSRF-safe DNS resolver.
 ///
+/// A default `User-Agent` is set so requests are not anonymous: some endpoints
+/// (e.g. fly.io-hosted services) reject requests that omit it. Nodes may still
+/// override it via an explicit `User-Agent` header.
+///
 /// Redirects are disabled to prevent redirect-based SSRF bypasses: an attacker
 /// could host a 302 redirecting to `http://169.254.169.254/...`, and reqwest
 /// would follow it without calling our DNS resolver (since the target is an IP
 /// literal).
 fn build_client(timeout: Duration) -> Result<reqwest::Client, String> {
     let builder = reqwest::Client::builder()
         .timeout(timeout)
+        .user_agent(concat!("pg_durable/", env!("CARGO_PKG_VERSION")))
         .redirect(reqwest::redirect::Policy::none());
 
     // Inject the SSRF-safe DNS resolver unless http-allow-all removes all guards.

tests/e2e/sql/06_http_and_ssrf.sql

@@ -580,10 +580,12 @@ END $$;
 DROP TABLE _test_ssrf7;
 
 -- Test 8: Redirects are not followed
+-- Uses httpbingo.org (maintained Go reimplementation) rather than httpbin.org,
+-- which is frequently unavailable and was the dominant source of CI flakiness.
 CREATE TEMP TABLE _test_ssrf8 (instance_id TEXT);
 
 INSERT INTO _test_ssrf8 SELECT df.start(
-    df.http('https://httpbin.org/status/302', 'GET', NULL, NULL, 10) |=> 'response'
+    df.http('https://httpbingo.org/status/302', 'GET') |=> 'response'
     ~> 'SELECT ($response::jsonb->>''status'')::int AS status_code',
     'test-ssrf-redirect-blocked'
 );
@@ -937,34 +939,9 @@ END $$;
 
 DROP TABLE _test_http_priv4;
 
--- Permission Test 5: grant_usage(false) does NOT grant df.http, but residual PUBLIC grant persists
-GRANT EXECUTE ON FUNCTION df.http(text, text, text, jsonb, integer) TO PUBLIC;
-
--- grant_usage is purely additive — it does not revoke df.http or warn about
--- residual PUBLIC grants.  The caller is responsible for revoking separately.
-
-DO $$
-BEGIN
-    BEGIN
-        PERFORM df.grant_usage('http_priv_bob', include_http => false);
-    EXCEPTION WHEN OTHERS THEN
-        RAISE EXCEPTION 'TEST FAILED: grant_usage raised an unexpected exception: %', SQLERRM;
-    END;
-    -- grant_usage(include_http => false) does not grant df.http, but the
-    -- PUBLIC grant above still gives http_priv_bob effective access.
-    IF NOT has_function_privilege('http_priv_bob'::regrole,
-                                   'df.http(text, text, text, jsonb, integer)', 'EXECUTE') THEN
-        RAISE EXCEPTION 'TEST FAILED: expected http_priv_bob to still have effective HTTP access via PUBLIC grant';
-    END IF;
-    RAISE NOTICE 'TEST PASSED: grant_usage_warns_on_residual_public_grant';
-END $$;
-
--- Restore: revoke the PUBLIC grant again so subsequent tests are unaffected
-REVOKE EXECUTE ON FUNCTION df.http(text, text, text, jsonb, integer) FROM PUBLIC;
-
--- Permission Test 6: Superuser always passes the execution-time privilege check
-CREATE TEMP TABLE _test_http_priv6 (instance_id TEXT);
-INSERT INTO _test_http_priv6
+-- Permission Test 5: Superuser always passes the execution-time privilege check
+CREATE TEMP TABLE _test_http_priv5 (instance_id TEXT);
+INSERT INTO _test_http_priv5
 SELECT df.start(
     '{"node_type":"HTTP","query":"{\"url\":\"https://api.github.com/\",\"method\":\"GET\",\"body\":null,\"headers\":null,\"timeout_seconds\":5}"}',
     'test-http-superuser'
@@ -976,7 +953,7 @@ DECLARE
     status      TEXT;
     node_result TEXT;
 BEGIN
-    SELECT instance_id INTO inst_id FROM _test_http_priv6;
+    SELECT instance_id INTO inst_id FROM _test_http_priv5;
 
     SELECT df.await_instance(inst_id, 30) INTO status;
 
@@ -995,7 +972,7 @@ BEGIN
     RAISE NOTICE 'TEST PASSED: superuser_bypasses_http_privilege_check';
 END $$;
 
-DROP TABLE _test_http_priv6;
+DROP TABLE _test_http_priv5;
 
 -- --- Permission Test Cleanup ---
 DO $cleanup$