fix(seeder): replace curlFetch with Node.js CONNECT tunnel in fredFetchJson (koala73#2451)

* chore: redeploy to pick up WORLDMONITOR_VALID_KEYS fix

* fix(seeder): replace curlFetch with Node.js CONNECT tunnel in fredFetchJson

Seeder Railway containers use node:22-alpine (no curl). fredFetchJson was
routing through curlFetch when PROXY_URL is set, causing spawnSync curl
ENOENT on every FRED series — all 22 series failed silently.

Fix: replace curlFetch call in fredFetchJson with a pure Node.js
HTTPS-through-HTTP-proxy CONNECT tunnel using built-in http/tls/https
modules. No new dependencies. curlFetch is kept for ais-relay.cjs callers
(Dockerfile.relay installs curl via apk add).

Root cause confirmed via logs: spawnSync curl ENOENT on all 22 FRED series.

* fix(seeder): try direct first, proxy as fallback in fredFetchJson

* chore(seeder): TODO to consolidate all curlFetch/proxy patterns into one helper

* fix(seeders): remove curl dependency from disease-outbreaks and fear-greed

Both seeders used curl as primary fetch path when PROXY_URL was set.
The Decodo proxy was returning SSL_ERROR_SYSCALL causing fetch failures.

Replace with native fetch() — same direct-first pattern as fredFetchJson
after the FRED fix. No fallback needed: these feeds are publicly accessible,
and partial failures are already handled by per-source try/catch.

* fix(seeder): add User-Agent to proxy tunnel; destroy socket on CONNECT failure

P1: httpsProxyFetchJson was missing User-Agent header — AGENTS.md requires
it for all server-side fetches; FRED CDN/WAF may reject headless requests.
P2: TCP socket left open on non-200 CONNECT response; call socket.destroy().

Author: koala73

scripts/_seed-utils.mjs

@@ -5,6 +5,13 @@ import { execFileSync } from 'node:child_process';
 import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { createRequire } from 'node:module';
+import * as http from 'node:http';
+import * as tls from 'node:tls';
+import * as https from 'node:https';
+import { promisify } from 'node:util';
+import { gunzip as _gunzip } from 'node:zlib';
+
+const gunzip = promisify(_gunzip);
 
 const CHROME_UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36';
 const MAX_PAYLOAD_BYTES = 5 * 1024 * 1024; // 5MB per key
@@ -301,13 +308,22 @@ export function sleep(ms) {
 }
 
 // ─── Proxy helpers for sources that block Railway container IPs ───
+// TODO: consolidate all fetch+proxy logic into a single proxyFetch(url, options) helper.
+// Current state: _seed-utils has fredFetchJson (fixed: direct-first, proxy fallback) and
+// curlFetch (curl-only, relay container). seed-disease-outbreaks.mjs and seed-fear-greed.mjs
+// each define their own local curlFetch that silently fails with ENOENT in seeder containers
+// (curl not in node:22-alpine). Each of those scripts should use a shared fetchWithProxyFallback
+// that tries native fetch first and falls back to httpsProxyFetchJson — same pattern as
+// fredFetchJson after this fix. Tracked: consolidate into one exported function.
 const { resolveProxyString } = createRequire(import.meta.url)('./_proxy-utils.cjs');
 
 export function resolveProxy() {
   return resolveProxyString();
 }
 
 // curl-based fetch; throws on non-2xx. Returns response body as string.
+// NOTE: requires curl binary — only available in Dockerfile.relay (apk add curl).
+// Do NOT call from standalone seed scripts; use fredFetchJson or httpsProxyFetchJson instead.
 export function curlFetch(url, proxyAuth, headers = {}) {
   const args = ['-sS', '--compressed', '--max-time', '15', '-L'];
   if (proxyAuth) args.push('-x', `http://${proxyAuth}`);
@@ -321,12 +337,84 @@ export function curlFetch(url, proxyAuth, headers = {}) {
   return raw.slice(0, nl);
 }
 
+// Pure Node.js HTTPS-through-HTTP-proxy (CONNECT tunnel).
+// Replaces curlFetch for seeder scripts running in containers without curl.
+// proxyAuth format: "user:pass@host:port"
+async function httpsProxyFetchJson(url, proxyAuth) {
+  const targetUrl = new URL(url);
+  const atIdx = proxyAuth.lastIndexOf('@');
+  const credentials = atIdx >= 0 ? proxyAuth.slice(0, atIdx) : '';
+  const hostPort = atIdx >= 0 ? proxyAuth.slice(atIdx + 1) : proxyAuth;
+  const colonIdx = hostPort.lastIndexOf(':');
+  const proxyHost = hostPort.slice(0, colonIdx);
+  const proxyPort = parseInt(hostPort.slice(colonIdx + 1), 10);
+
+  const connectHeaders = {};
+  if (credentials) {
+    connectHeaders['Proxy-Authorization'] = `Basic ${Buffer.from(credentials).toString('base64')}`;
+  }
+
+  const { socket } = await new Promise((resolve, reject) => {
+    http.request({
+      host: proxyHost, port: proxyPort,
+      method: 'CONNECT',
+      path: `${targetUrl.hostname}:443`,
+      headers: connectHeaders,
+    }).on('connect', (res, socket) => {
+      if (res.statusCode !== 200) {
+        socket.destroy();
+        return reject(Object.assign(new Error(`Proxy CONNECT: ${res.statusCode}`), { status: res.statusCode }));
+      }
+      resolve({ socket });
+    }).on('error', reject).end();
+  });
+
+  const tlsSock = tls.connect({ socket, servername: targetUrl.hostname });
+  await new Promise((resolve, reject) => {
+    tlsSock.on('secureConnect', resolve);
+    tlsSock.on('error', reject);
+  });
+
+  return new Promise((resolve, reject) => {
+    const timer = setTimeout(() => { tlsSock.destroy(); reject(new Error('FRED proxy fetch timeout')); }, 20000);
+    const fail = (e) => { clearTimeout(timer); tlsSock.destroy(); reject(e); };
+
+    https.request({
+      host: targetUrl.hostname,
+      path: targetUrl.pathname + targetUrl.search,
+      method: 'GET',
+      headers: { Accept: 'application/json', 'Accept-Encoding': 'gzip', 'User-Agent': CHROME_UA },
+      createConnection: () => tlsSock,
+    }, (resp) => {
+      clearTimeout(timer);
+      if (resp.statusCode < 200 || resp.statusCode >= 300) {
+        resp.resume();
+        return reject(Object.assign(new Error(`HTTP ${resp.statusCode}`), { status: resp.statusCode }));
+      }
+      const chunks = [];
+      resp.on('data', c => chunks.push(c));
+      resp.on('end', () => {
+        const body = Buffer.concat(chunks);
+        const isGzip = (resp.headers['content-encoding'] || '').includes('gzip');
+        (isGzip ? gunzip(body) : Promise.resolve(body))
+          .then(data => { try { resolve(JSON.parse(data.toString('utf8'))); } catch (e) { fail(e); } })
+          .catch(fail);
+      });
+      resp.on('error', fail);
+    }).on('error', fail).end();
+  });
+}
+
 // Fetch JSON from a FRED URL, routing through proxy when available.
 export async function fredFetchJson(url, proxyAuth) {
-  if (proxyAuth) return JSON.parse(curlFetch(url, proxyAuth, { Accept: 'application/json' }));
-  const r = await fetch(url, { headers: { Accept: 'application/json' }, signal: AbortSignal.timeout(10_000) });
-  if (!r.ok) throw Object.assign(new Error(`HTTP ${r.status}`), { status: r.status });
-  return r.json();
+  try {
+    const r = await fetch(url, { headers: { Accept: 'application/json' }, signal: AbortSignal.timeout(10_000) });
+    if (r.ok) return r.json();
+    throw Object.assign(new Error(`HTTP ${r.status}`), { status: r.status });
+  } catch (directErr) {
+    if (!proxyAuth) throw directErr;
+    return httpsProxyFetchJson(url, proxyAuth);
+  }
 }
 
 // ---------------------------------------------------------------------------

scripts/seed-disease-outbreaks.mjs

@@ -1,7 +1,6 @@
 #!/usr/bin/env node
 
-import { execFileSync } from 'node:child_process';
-import { loadEnvFile, CHROME_UA, runSeed, resolveProxy } from './_seed-utils.mjs';
+import { loadEnvFile, CHROME_UA, runSeed } from './_seed-utils.mjs';
 import { extractCountryCode } from './shared/geo-extract.mjs';
 
 loadEnvFile(import.meta.url);
@@ -42,22 +41,6 @@ const TGH_LOOKBACK_DAYS = 90;
 
 const RSS_MAX_BYTES = 500_000; // guard against oversized responses before regex
 
-const _proxyAuth = resolveProxy();
-
-// curl-based fetch for sources that block Railway IPs (e.g. Outbreak News Today).
-function curlFetch(url, extraHeaders = {}) {
-  const args = ['-sS', '--compressed', '--max-time', '15', '-L'];
-  if (_proxyAuth) args.push('-x', `http://${_proxyAuth}`);
-  args.push('-H', `User-Agent: ${CHROME_UA}`);
-  for (const [k, v] of Object.entries(extraHeaders)) args.push('-H', `${k}: ${v}`);
-  args.push('-w', '\n%{http_code}');
-  args.push(url);
-  const raw = execFileSync('curl', args, { encoding: 'utf8', timeout: 20000, stdio: ['pipe', 'pipe', 'pipe'] });
-  const nl = raw.lastIndexOf('\n');
-  const httpStatus = parseInt(raw.slice(nl + 1).trim(), 10);
-  if (httpStatus < 200 || httpStatus >= 300) throw Object.assign(new Error(`HTTP ${httpStatus}`), { status: httpStatus });
-  return raw.slice(0, nl);
-}
 
 function stableHash(str) {
   let h = 0;
@@ -131,19 +114,14 @@ async function fetchWhoDonApi() {
   }
 }
 
-async function fetchRssItems(url, sourceName, useCurl = false) {
+async function fetchRssItems(url, sourceName) {
   try {
-    let xml;
-    if (useCurl) {
-      xml = curlFetch(url, { Accept: 'application/rss+xml, application/xml, text/xml' });
-    } else {
-      const resp = await fetch(url, {
-        headers: { Accept: 'application/rss+xml, application/xml, text/xml', 'User-Agent': CHROME_UA },
-        signal: AbortSignal.timeout(15000),
-      });
-      if (!resp.ok) { console.warn(`[Disease] ${sourceName} HTTP ${resp.status}`); return []; }
-      xml = await resp.text();
-    }
+    const resp = await fetch(url, {
+      headers: { Accept: 'application/rss+xml, application/xml, text/xml', 'User-Agent': CHROME_UA },
+      signal: AbortSignal.timeout(15000),
+    });
+    if (!resp.ok) { console.warn(`[Disease] ${sourceName} HTTP ${resp.status}`); return []; }
+    const xml = await resp.text();
     const bounded = xml.length > RSS_MAX_BYTES ? xml.slice(0, RSS_MAX_BYTES) : xml;
     const items = [];
     const itemRe = /<item>([\s\S]*?)<\/item>/g;
@@ -268,7 +246,7 @@ async function fetchDiseaseOutbreaks() {
   const [whoItems, cdcItems, outbreakNewsItems, tghItems] = await Promise.all([
     fetchWhoDonApi(),
     fetchRssItems(CDC_FEED, 'CDC'),
-    fetchRssItems(OUTBREAK_NEWS_FEED, 'Outbreak News Today', true),
+    fetchRssItems(OUTBREAK_NEWS_FEED, 'Outbreak News Today'),
     fetchThinkGlobalHealth(),
   ]);
   console.log(`[Disease] Sources: WHO=${whoItems.length} CDC=${cdcItems.length} ONT=${outbreakNewsItems.length} TGH=${tghItems.length}`);

scripts/seed-fear-greed.mjs

@@ -1,26 +1,8 @@
 #!/usr/bin/env node
 
-import { loadEnvFile, CHROME_UA, runSeed, readSeedSnapshot, sleep, resolveProxy } from './_seed-utils.mjs';
-import { execFileSync } from 'child_process';
+import { loadEnvFile, CHROME_UA, runSeed, readSeedSnapshot, sleep } from './_seed-utils.mjs';
 loadEnvFile(import.meta.url);
 
-const _proxyAuth = resolveProxy();
-
-// curl-based fetch for sources that block Railway IPs (Yahoo Finance).
-// Returns response body as string; throws on non-2xx.
-function curlFetch(url, headers = {}) {
-  const args = ['-sS', '--compressed', '--max-time', '15', '-L'];
-  if (_proxyAuth) args.push('-x', `http://${_proxyAuth}`);
-  for (const [k, v] of Object.entries(headers)) args.push('-H', `${k}: ${v}`);
-  args.push('-w', '\n%{http_code}');
-  args.push(url);
-  const raw = execFileSync('curl', args, { encoding: 'utf8', timeout: 20000, stdio: ['pipe', 'pipe', 'pipe'] });
-  const nl = raw.lastIndexOf('\n');
-  const status = parseInt(raw.slice(nl + 1).trim(), 10);
-  if (status < 200 || status >= 300) throw Object.assign(new Error(`HTTP ${status}`), { status });
-  return raw.slice(0, nl);
-}
-
 const FEAR_GREED_KEY = 'market:fear-greed:v1';
 const FEAR_GREED_TTL = 64800; // 18h = 3x 6h interval
 
@@ -33,13 +15,10 @@ async function fetchYahooSymbol(symbol) {
   const url = `https://query1.finance.yahoo.com/v8/finance/chart/${encodeURIComponent(symbol)}?interval=1d&range=1y`;
   const headers = { 'User-Agent': CHROME_UA, Accept: 'application/json' };
   try {
-    // Use curl+proxy when available — Railway container IPs are periodically blocked by Yahoo.
-    const text = _proxyAuth
-      ? curlFetch(url, headers)
-      : await fetch(url, { headers, signal: AbortSignal.timeout(10_000) }).then(r => {
-          if (!r.ok) throw Object.assign(new Error(`HTTP ${r.status}`), { status: r.status });
-          return r.text();
-        });
+    const text = await fetch(url, { headers, signal: AbortSignal.timeout(10_000) }).then(r => {
+      if (!r.ok) throw Object.assign(new Error(`HTTP ${r.status}`), { status: r.status });
+      return r.text();
+    });
     const data = JSON.parse(text);
     const result = data?.chart?.result?.[0];
     if (!result) return null;