From cde986b97d61e5139f4c4bdfa906a4b1b42165dd Mon Sep 17 00:00:00 2001 From: jackwener Date: Mon, 6 Apr 2026 02:18:13 +0800 Subject: [PATCH 1/2] refactor: extract shared scoring logic and consolidate time format utils - Extract applyUrlScoreAdjustments() and scoreArrayResponse() to analysis.ts, eliminating duplicated endpoint scoring between explore.ts and record.ts - Consolidate formatDuration/formatUptime into a single formatDuration(ms) in download/progress.ts, reused by commands/daemon.ts --- src/analysis.ts | 26 ++++++++++++++++++++++++++ src/commands/daemon.ts | 11 ++--------- src/download/progress.ts | 9 +++++++-- src/explore.ts | 7 +++---- src/record.ts | 23 ++++------------------- 5 files changed, 42 insertions(+), 34 deletions(-) diff --git a/src/analysis.ts b/src/analysis.ts index 12bd0f176..f576b83f1 100644 --- a/src/analysis.ts +++ b/src/analysis.ts @@ -150,6 +150,32 @@ export function detectAuthFromContent(url: string, body: unknown): string[] { return indicators; } +// ── Shared scoring helpers ─────────────────────────────────────────────────── + +/** URL-based score adjustments shared by explore and record scoring. */ +export function applyUrlScoreAdjustments(url: string, score: number): number { + let s = score; + if (url.includes('/api/') || url.includes('/x/')) s += 3; + if (url.match(/\/(track|log|analytics|beacon|pixel|stats|metric)/i)) s -= 10; + if (url.match(/\/(ping|heartbeat|keep.?alive)/i)) s -= 10; + return s; +} + +/** Score an array response based on item count and detected field roles. */ +export function scoreArrayResponse(arrayResult: ArrayDiscovery | null): number { + if (!arrayResult) return 0; + let s = 10; + s += Math.min(arrayResult.items.length, 10); + const sample = arrayResult.items[0]; + if (sample && typeof sample === 'object') { + const keys = Object.keys(sample as object).map(k => k.toLowerCase()); + for (const aliases of Object.values(FIELD_ROLES)) { + if (aliases.some(a => keys.includes(a))) s += 2; + } + } + return s; +} + // ── Query param classification ────────────────────────────────────────────── /** Extract non-volatile query params and classify them. */ diff --git a/src/commands/daemon.ts b/src/commands/daemon.ts index 5a77ccffe..c763d0e69 100644 --- a/src/commands/daemon.ts +++ b/src/commands/daemon.ts @@ -7,14 +7,7 @@ import chalk from 'chalk'; import { fetchDaemonStatus, requestDaemonShutdown } from '../browser/daemon-client.js'; - -function formatUptime(seconds: number): string { - const h = Math.floor(seconds / 3600); - const m = Math.floor((seconds % 3600) / 60); - if (h > 0) return `${h}h ${m}m`; - if (m > 0) return `${m}m`; - return `${Math.floor(seconds)}s`; -} +import { formatDuration } from '../download/progress.js'; function formatTimeSince(timestampMs: number): string { const seconds = (Date.now() - timestampMs) / 1000; @@ -33,7 +26,7 @@ export async function daemonStatus(): Promise { } console.log(`Daemon: ${chalk.green('running')} (PID ${status.pid})`); - console.log(`Uptime: ${formatUptime(status.uptime)}`); + console.log(`Uptime: ${formatDuration(status.uptime * 1000)}`); console.log(`Extension: ${status.extensionConnected ? chalk.green('connected') : chalk.yellow('disconnected')}`); console.log(`Last CLI request: ${formatTimeSince(status.lastCliRequestTime)}`); console.log(`Memory: ${status.memoryMB} MB`); diff --git a/src/download/progress.ts b/src/download/progress.ts index b7c6ddbae..2f20da777 100644 --- a/src/download/progress.ts +++ b/src/download/progress.ts @@ -29,8 +29,13 @@ export function formatDuration(ms: number): string { const seconds = Math.floor(ms / 1000); if (seconds < 60) return `${seconds}s`; const minutes = Math.floor(seconds / 60); - const remainingSeconds = seconds % 60; - return `${minutes}m ${remainingSeconds}s`; + if (minutes < 60) { + const remainingSeconds = seconds % 60; + return remainingSeconds > 0 ? `${minutes}m ${remainingSeconds}s` : `${minutes}m`; + } + const hours = Math.floor(minutes / 60); + const remainingMinutes = minutes % 60; + return remainingMinutes > 0 ? `${hours}h ${remainingMinutes}m` : `${hours}h`; } /** diff --git a/src/explore.ts b/src/explore.ts index 3a19d31e4..cd47add04 100644 --- a/src/explore.ts +++ b/src/explore.ts @@ -25,6 +25,7 @@ import { inferStrategy, detectAuthFromHeaders, classifyQueryParams, + applyUrlScoreAdjustments, } from './analysis.js'; // ── Site name detection ──────────────────────────────────────────────────── @@ -194,18 +195,16 @@ function isBooleanRecord(value: unknown): value is Record { && Object.values(value as Record).every(v => typeof v === 'boolean'); } -function scoreEndpoint(ep: { contentType: string; responseAnalysis: AnalyzedEndpoint['responseAnalysis']; pattern: string; status: number | null; hasSearchParam: boolean; hasPaginationParam: boolean; hasLimitParam: boolean }): number { +function scoreEndpoint(ep: { contentType: string; responseAnalysis: AnalyzedEndpoint['responseAnalysis']; pattern: string; url: string; status: number | null; hasSearchParam: boolean; hasPaginationParam: boolean; hasLimitParam: boolean }): number { let s = 0; if (ep.contentType.includes('json')) s += 10; if (ep.responseAnalysis) { s += 5; s += Math.min(ep.responseAnalysis.itemCount, 10); s += Object.keys(ep.responseAnalysis.detectedFields).length * 2; } - if (ep.pattern.includes('/api/') || ep.pattern.includes('/x/')) s += 3; if (ep.hasSearchParam) s += 3; if (ep.hasPaginationParam) s += 2; if (ep.hasLimitParam) s += 2; if (ep.status === 200) s += 2; - // Anti-Bot Empty Value Detection: penalize JSON endpoints returning empty data if (ep.responseAnalysis && ep.responseAnalysis.itemCount === 0 && ep.contentType.includes('json')) s -= 3; - return s; + return applyUrlScoreAdjustments(ep.url, s); } diff --git a/src/record.ts b/src/record.ts index c4932b6f4..14a24929a 100644 --- a/src/record.ts +++ b/src/record.ts @@ -27,6 +27,8 @@ import { inferStrategy, detectAuthFromContent, classifyQueryParams, + applyUrlScoreAdjustments, + scoreArrayResponse, } from './analysis.js'; // ── Types ────────────────────────────────────────────────────────────────── @@ -82,11 +84,7 @@ function preferRecordedCandidate(current: RecordedCandidate, next: RecordedCandi /** Apply shared endpoint score tweaks. */ function applyCommonEndpointScoreAdjustments(req: RecordedRequest, score: number): number { - let adjusted = score; - if (req.url.includes('/api/')) adjusted += 3; - if (req.url.match(/\/(track|log|analytics|beacon|pixel|stats|metric)/i)) adjusted -= 10; - if (req.url.match(/\/(ping|heartbeat|keep.?alive)/i)) adjusted -= 10; - return adjusted; + return applyUrlScoreAdjustments(req.url, score); } /** Build a candidate-level dedupe key. */ @@ -330,20 +328,7 @@ function generateReadRecordedJs(): string { // ── Analysis helpers ─────────────────────────────────────────────────────── function scoreRequest(req: RecordedRequest, arrayResult: ReturnType | null): number { - let s = 0; - if (arrayResult) { - s += 10; - s += Math.min(arrayResult.items.length, 10); - // Bonus for detected semantic fields - const sample = arrayResult.items[0]; - if (sample && typeof sample === 'object') { - const keys = Object.keys(sample as object).map(k => k.toLowerCase()); - for (const aliases of Object.values(FIELD_ROLES)) { - if (aliases.some(a => keys.includes(a))) s += 2; - } - } - } - return applyCommonEndpointScoreAdjustments(req, s); + return applyCommonEndpointScoreAdjustments(req, scoreArrayResponse(arrayResult)); } /** Check whether one recorded request is safe to treat as a write candidate. */ From 96666f5660b63f626e87989ee091f3031b343a73 Mon Sep 17 00:00:00 2001 From: jackwener Date: Mon, 6 Apr 2026 02:49:32 +0800 Subject: [PATCH 2/2] fix: preserve explore scoring semantics and round daemon uptime - Revert explore.ts scoreEndpoint to original inline /api/ /x/ bonus without record's tracking/analytics penalty (blocker from review) - Math.round uptime*1000 to avoid floating-point noise in daemon status --- src/commands/daemon.ts | 2 +- src/explore.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/commands/daemon.ts b/src/commands/daemon.ts index c763d0e69..bc3a079f2 100644 --- a/src/commands/daemon.ts +++ b/src/commands/daemon.ts @@ -26,7 +26,7 @@ export async function daemonStatus(): Promise { } console.log(`Daemon: ${chalk.green('running')} (PID ${status.pid})`); - console.log(`Uptime: ${formatDuration(status.uptime * 1000)}`); + console.log(`Uptime: ${formatDuration(Math.round(status.uptime * 1000))}`); console.log(`Extension: ${status.extensionConnected ? chalk.green('connected') : chalk.yellow('disconnected')}`); console.log(`Last CLI request: ${formatTimeSince(status.lastCliRequestTime)}`); console.log(`Memory: ${status.memoryMB} MB`); diff --git a/src/explore.ts b/src/explore.ts index cd47add04..b5696e380 100644 --- a/src/explore.ts +++ b/src/explore.ts @@ -25,7 +25,6 @@ import { inferStrategy, detectAuthFromHeaders, classifyQueryParams, - applyUrlScoreAdjustments, } from './analysis.js'; // ── Site name detection ──────────────────────────────────────────────────── @@ -195,16 +194,17 @@ function isBooleanRecord(value: unknown): value is Record { && Object.values(value as Record).every(v => typeof v === 'boolean'); } -function scoreEndpoint(ep: { contentType: string; responseAnalysis: AnalyzedEndpoint['responseAnalysis']; pattern: string; url: string; status: number | null; hasSearchParam: boolean; hasPaginationParam: boolean; hasLimitParam: boolean }): number { +function scoreEndpoint(ep: { contentType: string; responseAnalysis: AnalyzedEndpoint['responseAnalysis']; pattern: string; status: number | null; hasSearchParam: boolean; hasPaginationParam: boolean; hasLimitParam: boolean }): number { let s = 0; if (ep.contentType.includes('json')) s += 10; if (ep.responseAnalysis) { s += 5; s += Math.min(ep.responseAnalysis.itemCount, 10); s += Object.keys(ep.responseAnalysis.detectedFields).length * 2; } + if (ep.pattern.includes('/api/') || ep.pattern.includes('/x/')) s += 3; if (ep.hasSearchParam) s += 3; if (ep.hasPaginationParam) s += 2; if (ep.hasLimitParam) s += 2; if (ep.status === 200) s += 2; if (ep.responseAnalysis && ep.responseAnalysis.itemCount === 0 && ep.contentType.includes('json')) s -= 3; - return applyUrlScoreAdjustments(ep.url, s); + return s; }