Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions src/analysis.ts
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,32 @@ export function detectAuthFromContent(url: string, body: unknown): string[] {
return indicators;
}

// ── Shared scoring helpers ───────────────────────────────────────────────────

/** URL-based score adjustments shared by explore and record scoring. */
export function applyUrlScoreAdjustments(url: string, score: number): number {
let s = score;
if (url.includes('/api/') || url.includes('/x/')) s += 3;
if (url.match(/\/(track|log|analytics|beacon|pixel|stats|metric)/i)) s -= 10;
if (url.match(/\/(ping|heartbeat|keep.?alive)/i)) s -= 10;
return s;
}

/** Score an array response based on item count and detected field roles. */
export function scoreArrayResponse(arrayResult: ArrayDiscovery | null): number {
if (!arrayResult) return 0;
let s = 10;
s += Math.min(arrayResult.items.length, 10);
const sample = arrayResult.items[0];
if (sample && typeof sample === 'object') {
const keys = Object.keys(sample as object).map(k => k.toLowerCase());
for (const aliases of Object.values(FIELD_ROLES)) {
if (aliases.some(a => keys.includes(a))) s += 2;
}
}
return s;
}

// ── Query param classification ──────────────────────────────────────────────

/** Extract non-volatile query params and classify them. */
Expand Down
11 changes: 2 additions & 9 deletions src/commands/daemon.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,7 @@

import chalk from 'chalk';
import { fetchDaemonStatus, requestDaemonShutdown } from '../browser/daemon-client.js';

function formatUptime(seconds: number): string {
const h = Math.floor(seconds / 3600);
const m = Math.floor((seconds % 3600) / 60);
if (h > 0) return `${h}h ${m}m`;
if (m > 0) return `${m}m`;
return `${Math.floor(seconds)}s`;
}
import { formatDuration } from '../download/progress.js';

function formatTimeSince(timestampMs: number): string {
const seconds = (Date.now() - timestampMs) / 1000;
Expand All @@ -33,7 +26,7 @@ export async function daemonStatus(): Promise<void> {
}

console.log(`Daemon: ${chalk.green('running')} (PID ${status.pid})`);
console.log(`Uptime: ${formatUptime(status.uptime)}`);
console.log(`Uptime: ${formatDuration(Math.round(status.uptime * 1000))}`);
console.log(`Extension: ${status.extensionConnected ? chalk.green('connected') : chalk.yellow('disconnected')}`);
console.log(`Last CLI request: ${formatTimeSince(status.lastCliRequestTime)}`);
console.log(`Memory: ${status.memoryMB} MB`);
Expand Down
9 changes: 7 additions & 2 deletions src/download/progress.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,13 @@ export function formatDuration(ms: number): string {
const seconds = Math.floor(ms / 1000);
if (seconds < 60) return `${seconds}s`;
const minutes = Math.floor(seconds / 60);
const remainingSeconds = seconds % 60;
return `${minutes}m ${remainingSeconds}s`;
if (minutes < 60) {
const remainingSeconds = seconds % 60;
return remainingSeconds > 0 ? `${minutes}m ${remainingSeconds}s` : `${minutes}m`;
}
const hours = Math.floor(minutes / 60);
const remainingMinutes = minutes % 60;
return remainingMinutes > 0 ? `${hours}h ${remainingMinutes}m` : `${hours}h`;
}

/**
Expand Down
1 change: 0 additions & 1 deletion src/explore.ts
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,6 @@ function scoreEndpoint(ep: { contentType: string; responseAnalysis: AnalyzedEndp
if (ep.hasPaginationParam) s += 2;
if (ep.hasLimitParam) s += 2;
if (ep.status === 200) s += 2;
// Anti-Bot Empty Value Detection: penalize JSON endpoints returning empty data
if (ep.responseAnalysis && ep.responseAnalysis.itemCount === 0 && ep.contentType.includes('json')) s -= 3;
return s;
}
Expand Down
23 changes: 4 additions & 19 deletions src/record.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ import {
inferStrategy,
detectAuthFromContent,
classifyQueryParams,
applyUrlScoreAdjustments,
scoreArrayResponse,
} from './analysis.js';

// ── Types ──────────────────────────────────────────────────────────────────
Expand Down Expand Up @@ -82,11 +84,7 @@ function preferRecordedCandidate(current: RecordedCandidate, next: RecordedCandi

/** Apply shared endpoint score tweaks. */
function applyCommonEndpointScoreAdjustments(req: RecordedRequest, score: number): number {
let adjusted = score;
if (req.url.includes('/api/')) adjusted += 3;
if (req.url.match(/\/(track|log|analytics|beacon|pixel|stats|metric)/i)) adjusted -= 10;
if (req.url.match(/\/(ping|heartbeat|keep.?alive)/i)) adjusted -= 10;
return adjusted;
return applyUrlScoreAdjustments(req.url, score);
}

/** Build a candidate-level dedupe key. */
Expand Down Expand Up @@ -330,20 +328,7 @@ function generateReadRecordedJs(): string {
// ── Analysis helpers ───────────────────────────────────────────────────────

function scoreRequest(req: RecordedRequest, arrayResult: ReturnType<typeof findArrayPath> | null): number {
let s = 0;
if (arrayResult) {
s += 10;
s += Math.min(arrayResult.items.length, 10);
// Bonus for detected semantic fields
const sample = arrayResult.items[0];
if (sample && typeof sample === 'object') {
const keys = Object.keys(sample as object).map(k => k.toLowerCase());
for (const aliases of Object.values(FIELD_ROLES)) {
if (aliases.some(a => keys.includes(a))) s += 2;
}
}
}
return applyCommonEndpointScoreAdjustments(req, s);
return applyCommonEndpointScoreAdjustments(req, scoreArrayResponse(arrayResult));
}

/** Check whether one recorded request is safe to treat as a write candidate. */
Expand Down