Skip to content

Commit b260cbf

Browse files
meirk-brdmeirk-brd
committed
fix polling
1 parent 542a1ae commit b260cbf

2 files changed

Lines changed: 70 additions & 104 deletions

File tree

‎app/api/scrape/trigger/route.ts‎

Lines changed: 9 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
import { NextRequest, NextResponse } from "next/server";
22
import { z } from "zod";
33
import { runAiScraper } from "@/lib/server/brightdata-scraper";
4-
import { createJob, resolveJob, failJob } from "@/lib/server/job-store";
54
import { checkAndRecordRun } from "@/lib/server/rate-limit";
65

6+
// Allow up to 5 minutes for the scraper to complete
7+
export const maxDuration = 300;
8+
79
const InputSchema = z.object({
810
provider: z.enum([
911
"chatgpt",
@@ -38,26 +40,17 @@ export async function POST(req: NextRequest) {
3840
}
3941

4042
const jobId = crypto.randomUUID();
43+
console.log(`[trigger] Job started jobId=${jobId} provider=${parsed.provider} prompt="${parsed.prompt.slice(0, 80)}..."`);
4144

42-
createJob(jobId);
43-
44-
console.log(`[trigger] Job created jobId=${jobId} provider=${parsed.provider} prompt="${parsed.prompt.slice(0, 80)}..."`);
45+
// Await the scraper — returns result directly, no polling needed
46+
const result = await runAiScraper(parsed);
4547

46-
// Fire and forget — do NOT await
47-
runAiScraper(parsed)
48-
.then((result) => {
49-
console.log(`[trigger] Job resolved jobId=${jobId} provider=${parsed.provider} answer.length=${result.answer.length} sources=${result.sources.length}`);
50-
resolveJob(jobId, result);
51-
})
52-
.catch((err: unknown) => {
53-
const message = err instanceof Error ? err.message : "Unknown error";
54-
console.error(`[trigger] Job FAILED jobId=${jobId} provider=${parsed.provider} error="${message}"`);
55-
failJob(jobId, message);
56-
});
48+
console.log(`[trigger] Job done jobId=${jobId} provider=${parsed.provider} answer.length=${result.answer.length} sources=${result.sources.length}`);
5749

58-
return NextResponse.json({ jobId, status: "pending" });
50+
return NextResponse.json({ jobId, status: "ready", result });
5951
} catch (error) {
6052
const message = error instanceof Error ? error.message : "Unknown error";
53+
console.error(`[trigger] Job FAILED error="${message}"`);
6154
return NextResponse.json({ error: message }, { status: 400 });
6255
}
6356
}

‎components/sovereign-dashboard.tsx‎

Lines changed: 61 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -570,13 +570,13 @@ export function SovereignDashboard({ demoMode = false }: { demoMode?: boolean }
570570
return Math.min(100, score);
571571
}
572572

573-
/** Run a single scrape against one specific provider (trigger + poll) */
573+
/** Run a single scrape against one specific provider */
574574
async function callScrapeOne(prompt: string, provider: Provider): Promise<ScrapeRun | null> {
575575
if (demoMode) { setMessage("Demo mode — API calls are disabled"); return null; }
576576
try {
577577
console.log(`[scrape] callScrapeOne: START provider=${provider} prompt="${prompt.slice(0, 80)}..."`);
578578

579-
// Trigger — returns immediately with a jobId
579+
// Trigger — waits for full scrape completion, returns result directly
580580
const triggerRes = await fetch(`${BASE_PATH}/api/scrape/trigger`, {
581581
method: "POST",
582582
headers: { "Content-Type": "application/json" },
@@ -592,101 +592,74 @@ export function SovereignDashboard({ demoMode = false }: { demoMode?: boolean }
592592
console.error(`[scrape] callScrapeOne: Trigger FAILED provider=${provider} status=${triggerRes.status} error="${triggerData.error}"`);
593593
throw new Error(triggerData.error || "Trigger failed");
594594
}
595-
const { jobId } = triggerData as { jobId: string };
596-
console.log(`[scrape] callScrapeOne: Triggered provider=${provider} jobId=${jobId}`);
597-
598-
// Poll until ready, failed, or 90s timeout
599-
const POLL_INTERVAL_MS = 2000;
600-
const TIMEOUT_MS = 240_000;
601-
const deadline = Date.now() + TIMEOUT_MS;
602-
let pollCount = 0;
603-
604-
while (Date.now() < deadline) {
605-
await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
606-
pollCount++;
607-
608-
const statusRes = await fetch(`${BASE_PATH}/api/scrape/status/${jobId}`);
609-
const statusData = await statusRes.json() as {
610-
status: "pending" | "ready" | "failed";
611-
result?: { provider: string; prompt: string; answer: string; sources: string[]; createdAt: string };
612-
error?: string;
613-
};
614-
615-
console.log(`[scrape] callScrapeOne: Poll #${pollCount} jobId=${jobId} provider=${provider} status=${statusData.status}`);
616595

617-
if (statusData.status === "failed") {
618-
console.error(`[scrape] callScrapeOne: Job FAILED jobId=${jobId} provider=${provider} error="${statusData.error}"`);
619-
throw new Error(statusData.error || "Scrape job failed");
620-
}
596+
const { jobId, result: jobResult } = triggerData as {
597+
jobId: string;
598+
status: "ready";
599+
result: { provider: string; prompt: string; answer: string; sources: string[]; createdAt: string };
600+
};
621601

622-
if (statusData.status === "ready" && statusData.result) {
623-
const { answer: answerText, sources: sourceList, provider: p, prompt: pr, createdAt } = statusData.result;
624-
console.log(`[scrape] callScrapeOne: READY provider=${provider} jobId=${jobId} answer.length=${answerText.length} sources=${sourceList.length}`);
602+
const { answer: answerText, sources: sourceList, provider: p, prompt: pr, createdAt } = jobResult;
603+
console.log(`[scrape] callScrapeOne: READY provider=${provider} jobId=${jobId} answer.length=${answerText.length} sources=${sourceList.length}`);
625604

626-
// AI-powered analysis (accumulation: only new runs get analyzed)
627-
let visibilityScore: number;
628-
let sentiment: ScrapeRun["sentiment"];
629-
let brandMentions: string[];
630-
let competitorMentions: string[];
631-
let aiAnalyzed = false;
605+
// AI-powered analysis
606+
let visibilityScore: number;
607+
let sentiment: ScrapeRun["sentiment"];
608+
let brandMentions: string[];
609+
let competitorMentions: string[];
610+
let aiAnalyzed = false;
632611

633-
try {
634-
const analysisRes = await fetch(`${BASE_PATH}/api/analyze-run`, {
635-
method: "POST",
636-
headers: { "Content-Type": "application/json" },
637-
body: JSON.stringify({
638-
answer: answerText,
639-
brandName: state.brand.brandName,
640-
brandAliases: state.brand.brandAliases,
641-
brandWebsite: state.brand.website,
642-
competitors: state.competitors,
643-
}),
644-
});
645-
if (analysisRes.ok) {
646-
const analysis = await analysisRes.json() as {
647-
visibilityScore: number;
648-
sentiment: ScrapeRun["sentiment"];
649-
brandMentioned: boolean;
650-
brandMentions: string[];
651-
competitorMentions: string[];
652-
};
653-
visibilityScore = analysis.visibilityScore;
654-
sentiment = analysis.sentiment;
655-
brandMentions = analysis.brandMentions;
656-
competitorMentions = analysis.competitorMentions;
657-
aiAnalyzed = true;
658-
console.log(`[scrape] callScrapeOne: AI analysis done provider=${provider} score=${visibilityScore} sentiment=${sentiment}`);
659-
} else {
660-
throw new Error(`analyze-run HTTP ${analysisRes.status}`);
661-
}
662-
} catch (analysisErr) {
663-
console.warn(`[scrape] callScrapeOne: AI analysis failed, falling back to heuristics. Error:`, analysisErr instanceof Error ? analysisErr.message : analysisErr);
664-
const brandTerms = getBrandTerms();
665-
const competitorTerms = getCompetitorTerms();
666-
visibilityScore = calcVisibilityScore(answerText, sourceList, brandTerms);
667-
sentiment = detectSentiment(answerText, brandTerms);
668-
brandMentions = findMentions(answerText, brandTerms);
669-
competitorMentions = findMentions(answerText, competitorTerms);
670-
}
671-
672-
return {
673-
provider: p as Provider,
674-
prompt: pr,
612+
try {
613+
const analysisRes = await fetch(`${BASE_PATH}/api/analyze-run`, {
614+
method: "POST",
615+
headers: { "Content-Type": "application/json" },
616+
body: JSON.stringify({
675617
answer: answerText,
676-
sources: sourceList,
677-
createdAt: createdAt || new Date().toISOString(),
678-
visibilityScore,
679-
sentiment,
680-
brandMentions,
681-
competitorMentions,
682-
aiAnalyzed,
618+
brandName: state.brand.brandName,
619+
brandAliases: state.brand.brandAliases,
620+
brandWebsite: state.brand.website,
621+
competitors: state.competitors,
622+
}),
623+
});
624+
if (analysisRes.ok) {
625+
const analysis = await analysisRes.json() as {
626+
visibilityScore: number;
627+
sentiment: ScrapeRun["sentiment"];
628+
brandMentioned: boolean;
629+
brandMentions: string[];
630+
competitorMentions: string[];
683631
};
632+
visibilityScore = analysis.visibilityScore;
633+
sentiment = analysis.sentiment;
634+
brandMentions = analysis.brandMentions;
635+
competitorMentions = analysis.competitorMentions;
636+
aiAnalyzed = true;
637+
console.log(`[scrape] callScrapeOne: AI analysis done provider=${provider} score=${visibilityScore} sentiment=${sentiment}`);
638+
} else {
639+
throw new Error(`analyze-run HTTP ${analysisRes.status}`);
684640
}
685-
// status === "pending" → keep polling
641+
} catch (analysisErr) {
642+
console.warn(`[scrape] callScrapeOne: AI analysis failed, falling back to heuristics. Error:`, analysisErr instanceof Error ? analysisErr.message : analysisErr);
643+
const brandTerms = getBrandTerms();
644+
const competitorTerms = getCompetitorTerms();
645+
visibilityScore = calcVisibilityScore(answerText, sourceList, brandTerms);
646+
sentiment = detectSentiment(answerText, brandTerms);
647+
brandMentions = findMentions(answerText, brandTerms);
648+
competitorMentions = findMentions(answerText, competitorTerms);
686649
}
687650

688-
console.error(`[scrape] callScrapeOne: TIMED OUT provider=${provider} jobId=${jobId} after ${pollCount} polls`);
689-
throw new Error("Timed out waiting for scrape result");
651+
return {
652+
provider: p as Provider,
653+
prompt: pr,
654+
answer: answerText,
655+
sources: sourceList,
656+
createdAt: createdAt || new Date().toISOString(),
657+
visibilityScore,
658+
sentiment,
659+
brandMentions,
660+
competitorMentions,
661+
aiAnalyzed,
662+
};
690663
} catch (err) {
691664
console.error(`[scrape] callScrapeOne: CAUGHT ERROR provider=${provider}`, err instanceof Error ? err.message : err);
692665
return null;

0 commit comments

Comments
 (0)