feat(examples): add polling loop to crawl examples

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

Author: FrancescoSaverioZuppichini

examples/crawl/crawl_basic.ts

@@ -4,18 +4,33 @@ import { ScrapeGraphAI } from "scrapegraph-js";
 const sgai = ScrapeGraphAI();
 
 const startRes = await sgai.crawl.start({
-	url: "https://example.com",
+	url: "https://scrapegraphai.com/",
 	maxPages: 5,
 	maxDepth: 2,
 });
 
 if (startRes.status !== "success" || !startRes.data) {
 	console.error("Failed to start:", startRes.error);
 } else {
-	console.log("Crawl started:", startRes.data.id);
-	console.log("Status:", startRes.data.status);
+	const crawlId = startRes.data.id;
+	console.log("Crawl started:", crawlId);
 
-	const getRes = await sgai.crawl.get(startRes.data.id);
-	console.log("\nProgress:", getRes.data?.finished, "/", getRes.data?.total);
-	console.log("Pages:", getRes.data?.pages.map((p) => p.url));
+	let status = startRes.data.status;
+	while (status === "running") {
+		await new Promise((r) => setTimeout(r, 2000));
+		const getRes = await sgai.crawl.get(crawlId);
+		if (getRes.status !== "success" || !getRes.data) {
+			console.error("Failed to get status:", getRes.error);
+			break;
+		}
+		status = getRes.data.status;
+		console.log(`Progress: ${getRes.data.finished}/${getRes.data.total} - ${status}`);
+
+		if (status === "completed" || status === "failed") {
+			console.log("\nPages crawled:");
+			for (const page of getRes.data.pages) {
+				console.log(`  ${page.url} - ${page.status}`);
+			}
+		}
+	}
 }

examples/crawl/crawl_with_formats.ts

@@ -4,7 +4,7 @@ import { ScrapeGraphAI } from "scrapegraph-js";
 const sgai = ScrapeGraphAI();
 
 const res = await sgai.crawl.start({
-	url: "https://example.com",
+	url: "https://scrapegraphai.com/",
 	formats: [
 		{ type: "markdown", mode: "reader" },
 		{ type: "screenshot", width: 1280, height: 720 },
@@ -15,10 +15,28 @@ const res = await sgai.crawl.start({
 	excludePatterns: ["/admin/*"],
 });
 
-if (res.status === "success") {
-	console.log("Crawl ID:", res.data?.id);
-	console.log("Status:", res.data?.status);
-	console.log("Total pages to crawl:", res.data?.total);
+if (res.status !== "success" || !res.data) {
+	console.error("Failed to start:", res.error);
 } else {
-	console.error("Failed:", res.error);
+	const crawlId = res.data.id;
+	console.log("Crawl started:", crawlId);
+
+	let status = res.data.status;
+	while (status === "running") {
+		await new Promise((r) => setTimeout(r, 2000));
+		const getRes = await sgai.crawl.get(crawlId);
+		if (getRes.status !== "success" || !getRes.data) {
+			console.error("Failed to get status:", getRes.error);
+			break;
+		}
+		status = getRes.data.status;
+		console.log(`Progress: ${getRes.data.finished}/${getRes.data.total} - ${status}`);
+
+		if (status === "completed" || status === "failed") {
+			console.log("\nPages crawled:");
+			for (const page of getRes.data.pages) {
+				console.log(`  ${page.url} - ${page.status}`);
+			}
+		}
+	}
 }