|
1 | 1 | import { defineCommand } from "citty"; |
2 | | -import * as scrapegraphai from "scrapegraph-js"; |
| 2 | +import { crawl } from "scrapegraph-js"; |
| 3 | +import type { ApiCrawlRequest, ApiScrapeFormatEntry } from "scrapegraph-js"; |
3 | 4 | import { resolveApiKey } from "../lib/folders.js"; |
4 | 5 | import * as log from "../lib/log.js"; |
5 | 6 |
|
| 7 | +const FORMATS = [ |
| 8 | + "markdown", |
| 9 | + "html", |
| 10 | + "screenshot", |
| 11 | + "branding", |
| 12 | + "links", |
| 13 | + "images", |
| 14 | + "summary", |
| 15 | +] as const; |
| 16 | +type Format = (typeof FORMATS)[number]; |
| 17 | + |
| 18 | +const POLL_INTERVAL_MS = 3000; |
| 19 | + |
| 20 | +function buildFormat(f: Format): ApiScrapeFormatEntry { |
| 21 | + if (f === "markdown" || f === "html") return { type: f, mode: "normal" }; |
| 22 | + return { type: f } as ApiScrapeFormatEntry; |
| 23 | +} |
| 24 | + |
6 | 25 | export default defineCommand({ |
7 | 26 | meta: { |
8 | 27 | name: "crawl", |
9 | | - description: "Crawl and extract data from multiple pages", |
| 28 | + description: "Crawl pages starting from a URL", |
10 | 29 | }, |
11 | 30 | args: { |
12 | 31 | url: { |
13 | 32 | type: "positional", |
14 | | - description: "Starting URL to crawl", |
| 33 | + description: "Starting URL", |
15 | 34 | required: true, |
16 | 35 | }, |
17 | | - prompt: { |
| 36 | + format: { |
| 37 | + type: "string", |
| 38 | + alias: "f", |
| 39 | + description: `Per-page format(s), comma-separated: ${FORMATS.join(", ")} (default: markdown)`, |
| 40 | + }, |
| 41 | + "max-pages": { type: "string", description: "Maximum pages to crawl (default 50, max 1000)" }, |
| 42 | + "max-depth": { type: "string", description: "Crawl depth (default 2)" }, |
| 43 | + "max-links-per-page": { type: "string", description: "Max links per page (default 10)" }, |
| 44 | + "allow-external": { type: "boolean", description: "Allow crawling external domains" }, |
| 45 | + "include-patterns": { |
18 | 46 | type: "string", |
19 | | - alias: "p", |
20 | | - description: "Extraction prompt (required when extraction mode is on)", |
| 47 | + description: "JSON array of regex patterns to include", |
21 | 48 | }, |
22 | | - "no-extraction": { |
23 | | - type: "boolean", |
24 | | - description: "Return markdown only (2 credits/page instead of 10)", |
| 49 | + "exclude-patterns": { |
| 50 | + type: "string", |
| 51 | + description: "JSON array of regex patterns to exclude", |
25 | 52 | }, |
26 | | - "max-pages": { type: "string", description: "Maximum pages to crawl (default 10)" }, |
27 | | - depth: { type: "string", description: "Crawl depth (default 1)" }, |
28 | | - schema: { type: "string", description: "Output JSON schema (as JSON string)" }, |
29 | | - rules: { type: "string", description: "Crawl rules as JSON object string" }, |
30 | | - "no-sitemap": { type: "boolean", description: "Disable sitemap-based URL discovery" }, |
31 | | - stealth: { type: "boolean", description: "Bypass bot detection (+4 credits)" }, |
| 53 | + mode: { type: "string", alias: "m", description: "Fetch mode: auto (default), fast, js" }, |
| 54 | + stealth: { type: "boolean", description: "Enable stealth mode" }, |
32 | 55 | json: { type: "boolean", description: "Output raw JSON (pipeable)" }, |
33 | 56 | }, |
34 | 57 | run: async ({ args }) => { |
35 | 58 | const out = log.create(!!args.json); |
36 | | - out.docs("https://docs.scrapegraphai.com/services/smartcrawler"); |
37 | | - const key = await resolveApiKey(!!args.json); |
38 | | - |
39 | | - const base: Record<string, unknown> = { url: args.url }; |
40 | | - if (args["max-pages"]) base.max_pages = Number(args["max-pages"]); |
41 | | - if (args.depth) base.depth = Number(args.depth); |
42 | | - if (args.rules) base.rules = JSON.parse(args.rules); |
43 | | - if (args["no-sitemap"]) base.sitemap = false; |
44 | | - if (args.stealth) base.stealth = true; |
45 | | - |
46 | | - if (args["no-extraction"]) { |
47 | | - base.extraction_mode = false; |
48 | | - } else { |
49 | | - if (args.prompt) base.prompt = args.prompt; |
50 | | - if (args.schema) base.schema = JSON.parse(args.schema); |
| 59 | + out.docs("https://docs.scrapegraphai.com/api-reference/crawl"); |
| 60 | + const apiKey = await resolveApiKey(!!args.json); |
| 61 | + |
| 62 | + const requested = (args.format ?? "markdown") |
| 63 | + .split(",") |
| 64 | + .map((f) => f.trim()) |
| 65 | + .filter(Boolean); |
| 66 | + for (const f of requested) { |
| 67 | + if (!FORMATS.includes(f as Format)) |
| 68 | + out.error(`Unknown format: ${f}. Valid: ${FORMATS.join(", ")}`); |
51 | 69 | } |
| 70 | + const formats = requested.map((f) => buildFormat(f as Format)); |
52 | 71 |
|
53 | | - const params = base as scrapegraphai.CrawlParams; |
| 72 | + const params: ApiCrawlRequest = { url: args.url, formats }; |
| 73 | + const mut = params as Record<string, unknown>; |
| 74 | + if (args["max-pages"]) mut.maxPages = Number(args["max-pages"]); |
| 75 | + if (args["max-depth"]) mut.maxDepth = Number(args["max-depth"]); |
| 76 | + if (args["max-links-per-page"]) mut.maxLinksPerPage = Number(args["max-links-per-page"]); |
| 77 | + if (args["allow-external"]) mut.allowExternal = true; |
| 78 | + if (args["include-patterns"]) mut.includePatterns = JSON.parse(args["include-patterns"]); |
| 79 | + if (args["exclude-patterns"]) mut.excludePatterns = JSON.parse(args["exclude-patterns"]); |
54 | 80 |
|
55 | | - out.start("Crawling"); |
56 | | - const result = await scrapegraphai.crawl(key, params, out.poll); |
57 | | - out.stop(result.elapsedMs); |
| 81 | + const fetchConfig: Record<string, unknown> = {}; |
| 82 | + if (args.mode) fetchConfig.mode = args.mode; |
| 83 | + if (args.stealth) fetchConfig.stealth = true; |
| 84 | + if (Object.keys(fetchConfig).length > 0) mut.fetchConfig = fetchConfig; |
58 | 85 |
|
59 | | - if (result.data) out.result(result.data); |
60 | | - else out.error(result.error); |
| 86 | + out.start("Starting crawl"); |
| 87 | + const job = await crawl.start(apiKey, params); |
| 88 | + if (!job.data) { |
| 89 | + out.error(job.error); |
| 90 | + return; |
| 91 | + } |
| 92 | + const jobId = job.data.id; |
| 93 | + let totalElapsed = job.elapsedMs; |
| 94 | + |
| 95 | + while (true) { |
| 96 | + await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS)); |
| 97 | + const status = await crawl.get(apiKey, jobId); |
| 98 | + totalElapsed += status.elapsedMs; |
| 99 | + if (!status.data) { |
| 100 | + out.error(status.error); |
| 101 | + return; |
| 102 | + } |
| 103 | + out.poll(`${status.data.status} (${status.data.finished}/${status.data.total})`); |
| 104 | + if ( |
| 105 | + status.data.status === "completed" || |
| 106 | + status.data.status === "failed" || |
| 107 | + status.data.status === "deleted" |
| 108 | + ) { |
| 109 | + out.stop(totalElapsed); |
| 110 | + out.result(status.data); |
| 111 | + return; |
| 112 | + } |
| 113 | + } |
61 | 114 | }, |
62 | 115 | }); |
0 commit comments