Skip to content

Commit 9ac576a

Browse files
FrancescoSaverioZuppichiniFrancescoSaverioZuppichini
committed
feat: add AI SDK tools and crawl pages support
1 parent aeb2ea8 commit 9ac576a

16 files changed

Lines changed: 1808 additions & 558 deletions

‎README.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,12 @@ const start = await sgai.crawl.start({
140140
// Check status
141141
const status = await sgai.crawl.get(start.data?.id!);
142142

143+
// Fetch paginated pages with resolved scrape results
144+
const pages = await sgai.crawl.pages(start.data?.id!, {
145+
cursor: 0,
146+
limit: 50,
147+
});
148+
143149
// Control
144150
await sgai.crawl.stop(id);
145151
await sgai.crawl.resume(id);

‎package.json‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,17 @@
11
{
22
"name": "scrapegraph-js",
3-
"version": "2.1.0",
3+
"version": "2.2.0",
44
"description": "Official JavaScript/TypeScript SDK for the ScrapeGraph AI API — smart web scraping powered by AI",
55
"type": "module",
66
"main": "dist/index.js",
77
"types": "dist/index.d.ts",
88
"exports": {
99
".": {
10-
"import": "./dist/index.js",
11-
"types": "./dist/index.d.ts"
10+
"types": "./dist/index.d.ts",
11+
"import": "./dist/index.js"
1212
}
1313
},
14+
"workspaces": ["packages/*"],
1415
"scripts": {
1516
"dev": "tsup --watch",
1617
"build": "tsup",

‎packages/ai-sdk/README.md‎

Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
# ScrapeGraphAI AI SDK Tools
2+
3+
[![npm version](https://badge.fury.io/js/%40scrapegraphai%2Fai-sdk.svg)](https://www.npmjs.com/package/@scrapegraphai/ai-sdk)
4+
[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
5+
6+
<p align="center">
7+
<a href="https://scrapegraphai.com">
8+
<img src="../../media/banner.png" alt="ScrapeGraphAI AI SDK Tools" style="width: 100%;">
9+
</a>
10+
</p>
11+
12+
Vercel [AI SDK](https://ai-sdk.dev/docs/introduction) tools for the ScrapeGraphAI API.
13+
14+
## Install
15+
16+
```bash
17+
npm i @scrapegraphai/ai-sdk ai
18+
# or
19+
bun add @scrapegraphai/ai-sdk ai
20+
```
21+
22+
`ai` is a peer dependency. Install the model provider package you use, for example:
23+
24+
```bash
25+
npm i @ai-sdk/openai
26+
# or
27+
bun add @ai-sdk/openai
28+
```
29+
30+
## Quick Start
31+
32+
Minimal scrape-only setup:
33+
34+
```ts
35+
import { openai } from "@ai-sdk/openai";
36+
import { generateText, stepCountIs } from "ai";
37+
import { scrapeTool } from "@scrapegraphai/ai-sdk";
38+
39+
const result = await generateText({
40+
model: openai("gpt-5-nano"),
41+
prompt: "Find the main headline on https://example.com",
42+
tools: {
43+
scrape: scrapeTool(),
44+
},
45+
stopWhen: stepCountIs(5),
46+
});
47+
48+
console.log(result.text);
49+
```
50+
51+
Use every ScrapeGraphAI tool group:
52+
53+
```ts
54+
import { openai } from "@ai-sdk/openai";
55+
import { generateText, stepCountIs } from "ai";
56+
import {
57+
crawlTools,
58+
extractTool,
59+
monitorTools,
60+
scrapeTool,
61+
searchTool,
62+
} from "@scrapegraphai/ai-sdk";
63+
64+
const result = await generateText({
65+
model: openai("gpt-5-nano"),
66+
prompt: "Search for ScrapeGraphAI docs, scrape the best page, and summarize it.",
67+
tools: {
68+
scrape: scrapeTool(),
69+
extract: extractTool(),
70+
search: searchTool(),
71+
...crawlTools(),
72+
...monitorTools(),
73+
},
74+
stopWhen: stepCountIs(10),
75+
});
76+
77+
console.log(result.text);
78+
```
79+
80+
Tools read `SGAI_API_KEY` from the environment by default. You can also pass it explicitly:
81+
82+
```ts
83+
const tools = {
84+
scrape: scrapeTool({ apiKey: process.env.SGAI_API_KEY }),
85+
};
86+
```
87+
88+
## Tools
89+
90+
### scrapeTool
91+
92+
Scrape a webpage with ScrapeGraphAI. Supports markdown, html, json extraction, links, images, summary, branding, and screenshots.
93+
94+
```ts
95+
import { scrapeTool } from "@scrapegraphai/ai-sdk";
96+
97+
const tools = {
98+
scrape: scrapeTool(),
99+
};
100+
```
101+
102+
### extractTool
103+
104+
Extract structured JSON from a URL, HTML, or markdown with a natural-language prompt.
105+
106+
```ts
107+
import { extractTool } from "@scrapegraphai/ai-sdk";
108+
109+
const tools = {
110+
extract: extractTool(),
111+
};
112+
```
113+
114+
### searchTool
115+
116+
Search the web and optionally extract structured data from search results.
117+
118+
```ts
119+
import { searchTool } from "@scrapegraphai/ai-sdk";
120+
121+
const tools = {
122+
search: searchTool(),
123+
};
124+
```
125+
126+
### crawlTools
127+
128+
Start, poll, page through, stop, resume, and delete ScrapeGraphAI crawl jobs.
129+
130+
```ts
131+
import { crawlTools } from "@scrapegraphai/ai-sdk";
132+
133+
const tools = {
134+
...crawlTools(),
135+
};
136+
```
137+
138+
Crawl page retrieval is paginated. Use `getCrawl` for status, then `getCrawlPages` for pages and resolved scrape results.
139+
140+
```ts
141+
const tools = {
142+
startCrawl: startCrawlTool(),
143+
getCrawl: getCrawlTool(),
144+
getCrawlPages: getCrawlPagesTool(),
145+
};
146+
```
147+
148+
### monitorTools
149+
150+
Create, list, update, pause, resume, delete, and fetch activity for ScrapeGraphAI monitors.
151+
152+
```ts
153+
import { monitorTools } from "@scrapegraphai/ai-sdk";
154+
155+
const tools = {
156+
...monitorTools(),
157+
};
158+
```
159+
160+
## Examples
161+
162+
| Example | Description |
163+
|---------|-------------|
164+
| [`hacker-news.ts`](examples/hacker-news.ts) | Scrape Hacker News with AI SDK tools |
165+
| [`crawl-blog.ts`](examples/crawl-blog.ts) | Crawl ScrapeGraphAI blog pages, fetch paginated crawl results, and summarize them |
166+
167+
Run an example:
168+
169+
```bash
170+
OPENAI_API_KEY=... SGAI_API_KEY=... bun examples/crawl-blog.ts
171+
```
172+
173+
## Environment Variables
174+
175+
| Variable | Description |
176+
|----------|-------------|
177+
| `SGAI_API_KEY` | ScrapeGraphAI API key |
178+
| `OPENAI_API_KEY` | Required by the OpenAI provider examples |
179+
180+
## Development
181+
182+
```bash
183+
bun install
184+
bun run build
185+
bun run check
186+
```
187+
188+
## License
189+
190+
MIT - [ScrapeGraphAI](https://scrapegraphai.com)

‎packages/ai-sdk/examples/crawl-blog.ts‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
import { openai } from "@ai-sdk/openai";
2+
import { generateText, stepCountIs, type ModelMessage } from "ai";
3+
import { stdin as input, stdout as output } from "node:process";
4+
import { createInterface } from "node:readline/promises";
5+
import { crawlTools } from "../src/index";
6+
7+
const initialPrompt =
8+
"Find 10 https://scrapegraphai.com/ blog posts. Start a crawl, poll its status, fetch crawled pages with getCrawlPages, then summarize what you found.";
9+
const messages: ModelMessage[] = [];
10+
let activeController: AbortController | undefined;
11+
12+
async function run(prompt: string) {
13+
messages.push({ role: "user", content: prompt });
14+
const controller = new AbortController();
15+
activeController = controller;
16+
17+
try {
18+
const result = await generateText({
19+
model: openai("gpt-5-nano"),
20+
messages,
21+
tools: { ...crawlTools() },
22+
stopWhen: stepCountIs(20),
23+
abortSignal: controller.signal,
24+
onStepFinish: ({ text, toolCalls, toolResults }) => {
25+
if (text) {
26+
console.log(`\n[assistant]\n${text}`);
27+
}
28+
29+
for (const toolCall of toolCalls) {
30+
console.log(`\n[tool] ${toolCall.toolName}`);
31+
console.log(JSON.stringify(toolCall.input, null, 2));
32+
}
33+
34+
for (const toolResult of toolResults) {
35+
console.log(`\n[result] ${toolResult.toolName}`);
36+
console.log(JSON.stringify(toolResult.output, null, 2));
37+
}
38+
},
39+
});
40+
41+
messages.push(...result.response.messages);
42+
console.log(`\n${result.text}\n`);
43+
} catch (error) {
44+
if (controller.signal.aborted) {
45+
console.error("[aborted]");
46+
} else {
47+
console.error(error instanceof Error ? error.message : error);
48+
}
49+
} finally {
50+
if (activeController === controller) {
51+
activeController = undefined;
52+
}
53+
}
54+
}
55+
56+
const rl = createInterface({ input, output });
57+
58+
process.on("SIGINT", () => {
59+
output.write("\n");
60+
if (activeController) {
61+
activeController.abort();
62+
return;
63+
}
64+
65+
rl.close();
66+
process.exit(0);
67+
});
68+
69+
await run(initialPrompt);
70+
71+
while (true) {
72+
const prompt = (await rl.question("> ")).trim();
73+
74+
if (prompt) {
75+
await run(prompt);
76+
}
77+
}

‎packages/ai-sdk/examples/hacker-news.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
import { openai } from "@ai-sdk/openai";
2+
import { generateText, stepCountIs } from "ai";
3+
import { scrapeTool } from "../src/index";
4+
5+
const { text } = await generateText({
6+
model: openai("gpt-5-nano"),
7+
prompt:
8+
"Scrape Hacker News and write a short, concise summary of what people are talking about today.",
9+
tools: {
10+
scrape: scrapeTool(),
11+
},
12+
stopWhen: stepCountIs(3),
13+
});
14+
15+
console.log(text);

‎packages/ai-sdk/package.json‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
{
2+
"name": "@scrapegraphai/ai-sdk",
3+
"version": "0.1.0",
4+
"description": "Vercel AI SDK tools integration for ScrapeGraphAI.",
5+
"type": "module",
6+
"main": "dist/index.js",
7+
"types": "dist/index.d.ts",
8+
"exports": {
9+
".": {
10+
"types": "./dist/index.d.ts",
11+
"import": "./dist/index.js"
12+
}
13+
},
14+
"files": ["dist"],
15+
"scripts": {
16+
"dev": "tsup --watch",
17+
"build": "tsup",
18+
"check": "tsc --noEmit",
19+
"prepublishOnly": "bun run build"
20+
},
21+
"keywords": ["scrapegraph", "ai-sdk", "tools", "scraping", "extraction"],
22+
"author": "ScrapeGraph Team",
23+
"license": "MIT",
24+
"homepage": "https://scrapegraphai.com",
25+
"repository": {
26+
"type": "git",
27+
"url": "https://github.com/ScrapeGraphAI/scrapegraph-sdk.git",
28+
"directory": "packages/ai-sdk"
29+
},
30+
"peerDependencies": {
31+
"ai": ">=6"
32+
},
33+
"dependencies": {
34+
"scrapegraph-js": "^2.2.0",
35+
"zod": "^4.3.6"
36+
},
37+
"devDependencies": {
38+
"@ai-sdk/openai": "^3.0.65",
39+
"ai": "^6.0.191",
40+
"tsup": "^8.3.6",
41+
"typescript": "^5.8.2"
42+
}
43+
}

0 commit comments

Comments
 (0)