Merge pull request #11 from ScrapeGraphAI/feat/typed-api-responses

feat: migrate to scrapegraph-js SDK

Author: VinciGit00

README.md

@@ -24,13 +24,9 @@ just-scrape/
 ├── src/
 │   ├── cli.ts                       # Entry point, citty main command + subcommands
 │   ├── lib/
-│   │   ├── env.ts                   # Zod-parsed env config (API key, debug, timeout)
+│   │   ├── env.ts                   # Env config (API key, JUST_SCRAPE_* → SGAI_* bridge)
 │   │   ├── folders.ts               # API key resolution + interactive prompt
-│   │   ├── scrapegraphai.ts         # SDK layer — all API functions (typed responses)
-│   │   ├── schemas.ts               # Zod validation schemas
 │   │   └── log.ts                   # Logger factory + syntax-highlighted JSON output
-│   ├── types/
-│   │   └── index.ts                 # Zod-derived types + ApiResult + response types
 │   ├── commands/
 │   │   ├── smart-scraper.ts
 │   │   ├── search-scraper.ts
@@ -45,8 +41,6 @@ just-scrape/
 │   │   └── validate.ts
 │   └── utils/
 │       └── banner.ts                # ASCII banner + version from package.json
-├── tests/
-│   └── scrapegraphai.test.ts        # SDK layer tests (mocked fetch)
 ├── dist/                            # Build output (git-ignored)
 │   └── cli.mjs                      # Bundled ESM with shebang
 ├── package.json
@@ -64,6 +58,7 @@ pnpm add -g just-scrape              # pnpm
 yarn global add just-scrape           # yarn
 bun add -g just-scrape               # bun
 npx just-scrape --help               # or run without installing
+bunx just-scrape --help              # bun equivalent
 ```
 
 Package: [just-scrape](https://www.npmjs.com/package/just-scrape) on npm.
@@ -397,10 +392,9 @@ bun run dev --help
 | CLI Framework | **citty** (unjs) |
 | Prompts | **@clack/prompts** |
 | Styling | **chalk** v5 (ESM) |
-| Validation | **zod** v4 |
+| SDK | **scrapegraph-js** |
 | Env | **dotenv** |
 | Lint / Format | **Biome** |
-| Testing | **Bun test** (built-in) |
 | Target | **Node.js 22+**, ESM-only |
 
 ### Scripts
@@ -410,16 +404,9 @@ bun run dev                          # Run CLI from TS source
 bun run build                        # Bundle ESM to dist/cli.mjs
 bun run lint                         # Lint + format check
 bun run format                       # Auto-format
-bun test                             # Run tests
 bun run check                        # Type-check + lint
 ```
 
-### Testing
-
-Tests mock all API calls via `spyOn(globalThis, "fetch")` — no network, no API key needed.
-
-Covers: success paths, polling, HTTP error mapping (401/402/422/429/500), Zod validation, timeouts, and network failures.
-
 ## License
 
 ISC

bun.lock

@@ -1,6 +1,6 @@
 {
 	"name": "just-scrape",
-	"version": "0.1.9",
+	"version": "0.2.0",
 	"description": "ScrapeGraph AI CLI tool",
 	"type": "module",
 	"main": "dist/cli.mjs",
@@ -28,7 +28,7 @@
 		"chalk": "^5.4.1",
 		"citty": "^0.1.6",
 		"dotenv": "^17.2.4",
-		"zod": "^4.3.6"
+		"scrapegraph-js": "^1.0.0"
 	},
 	"devDependencies": {
 		"@biomejs/biome": "^1.9.4",

package.json

@@ -1,4 +1,5 @@
 import "dotenv/config";
+import "./lib/env.js";
 import { defineCommand, runMain } from "citty";
 import { getVersion, showBanner } from "./utils/banner.js";
 

src/cli.ts

@@ -1,7 +1,7 @@
 import { defineCommand } from "citty";
+import * as scrapegraphai from "scrapegraph-js";
 import { resolveApiKey } from "../lib/folders.js";
 import * as log from "../lib/log.js";
-import * as scrapegraphai from "../lib/scrapegraphai.js";
 
 export default defineCommand({
 	meta: {
@@ -34,9 +34,8 @@ export default defineCommand({
 		out.docs("https://docs.scrapegraphai.com/services/agenticscraper");
 		const key = await resolveApiKey(!!args.json);
 
-		const params: scrapegraphai.AgenticScraperParams = { url: args.url };
-
-		if (args.steps) params.steps = args.steps.split(",").map((s) => s.trim());
+		const steps = args.steps ? args.steps.split(",").map((s) => s.trim()) : [];
+		const params: scrapegraphai.AgenticScraperParams = { url: args.url, steps };
 		if (args.prompt) params.user_prompt = args.prompt;
 		if (args.schema) params.output_schema = JSON.parse(args.schema);
 		if (args["ai-extraction"]) params.ai_extraction = true;

src/commands/agentic-scraper.ts

@@ -1,7 +1,7 @@
 import { defineCommand } from "citty";
+import * as scrapegraphai from "scrapegraph-js";
 import { resolveApiKey } from "../lib/folders.js";
 import * as log from "../lib/log.js";
-import * as scrapegraphai from "../lib/scrapegraphai.js";
 
 export default defineCommand({
 	meta: {
@@ -36,16 +36,21 @@ export default defineCommand({
 		out.docs("https://docs.scrapegraphai.com/services/smartcrawler");
 		const key = await resolveApiKey(!!args.json);
 
-		const params: scrapegraphai.CrawlParams = { url: args.url };
+		const base: Record<string, unknown> = { url: args.url };
+		if (args["max-pages"]) base.max_pages = Number(args["max-pages"]);
+		if (args.depth) base.depth = Number(args.depth);
+		if (args.rules) base.rules = JSON.parse(args.rules);
+		if (args["no-sitemap"]) base.sitemap = false;
+		if (args.stealth) base.stealth = true;
 
-		if (args.prompt) params.prompt = args.prompt;
-		if (args["no-extraction"]) params.extraction_mode = false;
-		if (args["max-pages"]) params.max_pages = Number(args["max-pages"]);
-		if (args.depth) params.depth = Number(args.depth);
-		if (args.schema) params.schema = JSON.parse(args.schema);
-		if (args.rules) params.rules = JSON.parse(args.rules);
-		if (args["no-sitemap"]) params.sitemap = false;
-		if (args.stealth) params.stealth = true;
+		if (args["no-extraction"]) {
+			base.extraction_mode = false;
+		} else {
+			if (args.prompt) base.prompt = args.prompt;
+			if (args.schema) base.schema = JSON.parse(args.schema);
+		}
+
+		const params = base as scrapegraphai.CrawlParams;
 
 		out.start("Crawling");
 		const result = await scrapegraphai.crawl(key, params, out.poll);

src/commands/crawl.ts

@@ -1,7 +1,7 @@
 import { defineCommand } from "citty";
+import * as scrapegraphai from "scrapegraph-js";
 import { resolveApiKey } from "../lib/folders.js";
 import * as log from "../lib/log.js";
-import * as scrapegraphai from "../lib/scrapegraphai.js";
 
 export default defineCommand({
 	meta: {

src/commands/credits.ts

@@ -1,7 +1,7 @@
 import { defineCommand } from "citty";
+import * as scrapegraphai from "scrapegraph-js";
 import { resolveApiKey } from "../lib/folders.js";
 import * as log from "../lib/log.js";
-import * as scrapegraphai from "../lib/scrapegraphai.js";
 
 export default defineCommand({
 	meta: {

src/commands/generate-schema.ts

@@ -1,10 +1,10 @@
 import * as p from "@clack/prompts";
 import chalk from "chalk";
 import { defineCommand } from "citty";
+import { HISTORY_SERVICES } from "scrapegraph-js";
+import * as scrapegraphai from "scrapegraph-js";
 import { resolveApiKey } from "../lib/folders.js";
 import * as log from "../lib/log.js";
-import { HISTORY_SERVICES } from "../lib/schemas.js";
-import * as scrapegraphai from "../lib/scrapegraphai.js";
 
 const VALID = HISTORY_SERVICES.join(", ");
 const LOAD_MORE = "__load_more__";

src/commands/history.ts

@@ -1,7 +1,7 @@
 import { defineCommand } from "citty";
+import * as scrapegraphai from "scrapegraph-js";
 import { resolveApiKey } from "../lib/folders.js";
 import * as log from "../lib/log.js";
-import * as scrapegraphai from "../lib/scrapegraphai.js";
 
 export default defineCommand({
 	meta: {