feat: replace internal API layer with scrapegraph-js SDK

FrancescoSaverioZuppichini · claude · FrancescoSaverioZuppichini · commit e830a1ad6dad · 2026-02-17T15:39:23.000+01:00
Remove zod, schemas, types, and the hand-rolled API client in favor
of the official scrapegraph-js SDK. All commands now import directly
from the package. env.ts bridges JUST_SCRAPE_* vars to SGAI_* so
the SDK picks up CLI-specific config.

Co-Authored-By: Claude Opus 4.6 &lt;noreply@anthropic.com&gt;
diff --git a/README.md b/README.md
@@ -24,13 +24,9 @@ just-scrape/
 ├── src/
 │   ├── cli.ts                       # Entry point, citty main command + subcommands
 │   ├── lib/
-│   │   ├── env.ts                   # Zod-parsed env config (API key, debug, timeout)
+│   │   ├── env.ts                   # Env config (API key, JUST_SCRAPE_* → SGAI_* bridge)
 │   │   ├── folders.ts               # API key resolution + interactive prompt
-│   │   ├── scrapegraphai.ts         # SDK layer — all API functions (typed responses)
-│   │   ├── schemas.ts               # Zod validation schemas
 │   │   └── log.ts                   # Logger factory + syntax-highlighted JSON output
-│   ├── types/
-│   │   └── index.ts                 # Zod-derived types + ApiResult + response types
 │   ├── commands/
 │   │   ├── smart-scraper.ts
 │   │   ├── search-scraper.ts
@@ -45,8 +41,6 @@ just-scrape/
 │   │   └── validate.ts
 │   └── utils/
 │       └── banner.ts                # ASCII banner + version from package.json
-├── tests/
-│   └── scrapegraphai.test.ts        # SDK layer tests (mocked fetch)
 ├── dist/                            # Build output (git-ignored)
 │   └── cli.mjs                      # Bundled ESM with shebang
 ├── package.json
@@ -396,10 +390,9 @@ bun run dev --help
 | CLI Framework | **citty** (unjs) |
 | Prompts | **@clack/prompts** |
 | Styling | **chalk** v5 (ESM) |
-| Validation | **zod** v4 |
+| SDK | **scrapegraph-js** |
 | Env | **dotenv** |
 | Lint / Format | **Biome** |
-| Testing | **Bun test** (built-in) |
 | Target | **Node.js 22+**, ESM-only |
 
 ### Scripts
@@ -409,16 +402,9 @@ bun run dev                          # Run CLI from TS source
 bun run build                        # Bundle ESM to dist/cli.mjs
 bun run lint                         # Lint + format check
 bun run format                       # Auto-format
-bun test                             # Run tests
 bun run check                        # Type-check + lint
 ```
 
-### Testing
-
-Tests mock all API calls via `spyOn(globalThis, "fetch")` — no network, no API key needed.
-
-Covers: success paths, polling, HTTP error mapping (401/402/422/429/500), Zod validation, timeouts, and network failures.
-
 ## License
 
 ISC
diff --git a/bun.lock b/bun.lock
diff --git a/package.json b/package.json
@@ -28,7 +28,7 @@
 		"chalk": "^5.4.1",
 		"citty": "^0.1.6",
 		"dotenv": "^17.2.4",
-		"zod": "^4.3.6"
+		"scrapegraph-js": "^1.0.0"
 	},
 	"devDependencies": {
 		"@biomejs/biome": "^1.9.4",
diff --git a/src/cli.ts b/src/cli.ts
@@ -1,4 +1,5 @@
 import "dotenv/config";
+import "./lib/env.js";
 import { defineCommand, runMain } from "citty";
 import { getVersion, showBanner } from "./utils/banner.js";
 
diff --git a/src/commands/agentic-scraper.ts b/src/commands/agentic-scraper.ts
@@ -1,7 +1,7 @@
 import { defineCommand } from "citty";
+import * as scrapegraphai from "scrapegraph-js";
 import { resolveApiKey } from "../lib/folders.js";
 import * as log from "../lib/log.js";
-import * as scrapegraphai from "../lib/scrapegraphai.js";
 
 export default defineCommand({
 	meta: {
@@ -34,9 +34,8 @@ export default defineCommand({
 		out.docs("https://docs.scrapegraphai.com/services/agenticscraper");
 		const key = await resolveApiKey(!!args.json);
 
-		const params: scrapegraphai.AgenticScraperParams = { url: args.url };
-
-		if (args.steps) params.steps = args.steps.split(",").map((s) => s.trim());
+		const steps = args.steps ? args.steps.split(",").map((s) => s.trim()) : [];
+		const params: scrapegraphai.AgenticScraperParams = { url: args.url, steps };
 		if (args.prompt) params.user_prompt = args.prompt;
 		if (args.schema) params.output_schema = JSON.parse(args.schema);
 		if (args["ai-extraction"]) params.ai_extraction = true;
diff --git a/src/commands/crawl.ts b/src/commands/crawl.ts
@@ -1,7 +1,7 @@
 import { defineCommand } from "citty";
+import * as scrapegraphai from "scrapegraph-js";
 import { resolveApiKey } from "../lib/folders.js";
 import * as log from "../lib/log.js";
-import * as scrapegraphai from "../lib/scrapegraphai.js";
 
 export default defineCommand({
 	meta: {
@@ -36,16 +36,21 @@ export default defineCommand({
 		out.docs("https://docs.scrapegraphai.com/services/smartcrawler");
 		const key = await resolveApiKey(!!args.json);
 
-		const params: scrapegraphai.CrawlParams = { url: args.url };
+		const base: Record<string, unknown> = { url: args.url };
+		if (args["max-pages"]) base.max_pages = Number(args["max-pages"]);
+		if (args.depth) base.depth = Number(args.depth);
+		if (args.rules) base.rules = JSON.parse(args.rules);
+		if (args["no-sitemap"]) base.sitemap = false;
+		if (args.stealth) base.stealth = true;
 
-		if (args.prompt) params.prompt = args.prompt;
-		if (args["no-extraction"]) params.extraction_mode = false;
-		if (args["max-pages"]) params.max_pages = Number(args["max-pages"]);
-		if (args.depth) params.depth = Number(args.depth);
-		if (args.schema) params.schema = JSON.parse(args.schema);
-		if (args.rules) params.rules = JSON.parse(args.rules);
-		if (args["no-sitemap"]) params.sitemap = false;
-		if (args.stealth) params.stealth = true;
+		if (args["no-extraction"]) {
+			base.extraction_mode = false;
+		} else {
+			if (args.prompt) base.prompt = args.prompt;
+			if (args.schema) base.schema = JSON.parse(args.schema);
+		}
+
+		const params = base as scrapegraphai.CrawlParams;
 
 		out.start("Crawling");
 		const result = await scrapegraphai.crawl(key, params, out.poll);
diff --git a/src/commands/credits.ts b/src/commands/credits.ts
@@ -1,7 +1,7 @@
 import { defineCommand } from "citty";
+import * as scrapegraphai from "scrapegraph-js";
 import { resolveApiKey } from "../lib/folders.js";
 import * as log from "../lib/log.js";
-import * as scrapegraphai from "../lib/scrapegraphai.js";
 
 export default defineCommand({
 	meta: {
diff --git a/src/commands/generate-schema.ts b/src/commands/generate-schema.ts
@@ -1,7 +1,7 @@
 import { defineCommand } from "citty";
+import * as scrapegraphai from "scrapegraph-js";
 import { resolveApiKey } from "../lib/folders.js";
 import * as log from "../lib/log.js";
-import * as scrapegraphai from "../lib/scrapegraphai.js";
 
 export default defineCommand({
 	meta: {
diff --git a/src/commands/history.ts b/src/commands/history.ts
@@ -1,10 +1,10 @@
 import * as p from "@clack/prompts";
 import chalk from "chalk";
 import { defineCommand } from "citty";
+import { HISTORY_SERVICES } from "scrapegraph-js";
+import * as scrapegraphai from "scrapegraph-js";
 import { resolveApiKey } from "../lib/folders.js";
 import * as log from "../lib/log.js";
-import { HISTORY_SERVICES } from "../lib/schemas.js";
-import * as scrapegraphai from "../lib/scrapegraphai.js";
 
 const VALID = HISTORY_SERVICES.join(", ");
 const LOAD_MORE = "__load_more__";
diff --git a/src/commands/markdownify.ts b/src/commands/markdownify.ts
@@ -1,7 +1,7 @@
 import { defineCommand } from "citty";
+import * as scrapegraphai from "scrapegraph-js";
 import { resolveApiKey } from "../lib/folders.js";
 import * as log from "../lib/log.js";
-import * as scrapegraphai from "../lib/scrapegraphai.js";
 
 export default defineCommand({
 	meta: {
diff --git a/src/commands/scrape.ts b/src/commands/scrape.ts
@@ -1,7 +1,7 @@
 import { defineCommand } from "citty";
+import * as scrapegraphai from "scrapegraph-js";
 import { resolveApiKey } from "../lib/folders.js";
 import * as log from "../lib/log.js";
-import * as scrapegraphai from "../lib/scrapegraphai.js";
 
 export default defineCommand({
 	meta: {
diff --git a/src/commands/search-scraper.ts b/src/commands/search-scraper.ts
@@ -1,7 +1,7 @@
 import { defineCommand } from "citty";
+import * as scrapegraphai from "scrapegraph-js";
 import { resolveApiKey } from "../lib/folders.js";
 import * as log from "../lib/log.js";
-import * as scrapegraphai from "../lib/scrapegraphai.js";
 
 export default defineCommand({
 	meta: {
diff --git a/src/commands/sitemap.ts b/src/commands/sitemap.ts
@@ -1,7 +1,7 @@
 import { defineCommand } from "citty";
+import * as scrapegraphai from "scrapegraph-js";
 import { resolveApiKey } from "../lib/folders.js";
 import * as log from "../lib/log.js";
-import * as scrapegraphai from "../lib/scrapegraphai.js";
 
 export default defineCommand({
 	meta: {
diff --git a/src/commands/smart-scraper.ts b/src/commands/smart-scraper.ts
@@ -1,7 +1,7 @@
 import { defineCommand } from "citty";
+import * as scrapegraphai from "scrapegraph-js";
 import { resolveApiKey } from "../lib/folders.js";
 import * as log from "../lib/log.js";
-import * as scrapegraphai from "../lib/scrapegraphai.js";
 
 export default defineCommand({
 	meta: {
diff --git a/src/commands/validate.ts b/src/commands/validate.ts
@@ -1,7 +1,7 @@
 import { defineCommand } from "citty";
+import * as scrapegraphai from "scrapegraph-js";
 import { resolveApiKey } from "../lib/folders.js";
 import * as log from "../lib/log.js";
-import * as scrapegraphai from "../lib/scrapegraphai.js";
 
 export default defineCommand({
 	meta: {
diff --git a/src/lib/env.ts b/src/lib/env.ts
@@ -1,11 +1,18 @@
 import { existsSync, readFileSync } from "node:fs";
 import { homedir } from "node:os";
 import { join } from "node:path";
-import { z } from "zod";
 
 export const CONFIG_DIR = join(homedir(), ".scrapegraphai");
 export const CONFIG_PATH = join(CONFIG_DIR, "config.json");
 
+if (process.env.JUST_SCRAPE_API_URL && !process.env.SGAI_API_URL)
+	process.env.SGAI_API_URL = process.env.JUST_SCRAPE_API_URL;
+
+if (process.env.JUST_SCRAPE_DEBUG === "1" && !process.env.SGAI_DEBUG) process.env.SGAI_DEBUG = "1";
+
+if (process.env.JUST_SCRAPE_TIMEOUT_S && !process.env.SGAI_TIMEOUT_S)
+	process.env.SGAI_TIMEOUT_S = process.env.JUST_SCRAPE_TIMEOUT_S;
+
 function loadConfigFile(): Record<string, unknown> {
 	if (!existsSync(CONFIG_PATH)) return {};
 	try {
@@ -15,24 +22,15 @@ function loadConfigFile(): Record<string, unknown> {
 	}
 }
 
-const EnvSchema = z.object({
-	apiKey: z.string().optional(),
-	debug: z.boolean().default(false),
-	timeoutS: z.number().int().positive().default(120),
-});
-
-export type Env = z.infer<typeof EnvSchema>;
+export type Env = {
+	apiKey?: string;
+};
 
 function resolve(): Env {
 	const config = loadConfigFile();
-
-	return EnvSchema.parse({
+	return {
 		apiKey: process.env.SGAI_API_KEY || (config["api-key"] as string) || undefined,
-		debug: process.env.JUST_SCRAPE_DEBUG === "1",
-		timeoutS: process.env.JUST_SCRAPE_TIMEOUT_S
-			? Number(process.env.JUST_SCRAPE_TIMEOUT_S)
-			: 60 * 2,
-	});
+	};
 }
 
 export const env = resolve();
diff --git a/src/lib/schemas.ts b/src/lib/schemas.ts
diff --git a/src/lib/scrapegraphai.ts b/src/lib/scrapegraphai.ts
diff --git a/src/types/index.ts b/src/types/index.ts
diff --git a/tests/scrapegraphai.test.ts b/tests/scrapegraphai.test.ts
diff --git a/tests/smoke.test.ts b/tests/smoke.test.ts

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`import "dotenv/config";`
	`2`	`+import "./lib/env.js";`
`2`	`3`	`import { defineCommand, runMain } from "citty";`
`3`	`4`	`import { getVersion, showBanner } from "./utils/banner.js";`
`4`	`5`