Skip to content

Commit 1d16745

Browse files
VinciGit00claude
andcommitted
feat: sync CLI with scrapegraph-js v2 PR #13 head (096c110)
- bump scrapegraph-js to latest PR HEAD (adds monitor.activity, renames SGAI_TIMEOUT_S → SGAI_TIMEOUT, bakes /api/v2 into default base URL) - add `just-scrape monitor activity --id <id> [--limit] [--cursor]` for paginated tick history - bridge legacy SGAI_TIMEOUT_S (and JUST_SCRAPE_TIMEOUT_S) to new SGAI_TIMEOUT - README: document activity command, update default base URL, note SGAI_DEBUG - bump CLI to v1.0.0 to track SDK v2.0.0 Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent d5211ae commit 1d16745

7 files changed

Lines changed: 76 additions & 18 deletions

File tree

README.md

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -78,10 +78,11 @@ Four ways to provide it (checked in order):
7878
| Variable | Description | Default |
7979
|---|---|---|
8080
| `SGAI_API_KEY` | ScrapeGraph API key ||
81-
| `SGAI_API_URL` | Override API base URL | `https://api.scrapegraphai.com` |
82-
| `SGAI_TIMEOUT_S` | Request timeout in seconds | `30` |
81+
| `SGAI_API_URL` | Override API base URL | `https://api.scrapegraphai.com/api/v2` |
82+
| `SGAI_TIMEOUT` | Request timeout in seconds | `120` |
83+
| `SGAI_DEBUG` | Set to `1` to log requests/responses ||
8384

84-
Legacy variables (`JUST_SCRAPE_API_URL`, `JUST_SCRAPE_TIMEOUT_S`, `JUST_SCRAPE_DEBUG`) are still bridged.
85+
Legacy variables are still bridged transparently: `JUST_SCRAPE_API_URL``SGAI_API_URL`, `JUST_SCRAPE_TIMEOUT_S` / `SGAI_TIMEOUT_S``SGAI_TIMEOUT`, `JUST_SCRAPE_DEBUG``SGAI_DEBUG`.
8586

8687
## JSON Mode (`--json`)
8788

@@ -272,6 +273,9 @@ just-scrape monitor update --id <id> --interval 2h # Update interval
272273
just-scrape monitor pause --id <id> # Pause a monitor
273274
just-scrape monitor resume --id <id> # Resume a paused monitor
274275
just-scrape monitor delete --id <id> # Delete a monitor
276+
just-scrape monitor activity --id <id> # Paginated tick history
277+
just-scrape monitor activity --id <id> --limit 50 # Ticks per page (max 100)
278+
just-scrape monitor activity --id <id> --cursor <cursor> # Paginate with a cursor
275279
```
276280

277281
### Examples
@@ -294,6 +298,10 @@ just-scrape monitor list
294298
# Pause and resume
295299
just-scrape monitor pause --id abc123
296300
just-scrape monitor resume --id abc123
301+
302+
# Inspect recent ticks (checks the monitor performed) with their diffs
303+
just-scrape monitor activity --id abc123 --limit 20
304+
just-scrape monitor activity --id abc123 --json | jq '.ticks[] | select(.hasChanges == true)'
297305
```
298306

299307
## History

bun.lock

Lines changed: 2 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

package.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "just-scrape",
3-
"version": "0.3.0",
3+
"version": "1.0.0",
44
"description": "ScrapeGraph AI CLI tool",
55
"type": "module",
66
"main": "dist/cli.mjs",
@@ -28,7 +28,7 @@
2828
"chalk": "^5.4.1",
2929
"citty": "^0.1.6",
3030
"dotenv": "^17.2.4",
31-
"scrapegraph-js": "github:ScrapeGraphAI/scrapegraph-js#0738786"
31+
"scrapegraph-js": "github:ScrapeGraphAI/scrapegraph-js#096c110"
3232
},
3333
"devDependencies": {
3434
"@biomejs/biome": "^1.9.4",

src/commands/crawl.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,8 @@ export default defineCommand({
4545
.filter(Boolean);
4646

4747
const formats = requestedFormats.map((f) => {
48-
if (f === "markdown" || f === "html") return { type: f as "markdown" | "html", mode: "normal" as const };
48+
if (f === "markdown" || f === "html")
49+
return { type: f as "markdown" | "html", mode: "normal" as const };
4950
return { type: f };
5051
});
5152

src/commands/history.ts

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,7 @@ function label(row: HistoryRow): string {
2525
const urlShort = url.length > 50 ? `${url.slice(0, 49)}...` : url;
2626

2727
const color =
28-
status === "completed"
29-
? chalk.green
30-
: status === "failed"
31-
? chalk.red
32-
: chalk.yellow;
28+
status === "completed" ? chalk.green : status === "failed" ? chalk.red : chalk.yellow;
3329

3430
return `${chalk.dim(short)} ${color(status)} ${urlShort}`;
3531
}

src/commands/monitor.ts

Lines changed: 51 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,28 @@ import { monitor } from "scrapegraph-js";
55
import { getApiKey } from "../lib/client.js";
66
import * as log from "../lib/log.js";
77

8-
const ACTIONS = ["create", "list", "get", "update", "delete", "pause", "resume"] as const;
8+
const ACTIONS = [
9+
"create",
10+
"list",
11+
"get",
12+
"update",
13+
"delete",
14+
"pause",
15+
"resume",
16+
"activity",
17+
] as const;
918
type Action = (typeof ACTIONS)[number];
1019

11-
const FORMATS = ["markdown", "html", "screenshot", "branding", "links", "images", "summary", "json"] as const;
20+
const FORMATS = [
21+
"markdown",
22+
"html",
23+
"screenshot",
24+
"branding",
25+
"links",
26+
"images",
27+
"summary",
28+
"json",
29+
] as const;
1230

1331
export default defineCommand({
1432
meta: {
@@ -52,6 +70,14 @@ export default defineCommand({
5270
description: "Fetch mode: auto (default), fast, js",
5371
},
5472
stealth: { type: "boolean", description: "Enable stealth mode" },
73+
limit: {
74+
type: "string",
75+
description: "Ticks per page for activity (max 100)",
76+
},
77+
cursor: {
78+
type: "string",
79+
description: "Pagination cursor for activity",
80+
},
5581
json: { type: "boolean", description: "Output raw JSON (pipeable)" },
5682
},
5783
run: async ({ args }) => {
@@ -76,7 +102,8 @@ export default defineCommand({
76102
.filter(Boolean);
77103

78104
const formats = requestedFormats.map((f) => {
79-
if (f === "markdown" || f === "html") return { type: f as "markdown" | "html", mode: "normal" as const };
105+
if (f === "markdown" || f === "html")
106+
return { type: f as "markdown" | "html", mode: "normal" as const };
80107
return { type: f };
81108
});
82109

@@ -236,6 +263,27 @@ export default defineCommand({
236263
break;
237264
}
238265

266+
case "activity": {
267+
if (!args.id) {
268+
out.error("--id is required for activity");
269+
return;
270+
}
271+
const params: { limit?: number; cursor?: string } = {};
272+
if (args.limit) params.limit = Number(args.limit);
273+
if (args.cursor) params.cursor = args.cursor;
274+
275+
out.start("Fetching monitor activity");
276+
try {
277+
const result = await monitor.activity(apiKey, args.id, params);
278+
out.stop(result.elapsedMs);
279+
out.result(result.data);
280+
} catch (err) {
281+
out.stop(0);
282+
out.error(err instanceof Error ? err.message : String(err));
283+
}
284+
break;
285+
}
286+
239287
default:
240288
out.error(`Unknown action: ${action}. Valid: ${ACTIONS.join(", ")}`);
241289
}

src/lib/env.ts

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,13 @@ if (process.env.JUST_SCRAPE_API_URL && !process.env.SGAI_API_URL)
1010

1111
if (process.env.JUST_SCRAPE_DEBUG === "1" && !process.env.SGAI_DEBUG) process.env.SGAI_DEBUG = "1";
1212

13-
if (process.env.JUST_SCRAPE_TIMEOUT_S && !process.env.SGAI_TIMEOUT_S)
14-
process.env.SGAI_TIMEOUT_S = process.env.JUST_SCRAPE_TIMEOUT_S;
13+
// Bridge legacy JUST_SCRAPE_TIMEOUT_S and SGAI_TIMEOUT_S to the new SGAI_TIMEOUT var
14+
// (renamed in scrapegraph-js v2; see scrapegraph-js PR #13 / commit 2eba148).
15+
if (process.env.JUST_SCRAPE_TIMEOUT_S && !process.env.SGAI_TIMEOUT)
16+
process.env.SGAI_TIMEOUT = process.env.JUST_SCRAPE_TIMEOUT_S;
17+
18+
if (process.env.SGAI_TIMEOUT_S && !process.env.SGAI_TIMEOUT)
19+
process.env.SGAI_TIMEOUT = process.env.SGAI_TIMEOUT_S;
1520

1621
function loadConfigFile(): Record<string, unknown> {
1722
if (!existsSync(CONFIG_PATH)) return {};

0 commit comments

Comments
 (0)