feat: add crawl pages support

Author: FrancescoSaverioZuppichini

README.md

@@ -134,14 +134,14 @@ res = sgai.search(
 Crawl a website and its linked pages.
 
 ```python
-from scrapegraph_py import ScrapeGraphAI, MarkdownFormatConfig
+from scrapegraph_py import ScrapeGraphAI, ScrapeMarkdownFormatEntry
 
 sgai = ScrapeGraphAI()
 
 # Start a crawl
 start = sgai.crawl.start(
     "https://example.com",
-    formats=[MarkdownFormatConfig()],
+    formats=[ScrapeMarkdownFormatEntry()],
     max_pages=50,
     max_depth=2,
     max_links_per_page=10,
@@ -151,6 +151,7 @@ start = sgai.crawl.start(
 
 # Check status
 status = sgai.crawl.get(start.data["id"])
+pages = sgai.crawl.pages(start.data["id"], cursor=0, limit=50)
 
 # Control
 sgai.crawl.stop(crawl_id)
@@ -259,6 +260,7 @@ async with AsyncScrapeGraphAI() as sgai:
 async with AsyncScrapeGraphAI() as sgai:
     start = await sgai.crawl.start("https://example.com", max_pages=50)
     status = await sgai.crawl.get(start.data["id"])
+    pages = await sgai.crawl.pages(start.data["id"], cursor=0, limit=50)
 ```
 
 ### Async Monitor

src/scrapegraph_py/__init__.py

@@ -4,12 +4,15 @@
     ApiResult,
     BrandingFormatConfig,
     CrawlPage,
+    CrawlPagesQuery,
+    CrawlPagesResponse,
     CrawlPageStatus,
     CrawlRequest,
     CrawlResponse,
     CrawlStatus,
     CreditsResponse,
     ExtractRequest,
+    ExtractRequestBase,
     ExtractResponse,
     FetchConfig,
     FetchContentType,
@@ -26,6 +29,7 @@
     JsonFormatConfig,
     LinksFormatConfig,
     MarkdownFormatConfig,
+    MonitorActivityQuery,
     MonitorActivityRequest,
     MonitorActivityResponse,
     MonitorCreateRequest,
@@ -35,8 +39,22 @@
     MonitorTickEntry,
     MonitorTickStatus,
     MonitorUpdateRequest,
+    ScrapeBrandingFormatEntry,
+    ScrapeCaptureFormat,
+    ScrapeContentFormat,
+    ScrapeFormat,
+    ScrapeFormatEntry,
+    ScrapeFormatError,
+    ScrapeHtmlFormatEntry,
+    ScrapeImagesFormatEntry,
+    ScrapeJsonFormatEntry,
+    ScrapeLinksFormatEntry,
+    ScrapeMarkdownFormatEntry,
     ScrapeRequest,
     ScrapeResponse,
+    ScrapeScreenshotData,
+    ScrapeScreenshotFormatEntry,
+    ScrapeSummaryFormatEntry,
     ScreenshotFormatConfig,
     SearchRequest,
     SearchResponse,
@@ -54,13 +72,16 @@
     "ScrapeRequest",
     "ScrapeResponse",
     "ExtractRequest",
+    "ExtractRequestBase",
     "ExtractResponse",
     "SearchRequest",
     "SearchResponse",
     "SearchResult",
     "CrawlRequest",
     "CrawlResponse",
     "CrawlPage",
+    "CrawlPagesQuery",
+    "CrawlPagesResponse",
     "CrawlPageStatus",
     "CrawlStatus",
     "MonitorCreateRequest",
@@ -69,6 +90,7 @@
     "MonitorResult",
     "MonitorDiffs",
     "MonitorActivityRequest",
+    "MonitorActivityQuery",
     "MonitorActivityResponse",
     "MonitorTickEntry",
     "MonitorTickStatus",
@@ -83,6 +105,20 @@
     "FetchContentType",
     "FetchMode",
     "FormatConfig",
+    "ScrapeFormat",
+    "ScrapeContentFormat",
+    "ScrapeCaptureFormat",
+    "ScrapeFormatEntry",
+    "ScrapeMarkdownFormatEntry",
+    "ScrapeHtmlFormatEntry",
+    "ScrapeScreenshotFormatEntry",
+    "ScrapeJsonFormatEntry",
+    "ScrapeLinksFormatEntry",
+    "ScrapeImagesFormatEntry",
+    "ScrapeSummaryFormatEntry",
+    "ScrapeBrandingFormatEntry",
+    "ScrapeFormatError",
+    "ScrapeScreenshotData",
     "HtmlMode",
     "Service",
     "TimeRange",

src/scrapegraph_py/async_client.py

@@ -14,6 +14,8 @@
 from .env import env
 from .schemas import (
     ApiResult,
+    CrawlPagesQuery,
+    CrawlPagesResponse,
     CrawlRequest,
     CrawlResponse,
     CreditsResponse,
@@ -116,6 +118,18 @@ async def start(
     async def get(self, id: str) -> ApiResult[CrawlResponse]:
         return await self._client._get(f"/crawl/{id}", CrawlResponse)
 
+    async def pages(
+        self,
+        id: str,
+        *,
+        cursor: int | None = None,
+        limit: int | None = None,
+    ) -> ApiResult[CrawlPagesResponse]:
+        kwargs = _compact(cursor=cursor, limit=limit)
+        query = CrawlPagesQuery(**kwargs) if kwargs else None
+        qs = {key: getattr(query, key) for key in kwargs} if query else None
+        return await self._client._get(f"/crawl/{id}/pages", CrawlPagesResponse, params=qs or None)
+
     async def stop(self, id: str) -> ApiResult[dict]:
         return await self._client._post_empty(f"/crawl/{id}/stop")
 

src/scrapegraph_py/client.py

@@ -14,6 +14,8 @@
 from .env import env
 from .schemas import (
     ApiResult,
+    CrawlPagesQuery,
+    CrawlPagesResponse,
     CrawlRequest,
     CrawlResponse,
     CreditsResponse,
@@ -116,6 +118,18 @@ def start(
     def get(self, id: str) -> ApiResult[CrawlResponse]:
         return self._client._get(f"/crawl/{id}", CrawlResponse)
 
+    def pages(
+        self,
+        id: str,
+        *,
+        cursor: int | None = None,
+        limit: int | None = None,
+    ) -> ApiResult[CrawlPagesResponse]:
+        kwargs = _compact(cursor=cursor, limit=limit)
+        query = CrawlPagesQuery(**kwargs) if kwargs else None
+        qs = {key: getattr(query, key) for key in kwargs} if query else None
+        return self._client._get(f"/crawl/{id}/pages", CrawlPagesResponse, params=qs or None)
+
     def stop(self, id: str) -> ApiResult[dict]:
         return self._client._post_empty(f"/crawl/{id}/stop")