feat: edit ui + add return html option (#90)

* fix: restyle the element table

* chore: wip ui

* wip: edit styles

* feat: add html return

* fix: build

* fix: workflow

* fix: workflow

* fix: workflow

* fix: workflow

* fix: workflow

* fix: workflow

* fix: workflow

* fix: cypress test

* chore: update photo [skip ci]

Author: jaypyles

.github/actions/run-cypress-tests/action.yaml

@@ -73,5 +73,8 @@ runs:
 
     - name: Run Cypress tests
       shell: bash
-      run: npm run cy:run
+      run: |
+        set -e
+        npm run cy:run
+
 

.github/workflows/cypress-tests.yml

@@ -18,15 +18,14 @@ jobs:
         uses: ./.github/actions/run-cypress-tests
         with:
           openai_key: ${{ secrets.openai_key }}
-        continue-on-error: true
 
       - name: Check container logs on failure
-        if: steps.run-tests.outcome == 'failure'
+        if: steps.run-tests.conclusion == 'failure'
         run: |
           echo "Cypress tests failed. Dumping container logs..."
           docker logs scraperr_api || true
 
       - name: Fail job if Cypress failed
-        if: steps.run-tests.outcome == 'failure'
+        if: steps.run-tests.conclusion == 'failure'
         run: exit 1
-      - uses: actions/checkout@v4
+

.github/workflows/tests.yml

@@ -22,7 +22,6 @@ jobs:
         uses: ./.github/actions/run-cypress-tests
         with:
           openai_key: ${{ secrets.openai_key }}
-        continue-on-error: true
 
   success-message:
     runs-on: ubuntu-latest

api/backend/ai/agent/agent.py

@@ -63,7 +63,9 @@ async def scrape_with_agent(agent_job: dict[str, Any]):
 
             xpaths = parse_response(response)
 
-            captured_elements = await capture_elements(page, xpaths)
+            captured_elements = await capture_elements(
+                page, xpaths, agent_job["job_options"]["return_html"]
+            )
 
             final_url = page.url
 

api/backend/ai/agent/utils.py

@@ -206,7 +206,7 @@ def parse_next_page(text: str) -> str | None:
 
 
 async def capture_elements(
-    page: Page, xpaths: list[dict[str, str]]
+    page: Page, xpaths: list[dict[str, str]], return_html: bool
 ) -> list[CapturedElement]:
     captured_elements = []
     seen_texts = set()
@@ -217,6 +217,23 @@ async def capture_elements(
             count = await locator.count()
 
             for i in range(count):
+                if return_html:
+                    element_text = (
+                        await page.locator(f"xpath={xpath['xpath']}")
+                        .nth(i)
+                        .inner_html()
+                    )
+
+                    seen_texts.add(element_text)
+                    captured_elements.append(
+                        CapturedElement(
+                            name=xpath["name"],
+                            text=element_text,
+                            xpath=xpath["xpath"],
+                        )
+                    )
+                    continue
+
                 element_text = ""
 
                 element_handle = await locator.nth(i).element_handle()

api/backend/job/models/job_options.py

@@ -25,3 +25,4 @@ class JobOptions(BaseModel):
     site_map: Optional[SiteMap] = None
     collect_media: bool = False
     custom_cookies: list[dict[str, Any]] = []
+    return_html: bool = False

api/backend/job/scraping/scraping.py

@@ -110,7 +110,9 @@ async def make_site_request(
             )
 
 
-async def collect_scraped_elements(page: tuple[str, str], xpaths: list[Element]):
+async def collect_scraped_elements(
+    page: tuple[str, str], xpaths: list[Element], return_html: bool
+):
     soup = BeautifulSoup(page[0], "lxml")
     root = etree.HTML(str(soup))
 
@@ -120,6 +122,16 @@ async def collect_scraped_elements(page: tuple[str, str], xpaths: list[Element])
         el = sxpath(root, elem.xpath)
 
         for e in el:  # type: ignore
+            if return_html:
+                elements[elem.name] = [
+                    CapturedElement(
+                        xpath=elem.xpath,
+                        text=page[0],
+                        name=elem.name,
+                    )
+                ]
+                continue
+
             text = (
                 " ".join(str(t) for t in e.itertext())
                 if isinstance(e, etree._Element)
@@ -161,6 +173,8 @@ async def scrape(
     elements: list[dict[str, dict[str, list[CapturedElement]]]] = []
 
     for page in pages:
-        elements.append(await collect_scraped_elements(page, xpaths))
+        elements.append(
+            await collect_scraped_elements(page, xpaths, job_options["return_html"])
+        )
 
     return elements

cypress/utilities/job.utilities.ts

@@ -100,27 +100,27 @@ export const waitForJobCompletion = (url: string) => {
 };
 
 export const enableMultiPageScraping = () => {
-  cy.get("button").contains("Advanced Job Options").click();
+  cy.get("button").contains("Advanced Options").click();
   cy.get('[data-cy="multi-page-toggle"]').click();
   cy.get("body").type("{esc}");
 };
 
 export const addCustomHeaders = (headers: Record<string, string>) => {
-  cy.get("button").contains("Advanced Job Options").click();
+  cy.get("button").contains("Advanced Options").click();
   cy.get('[name="custom_headers"]').type(JSON.stringify(headers), {
     parseSpecialCharSequences: false,
   });
   cy.get("body").type("{esc}");
 };
 
 export const addCustomCookies = (cookies: Record<string, string>) => {
-  cy.get("button").contains("Advanced Job Options").click();
+  cy.get("button").contains("Advanced Options").click();
   cy.get('[name="custom_cookies"]').type(JSON.stringify(cookies));
   cy.get("body").type("{esc}");
 };
 
 export const openAdvancedJobOptions = () => {
-  cy.get("button").contains("Advanced Job Options").click();
+  cy.get("button").contains("Advanced Options").click();
 };
 
 export const selectJobFromSelector = () => {

docs/main_page.png

@@ -1,7 +1,8 @@
-import { Box, Link, Typography } from "@mui/material";
-import { SetStateAction, Dispatch, useState } from "react";
-import { AdvancedJobOptionsDialog } from "./dialog/advanced-job-options-dialog";
 import { RawJobOptions } from "@/types";
+import SettingsIcon from "@mui/icons-material/Settings";
+import { Box, Button, Typography } from "@mui/material";
+import { Dispatch, SetStateAction, useState } from "react";
+import { AdvancedJobOptionsDialog } from "./dialog/advanced-job-options-dialog";
 
 export type AdvancedJobOptionsProps = {
   jobOptions: RawJobOptions;
@@ -17,26 +18,27 @@ export const AdvancedJobOptions = ({
   const [open, setOpen] = useState(false);
 
   return (
-    <Box sx={{ mb: 2 }}>
-      <Link
-        component="button"
-        variant="body2"
+    <Box sx={{ display: "flex", alignItems: "center", gap: 1 }}>
+      <Button
+        variant="outlined"
         onClick={() => setOpen(true)}
+        startIcon={<SettingsIcon />}
         sx={{
-          textDecoration: "none",
-          color: "primary.main",
+          textTransform: "none",
+          borderRadius: 2,
+          px: 2,
+          py: 1,
+          borderColor: "divider",
+          color: "text.secondary",
           "&:hover": {
-            color: "primary.dark",
-            textDecoration: "underline",
+            borderColor: "primary.main",
+            color: "primary.main",
+            bgcolor: "action.hover",
           },
-          paddingLeft: 1,
-          display: "inline-flex",
-          alignItems: "center",
-          gap: 0.5,
         }}
       >
-        <Typography variant="body2">Advanced Job Options</Typography>
-      </Link>
+        <Typography variant="body2">Advanced Options</Typography>
+      </Button>
 
       <AdvancedJobOptionsDialog
         open={open}