Skip to content

Commit 4664d1e

Browse files
committed
Enhance HTMLExtractionOptions with new annotateNumberClasses feature
- Added `annotateNumberClasses` option to the HTMLExtractionOptions interface for appending CSS class names to numeric elements in markdown output. - Updated README to include documentation for the new feature. - Improved HTML to Markdown conversion logic to support number class annotation, preserving semantic meaning. - Added unit tests to ensure correct functionality across various scenarios.
1 parent 79ae1da commit 4664d1e

92 files changed

Lines changed: 102741 additions & 8 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

a.html

Lines changed: 22663 additions & 0 deletions
Large diffs are not rendered by default.

a.out

Lines changed: 2343 additions & 0 deletions
Large diffs are not rendered by default.

aa.html

Lines changed: 32 additions & 0 deletions
Large diffs are not rendered by default.

scrape-debug-1775576198343/annotated-markdown.md

Lines changed: 2226 additions & 0 deletions
Large diffs are not rendered by default.
Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
function scrape(document) {
2+
const products = [];
3+
const productLinks = document.querySelectorAll('div[data-testid="list-view"] a.w-100.h-100.z-1');
4+
5+
productLinks.forEach(link => {
6+
const productTile = link.closest('div[data-testid="list-view"] > div > div');
7+
if (!productTile) return;
8+
9+
const name = link.textContent.trim();
10+
const productUrl = link.href;
11+
12+
const imageUrl = productTile.querySelector('img[data-testid="productTileImage"]')?.src;
13+
14+
let price;
15+
const priceElement = productTile.querySelector('div.flex.flex-wrap.justify-start > div.mr1.mr2-xl.b');
16+
if (priceElement) {
17+
const priceText = priceElement.textContent.replace(/[^0-9.]/g, '');
18+
price = parseFloat(priceText);
19+
}
20+
21+
let originalPrice;
22+
const originalPriceElement = productTile.querySelector('div.flex.flex-wrap.justify-start > div.gray.mr1.f7:nth-of-type(2)');
23+
if (originalPriceElement) {
24+
const originalPriceText = originalPriceElement.textContent.replace(/[^0-9.]/g, '');
25+
originalPrice = parseFloat(originalPriceText);
26+
}
27+
28+
const brand = productTile.querySelector('div.mb1.mt2.b')?.textContent.trim();
29+
30+
let rating;
31+
let reviewCount;
32+
const ratingReviewElement = productTile.querySelector('div.flex.items-center.mt2');
33+
if (ratingReviewElement) {
34+
const ratingReviewText = ratingReviewElement.textContent;
35+
const match = ratingReviewText.match(/(\d+\.?\d*)\s+out of 5 stars\.\s+(\d+)\s+reviews/);
36+
if (match) {
37+
rating = parseFloat(match[1]);
38+
reviewCount = parseInt(match[2], 10);
39+
}
40+
}
41+
42+
products.push({
43+
name,
44+
brand: brand || undefined,
45+
price: price || undefined,
46+
originalPrice: originalPrice || undefined,
47+
rating: rating || undefined,
48+
reviewCount: reviewCount || undefined,
49+
productUrl,
50+
imageUrl: imageUrl || undefined
51+
});
52+
});
53+
54+
return { products };
55+
}
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"products": []
3+
}

scrape-debug-1775576198343/attempt-1/prompt.txt

Lines changed: 2262 additions & 0 deletions
Large diffs are not rendered by default.
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
{
2+
"products": []
3+
}

scrape-debug-1775576198343/attempt-1/validation-prompt.txt

Lines changed: 2259 additions & 0 deletions
Large diffs are not rendered by default.
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"isValid": false,
3+
"issues": "The 'products' array is empty, but the annotated content clearly shows multiple product listings with details such as name, price, brand, rating, review count, product URL, and image URL. The scraping code needs to be fixed to extract all these product details and populate the 'products' array accordingly."
4+
}

0 commit comments

Comments
 (0)