-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathscraper.js
More file actions
68 lines (48 loc) · 2.42 KB
/
scraper.js
File metadata and controls
68 lines (48 loc) · 2.42 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
const puppeteer = require("puppeteer");
const path = require('path');
const fs = require("fs");
async function loadPage() {
let browser = await puppeteer.launch(); //starts brwoser instance
let page = await browser.newPage(); // starts page instance
await page.goto('https://based.win'); // select webpage to visit
let pageTitle = await page.title(); // get page title
let pageContents = await page.content(); // get page content
console.log(pageContents);
console.log('pageTitle= ', pageTitle);
console.log('it worked');
await page.screenshot({path: "screenshot1.png"}); // get page screenshot
await page.pdf({path: "web.pdf", format: "a4"}); // get page pdf
page.evaluate
let textSelector = 'li.ast-grid-common-col:nth-child(1) > div:nth-child(2) > a:nth-child(1) > h2:nth-child(1)';
let selectedContent = await page.evaluate((sel) => {
return document.querySelector(sel).innerText;
}, textSelector)
console.log('Returned Page content:', selectedContent)
let linkselect ='li.ast-grid-common-col:nth-child(1) > div:nth-child(2) > a:nth-child(1)'
page.click(linkselect);
await new Promise(resolve => setTimeout(resolve, 2000));
await page.waitForSelector('.variations_form', { visible: true });
await page.screenshot({path: "screenshot2.png"});
let searchTerm = "shirt";
page.evaluate((searchTerm)=> {
let input = document.querySelector('#wp-block-search__input-1')
let form = document.querySelector('aside.header-widget-area:nth-child(1) > section:nth-child(1) > form:nth-child(1)')
input.value = searchTerm
form.submit();
}, searchTerm);
await new Promise(resolve => setTimeout(resolve, 2000));
await page.screenshot({path: "screenshot3.png"});
// (change selector as needed)
let imageselect = 'li.ast-grid-common-col:nth-child(1) > div:nth-child(1) > a:nth-child(1) > img:nth-child(1)'
// Get the image URL
const imageUrl = await page.$eval(imageselect, img => img.src);
// Download the image using Node.js
const viewSource = await page.goto(imageUrl);
const buffer = await viewSource.buffer();
// Save the image to disk
const downloadPath = path.resolve('./downloads');
if (!fs.existsSync(downloadPath)) fs.mkdirSync(downloadPath);
fs.writeFileSync(path.join(downloadPath, 'downloaded_image.jpg'), buffer);
await browser.close();
}
loadPage();