Skip to content

Commit 067f38a

Browse files
committed
update examples
1 parent 54585cb commit 067f38a

File tree

5 files changed

+399
-115
lines changed

5 files changed

+399
-115
lines changed

docs/guides/examples/lightpanda.mdx

Lines changed: 94 additions & 112 deletions
Original file line numberDiff line numberDiff line change
@@ -34,58 +34,57 @@ LIGHTPANDA_TOKEN: "<your-token>",
3434
```
3535

3636
```ts trigger/lightpanda-cloud-puppeteer.ts
37-
import { logger, task } from '@trigger.dev/sdk'
38-
import puppeteer from 'puppeteer'
37+
import { logger, task } from "@trigger.dev/sdk";
38+
import puppeteer from "puppeteer-core";
3939

4040
export const lightpandaCloudPuppeteer = task({
41-
id: 'lightpanda-cloud-puppeteer',
41+
id: "lightpanda-cloud-puppeteer",
4242
machine: {
43-
preset: 'micro',
43+
preset: "micro",
4444
},
4545
run: async (payload: { url: string }, { ctx }) => {
46-
logger.log("Lets get a page's links with Lightpanda!", { payload, ctx })
46+
logger.log("Lets get a page's links with Lightpanda!", { payload, ctx });
47+
4748
if (!payload.url) {
48-
logger.warn('Please define the payload url')
49-
throw new Error('payload.url is undefined')
49+
logger.warn("Please define the payload url");
50+
throw new Error("payload.url is undefined");
5051
}
5152

52-
if (typeof process.env.LIGHTPANDA_TOKEN === 'undefined') {
53-
logger.warn('Please define the env variable $LIGHTPANDA_TOKEN', {
54-
env: process.env,
55-
})
56-
throw new Error('$LIGHTPANDA_TOKEN is undefined')
53+
const token = process.env.LIGHTPANDA_TOKEN;
54+
if (!token) {
55+
logger.warn("Please define the env variable LIGHTPANDA_TOKEN");
56+
throw new Error("LIGHTPANDA_TOKEN is undefined");
5757
}
5858

5959
// Connect to Lightpanda's cloud
6060
const browser = await puppeteer.connect({
61-
browserWSEndpoint: `wss://cloud.lightpanda.io/ws?browser=lightpanda&token=${process.env.LIGHTPANDA_TOKEN}`,
62-
})
63-
const context = await browser.createBrowserContext()
64-
const page = await context.newPage()
61+
browserWSEndpoint: `wss://cloud.lightpanda.io/ws?browser=lightpanda&token=${token}`,
62+
});
63+
const context = await browser.createBrowserContext();
64+
const page = await context.newPage();
6565

6666
// Dump all the links from the page.
67-
await page.goto(payload.url)
67+
await page.goto(payload.url);
6868

6969
const links = await page.evaluate(() => {
70-
return Array.from(document.querySelectorAll('a')).map(row => {
71-
return row.getAttribute('href')
72-
})
73-
})
70+
return Array.from(document.querySelectorAll("a")).map((row) => {
71+
return row.getAttribute("href");
72+
});
73+
});
7474

75-
logger.info('Processing done')
76-
logger.info('Shutting down…')
75+
logger.info("Processing done, shutting down…");
7776

78-
await page.close()
79-
await context.close()
80-
await browser.disconnect()
77+
await page.close();
78+
await context.close();
79+
await browser.disconnect();
8180

82-
logger.info('✅ Completed')
81+
logger.info("✅ Completed");
8382

8483
return {
8584
links,
86-
}
85+
};
8786
},
88-
})
87+
});
8988
```
9089
### Proxies
9190

@@ -111,39 +110,32 @@ You will have to pass the URL as a payload when triggering the task.
111110
- Setup the [Lightpanda build extension](/config/extensions/lightpanda)
112111

113112
### Task
114-
```ts trigger/lightpanda-lightpanda-fetch.ts
115-
import { logger, task } from '@trigger.dev/sdk'
116-
import { execSync } from 'node:child_process'
113+
```ts trigger/lightpanda-fetch.ts
114+
import { logger, task } from "@trigger.dev/sdk";
115+
import { execSync } from "node:child_process";
117116

118117
export const lightpandaFetch = task({
119-
id: 'lightpanda-fetch',
118+
id: "lightpanda-fetch",
120119
machine: {
121120
preset: "micro",
122121
},
123122
run: async (payload: { url: string }, { ctx }) => {
124-
logger.log("Lets get a page's content with Lightpanda!", { payload, ctx })
123+
logger.log("Lets get a page's content with Lightpanda!", { payload, ctx });
125124

126125
if (!payload.url) {
127-
logger.warn('Please define the payload url')
128-
throw new Error('payload.url is undefined')
126+
logger.warn("Please define the payload url");
127+
throw new Error("payload.url is undefined");
129128
}
130129

131-
if (typeof process.env.LIGHTPANDA_BROWSER_PATH === 'undefined') {
132-
logger.warn('Please define the env variable $LIGHTPANDA_BROWSER_PATH', {
133-
env: process.env,
134-
})
135-
throw new Error('$LIGHTPANDA_BROWSER_PATH is undefined')
136-
}
130+
const buffer = execSync(`lightpanda fetch --dump ${payload.url}`);
137131

138-
const e = execSync(`${process.env.LIGHTPANDA_BROWSER_PATH} fetch --dump ${payload.url}`)
139-
140-
logger.info('✅ Completed')
132+
logger.info("✅ Completed");
141133

142134
return {
143-
message: e.toString(),
144-
}
135+
message: buffer.toString(),
136+
};
145137
},
146-
})
138+
});
147139
```
148140

149141
## Example \#3 - Launch and use a Lightpanda CDP server
@@ -156,92 +148,82 @@ This task initialises a Lightpanda CDP server to allow you to scrape directly vi
156148
### Task
157149
Your task will have to launch a child process in order to have the websocket available to scrape using Puppeteer.
158150

159-
```ts trigger/lightpandaCDP.ts
160-
import { logger, task } from '@trigger.dev/sdk'
161-
import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process'
162-
import puppeteer from 'puppeteer'
151+
```ts trigger/lightpanda-cdp.ts
152+
import { logger, task } from "@trigger.dev/sdk";
153+
import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
154+
import puppeteer from "puppeteer-core";
163155

164-
const spawnLightpanda = async (log: typeof logger) =>
156+
const spawnLightpanda = async (host: string, port: string) =>
165157
new Promise<ChildProcessWithoutNullStreams>((resolve, reject) => {
166-
const child = spawn(process.env.LIGHTPANDA_BROWSER_PATH as string, [
167-
'serve',
168-
'--host',
169-
'127.0.0.1',
170-
'--port',
171-
'9222',
172-
'--log_level',
173-
'info',
174-
])
175-
176-
child.on('spawn', async () => {
158+
const child = spawn("lightpanda", [
159+
"serve",
160+
"--host",
161+
host,
162+
"--port",
163+
port,
164+
"--log_level",
165+
"info",
166+
]);
167+
168+
child.on("spawn", async () => {
177169
logger.info("Running Lightpanda's CDP server…", {
178170
pid: child.pid,
179-
})
171+
});
180172

181-
await new Promise(resolve => setTimeout(resolve, 250))
182-
resolve(child)
183-
})
184-
child.on('error', e => reject(e))
185-
})
173+
await new Promise((resolve) => setTimeout(resolve, 250));
174+
resolve(child);
175+
});
176+
child.on("error", (e) => reject(e));
177+
});
186178

187179
export const lightpandaCDP = task({
188-
id: 'lightpanda-cdp',
180+
id: "lightpanda-cdp",
189181
machine: {
190-
preset: 'micro',
182+
preset: "micro",
191183
},
192184
run: async (payload: { url: string }, { ctx }) => {
193-
logger.log("Lets get a page's links with Lightpanda!", { payload, ctx })
185+
logger.log("Lets get a page's links with Lightpanda!", { payload, ctx });
194186

195187
if (!payload.url) {
196-
logger.warn('Please define the payload url')
197-
throw new Error('payload.url is undefined')
188+
logger.warn("Please define the payload url");
189+
throw new Error("payload.url is undefined");
198190
}
199191

200-
if (typeof process.env.LIGHTPANDA_BROWSER_PATH === 'undefined') {
201-
logger.warn('Please define the env variable $LIGHTPANDA_BROWSER_PATH', {
202-
env: process.env,
203-
})
204-
throw new Error('$LIGHTPANDA_BROWSER_PATH is undefined')
205-
}
192+
const host = process.env.LIGHTPANDA_CDP_HOST ?? "127.0.0.1";
193+
const port = process.env.LIGHTPANDA_CDP_PORT ?? "9222";
206194

207-
try {
208-
// Launch Lightpanda's CDP server
209-
const lpProcess = await spawnLightpanda(logger)
195+
// Launch Lightpanda's CDP server
196+
const lpProcess = await spawnLightpanda(host, port);
210197

211-
const browser = await puppeteer.connect({
212-
browserWSEndpoint: 'ws://127.0.0.1:9222',
213-
})
214-
const context = await browser.createBrowserContext()
215-
const page = await context.newPage()
198+
const browser = await puppeteer.connect({
199+
browserWSEndpoint: `ws://${host}:${port}`,
200+
});
201+
const context = await browser.createBrowserContext();
202+
const page = await context.newPage();
216203

217-
// Dump all the links from the page.
218-
await page.goto(payload.url)
204+
// Dump all the links from the page.
205+
await page.goto(payload.url);
219206

220-
const links = await page.evaluate(() => {
221-
return Array.from(document.querySelectorAll('a')).map(row => {
222-
return row.getAttribute('href')
223-
})
224-
})
207+
const links = await page.evaluate(() => {
208+
return Array.from(document.querySelectorAll("a")).map((row) => {
209+
return row.getAttribute("href");
210+
});
211+
});
225212

226-
logger.info('Processing done')
227-
logger.info('Shutting down…')
213+
logger.info("Processing done");
214+
logger.info("Shutting down…");
228215

229-
// Close Puppeteer instance
230-
await browser.close()
216+
// Close Puppeteer instance
217+
await browser.close();
231218

232-
// Stop Lightpanda's CDP Server
233-
lpProcess.stdout.destroy()
234-
lpProcess.stderr.destroy()
235-
lpProcess.kill()
219+
// Stop Lightpanda's CDP Server
220+
lpProcess.kill();
236221

237-
logger.info('✅ Completed')
222+
logger.info("✅ Completed");
238223

239-
return {
240-
links,
241-
}
242-
} catch (e: any) {
243-
throw new Error(e)
244-
}
224+
return {
225+
links,
226+
};
245227
},
246-
})
228+
});
247229
```

0 commit comments

Comments
 (0)