@@ -34,58 +34,57 @@ LIGHTPANDA_TOKEN: "<your-token>",
3434```
3535
3636``` ts trigger/lightpanda-cloud-puppeteer.ts
37- import { logger , task } from ' @trigger.dev/sdk'
38- import puppeteer from ' puppeteer'
37+ import { logger , task } from " @trigger.dev/sdk" ;
38+ import puppeteer from " puppeteer-core " ;
3939
4040export const lightpandaCloudPuppeteer = task ({
41- id: ' lightpanda-cloud-puppeteer' ,
41+ id: " lightpanda-cloud-puppeteer" ,
4242 machine: {
43- preset: ' micro' ,
43+ preset: " micro" ,
4444 },
4545 run : async (payload : { url: string }, { ctx }) => {
46- logger .log (" Lets get a page's links with Lightpanda!" , { payload , ctx })
46+ logger .log (" Lets get a page's links with Lightpanda!" , { payload , ctx });
47+
4748 if (! payload .url ) {
48- logger .warn (' Please define the payload url' )
49- throw new Error (' payload.url is undefined' )
49+ logger .warn (" Please define the payload url" );
50+ throw new Error (" payload.url is undefined" );
5051 }
5152
52- if (typeof process .env .LIGHTPANDA_TOKEN === ' undefined' ) {
53- logger .warn (' Please define the env variable $LIGHTPANDA_TOKEN' , {
54- env: process .env ,
55- })
56- throw new Error (' $LIGHTPANDA_TOKEN is undefined' )
53+ const token = process .env .LIGHTPANDA_TOKEN ;
54+ if (! token ) {
55+ logger .warn (" Please define the env variable LIGHTPANDA_TOKEN" );
56+ throw new Error (" LIGHTPANDA_TOKEN is undefined" );
5757 }
5858
5959 // Connect to Lightpanda's cloud
6060 const browser = await puppeteer .connect ({
61- browserWSEndpoint: ` wss://cloud.lightpanda.io/ws?browser=lightpanda&token=${process . env . LIGHTPANDA_TOKEN } ` ,
62- })
63- const context = await browser .createBrowserContext ()
64- const page = await context .newPage ()
61+ browserWSEndpoint: ` wss://cloud.lightpanda.io/ws?browser=lightpanda&token=${token } ` ,
62+ });
63+ const context = await browser .createBrowserContext ();
64+ const page = await context .newPage ();
6565
6666 // Dump all the links from the page.
67- await page .goto (payload .url )
67+ await page .goto (payload .url );
6868
6969 const links = await page .evaluate (() => {
70- return Array .from (document .querySelectorAll (' a ' )).map (row => {
71- return row .getAttribute (' href' )
72- })
73- })
70+ return Array .from (document .querySelectorAll (" a " )).map (( row ) => {
71+ return row .getAttribute (" href" );
72+ });
73+ });
7474
75- logger .info (' Processing done' )
76- logger .info (' Shutting down…' )
75+ logger .info (" Processing done, shutting down…" );
7776
78- await page .close ()
79- await context .close ()
80- await browser .disconnect ()
77+ await page .close ();
78+ await context .close ();
79+ await browser .disconnect ();
8180
82- logger .info (' ✅ Completed' )
81+ logger .info (" ✅ Completed" );
8382
8483 return {
8584 links ,
86- }
85+ };
8786 },
88- })
87+ });
8988```
9089### Proxies
9190
@@ -111,39 +110,32 @@ You will have to pass the URL as a payload when triggering the task.
111110- Setup the [ Lightpanda build extension] ( /config/extensions/lightpanda )
112111
113112### Task
114- ``` ts trigger/lightpanda-lightpanda- fetch.ts
115- import { logger , task } from ' @trigger.dev/sdk'
116- import { execSync } from ' node:child_process'
113+ ``` ts trigger/lightpanda-fetch.ts
114+ import { logger , task } from " @trigger.dev/sdk" ;
115+ import { execSync } from " node:child_process" ;
117116
118117export const lightpandaFetch = task ({
119- id: ' lightpanda-fetch' ,
118+ id: " lightpanda-fetch" ,
120119 machine: {
121120 preset: " micro" ,
122121 },
123122 run : async (payload : { url: string }, { ctx }) => {
124- logger .log (" Lets get a page's content with Lightpanda!" , { payload , ctx })
123+ logger .log (" Lets get a page's content with Lightpanda!" , { payload , ctx });
125124
126125 if (! payload .url ) {
127- logger .warn (' Please define the payload url' )
128- throw new Error (' payload.url is undefined' )
126+ logger .warn (" Please define the payload url" );
127+ throw new Error (" payload.url is undefined" );
129128 }
130129
131- if (typeof process .env .LIGHTPANDA_BROWSER_PATH === ' undefined' ) {
132- logger .warn (' Please define the env variable $LIGHTPANDA_BROWSER_PATH' , {
133- env: process .env ,
134- })
135- throw new Error (' $LIGHTPANDA_BROWSER_PATH is undefined' )
136- }
130+ const buffer = execSync (` lightpanda fetch --dump ${payload .url } ` );
137131
138- const e = execSync (` ${process .env .LIGHTPANDA_BROWSER_PATH } fetch --dump ${payload .url } ` )
139-
140- logger .info (' ✅ Completed' )
132+ logger .info (" ✅ Completed" );
141133
142134 return {
143- message: e .toString (),
144- }
135+ message: buffer .toString (),
136+ };
145137 },
146- })
138+ });
147139```
148140
149141## Example \# 3 - Launch and use a Lightpanda CDP server
@@ -156,92 +148,82 @@ This task initialises a Lightpanda CDP server to allow you to scrape directly vi
156148### Task
157149Your task will have to launch a child process in order to have the websocket available to scrape using Puppeteer.
158150
159- ``` ts trigger/lightpandaCDP .ts
160- import { logger , task } from ' @trigger.dev/sdk'
161- import { spawn , type ChildProcessWithoutNullStreams } from ' node:child_process'
162- import puppeteer from ' puppeteer'
151+ ``` ts trigger/lightpanda-cdp .ts
152+ import { logger , task } from " @trigger.dev/sdk" ;
153+ import { spawn , type ChildProcessWithoutNullStreams } from " node:child_process" ;
154+ import puppeteer from " puppeteer-core " ;
163155
164- const spawnLightpanda = async (log : typeof logger ) =>
156+ const spawnLightpanda = async (host : string , port : string ) =>
165157 new Promise <ChildProcessWithoutNullStreams >((resolve , reject ) => {
166- const child = spawn (process . env . LIGHTPANDA_BROWSER_PATH as string , [
167- ' serve' ,
168- ' --host' ,
169- ' 127.0.0.1 ' ,
170- ' --port' ,
171- ' 9222 ' ,
172- ' --log_level' ,
173- ' info' ,
174- ])
175-
176- child .on (' spawn' , async () => {
158+ const child = spawn (" lightpanda " , [
159+ " serve" ,
160+ " --host" ,
161+ host ,
162+ " --port" ,
163+ port ,
164+ " --log_level" ,
165+ " info" ,
166+ ]);
167+
168+ child .on (" spawn" , async () => {
177169 logger .info (" Running Lightpanda's CDP server…" , {
178170 pid: child .pid ,
179- })
171+ });
180172
181- await new Promise (resolve => setTimeout (resolve , 250 ))
182- resolve (child )
183- })
184- child .on (' error' , e => reject (e ))
185- })
173+ await new Promise (( resolve ) => setTimeout (resolve , 250 ));
174+ resolve (child );
175+ });
176+ child .on (" error" , ( e ) => reject (e ));
177+ });
186178
187179export const lightpandaCDP = task ({
188- id: ' lightpanda-cdp' ,
180+ id: " lightpanda-cdp" ,
189181 machine: {
190- preset: ' micro' ,
182+ preset: " micro" ,
191183 },
192184 run : async (payload : { url: string }, { ctx }) => {
193- logger .log (" Lets get a page's links with Lightpanda!" , { payload , ctx })
185+ logger .log (" Lets get a page's links with Lightpanda!" , { payload , ctx });
194186
195187 if (! payload .url ) {
196- logger .warn (' Please define the payload url' )
197- throw new Error (' payload.url is undefined' )
188+ logger .warn (" Please define the payload url" );
189+ throw new Error (" payload.url is undefined" );
198190 }
199191
200- if (typeof process .env .LIGHTPANDA_BROWSER_PATH === ' undefined' ) {
201- logger .warn (' Please define the env variable $LIGHTPANDA_BROWSER_PATH' , {
202- env: process .env ,
203- })
204- throw new Error (' $LIGHTPANDA_BROWSER_PATH is undefined' )
205- }
192+ const host = process .env .LIGHTPANDA_CDP_HOST ?? " 127.0.0.1" ;
193+ const port = process .env .LIGHTPANDA_CDP_PORT ?? " 9222" ;
206194
207- try {
208- // Launch Lightpanda's CDP server
209- const lpProcess = await spawnLightpanda (logger )
195+ // Launch Lightpanda's CDP server
196+ const lpProcess = await spawnLightpanda (host , port );
210197
211- const browser = await puppeteer .connect ({
212- browserWSEndpoint: ' ws://127.0.0.1:9222 ' ,
213- })
214- const context = await browser .createBrowserContext ()
215- const page = await context .newPage ()
198+ const browser = await puppeteer .connect ({
199+ browserWSEndpoint: ` ws://${ host }:${ port } ` ,
200+ });
201+ const context = await browser .createBrowserContext ();
202+ const page = await context .newPage ();
216203
217- // Dump all the links from the page.
218- await page .goto (payload .url )
204+ // Dump all the links from the page.
205+ await page .goto (payload .url );
219206
220- const links = await page .evaluate (() => {
221- return Array .from (document .querySelectorAll (' a ' )).map (row => {
222- return row .getAttribute (' href' )
223- })
224- })
207+ const links = await page .evaluate (() => {
208+ return Array .from (document .querySelectorAll (" a " )).map (( row ) => {
209+ return row .getAttribute (" href" );
210+ });
211+ });
225212
226- logger .info (' Processing done' )
227- logger .info (' Shutting down…' )
213+ logger .info (" Processing done" );
214+ logger .info (" Shutting down…" );
228215
229- // Close Puppeteer instance
230- await browser .close ()
216+ // Close Puppeteer instance
217+ await browser .close ();
231218
232- // Stop Lightpanda's CDP Server
233- lpProcess .stdout .destroy ()
234- lpProcess .stderr .destroy ()
235- lpProcess .kill ()
219+ // Stop Lightpanda's CDP Server
220+ lpProcess .kill ();
236221
237- logger .info (' ✅ Completed' )
222+ logger .info (" ✅ Completed" );
238223
239- return {
240- links ,
241- }
242- } catch (e : any ) {
243- throw new Error (e )
244- }
224+ return {
225+ links ,
226+ };
245227 },
246- })
228+ });
247229```
0 commit comments