1+ import axios from "axios" ;
2+ import * as cheerio from "cheerio" ;
13import { logInfo , logWarn } from "./logger.js" ;
24
3- function buildSearchUrl ( keyword , config ) {
4- const url = new URL ( "https://www.linkedin.com/jobs/search/ " ) ;
5+ function buildSearchUrl ( keyword , config , start = 0 ) {
6+ const url = new URL ( "https://www.linkedin.com/jobs-guest/jobs/api/seeMoreJobPostings/search " ) ;
57 url . searchParams . set ( "keywords" , keyword ) ;
68 url . searchParams . set ( "location" , config . searchLocation ) ;
79 url . searchParams . set ( "geoId" , config . searchGeoId ) ;
810 url . searchParams . set ( "lang" , config . searchLanguage ) ;
9- // f_WT=2 = remoto; f_WT=3 = híbrido ; f_WT=1 = presencial
11+ // f_WT=2 = remoto; f_WT=3 = hibrido ; f_WT=1 = presencial
1012 url . searchParams . set ( "f_WT" , "2" ) ;
1113 if ( config . jobTypes ) {
1214 url . searchParams . set ( "f_JT" , config . jobTypes ) ;
1315 }
16+ url . searchParams . set ( "start" , String ( start ) ) ;
1417 return url . toString ( ) ;
1518}
1619
20+ async function fetchJobsChunk ( keyword , config , start ) {
21+ const url = buildSearchUrl ( keyword , config , start ) ;
22+
23+ const response = await axios . get ( url , {
24+ timeout : config . pageTimeoutMs ,
25+ headers : {
26+ "user-agent" :
27+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36" ,
28+ "accept-language" : "pt-BR,pt;q=0.9,en-US;q=0.8,en;q=0.7"
29+ }
30+ } ) ;
31+
32+ const $ = cheerio . load ( response . data ) ;
33+ const jobs = [ ] ;
34+
35+ $ ( ".base-card, .job-search-card" ) . each ( ( _ , card ) => {
36+ const node = $ ( card ) ;
37+ const titulo =
38+ node . find ( ".base-search-card__title" ) . text ( ) . trim ( ) ||
39+ node . find ( "h3" ) . first ( ) . text ( ) . trim ( ) ||
40+ "" ;
41+
42+ const empresa =
43+ node . find ( ".base-search-card__subtitle" ) . text ( ) . trim ( ) ||
44+ node . find ( "h4" ) . first ( ) . text ( ) . trim ( ) ||
45+ "" ;
46+
47+ const local = node . find ( ".job-search-card__location" ) . text ( ) . trim ( ) || "" ;
48+
49+ const link =
50+ node . find ( "a.base-card__full-link" ) . attr ( "href" ) ||
51+ node . find ( "a[href*='/jobs/view/']" ) . attr ( "href" ) ||
52+ "" ;
53+
54+ jobs . push ( { titulo, empresa, local, link } ) ;
55+ } ) ;
56+
57+ return jobs ;
58+ }
59+
1760function normalizeJob ( job ) {
1861 return {
1962 titulo : job . titulo ?. trim ( ) || "" ,
@@ -37,66 +80,48 @@ function dedupeJobs(jobs) {
3780 return [ ...unique . values ( ) ] ;
3881}
3982
40- export async function scrapeLinkedinJobs ( page , config ) {
83+ export async function scrapeLinkedinJobs ( config ) {
4184 const allJobs = [ ] ;
85+ const maxPagesPerKeyword = config . maxPagesPerKeyword || 5 ;
86+ const pageStep = 25 ;
4287
4388 for ( const keyword of config . keywords ) {
4489 logInfo ( `Buscando vagas para: ${ keyword } ) ;
4590
46- const url = buildSearchUrl ( keyword , config ) ;
47- await page . goto ( url , { waitUntil : "networkidle2" } ) ;
48- await new Promise ( ( resolve ) => setTimeout ( resolve , config . waitBetweenSearchesMs ) ) ;
91+ let jobsCountForKeyword = 0 ;
92+
93+ for ( let pageIndex = 0 ; pageIndex < maxPagesPerKeyword ; pageIndex ++ ) {
94+ const start = pageIndex * pageStep ;
95+ let jobs = [ ] ;
96+
97+ try {
98+ jobs = await fetchJobsChunk ( keyword , config , start ) ;
99+ } catch {
100+ logWarn ( `Falha HTTP na busca para: ${ keyword } ${ start } ) ;
101+ }
102+
103+ if ( jobs . length === 0 ) {
104+ if ( pageIndex === 0 ) {
105+ logWarn ( `Elementos de vaga nao encontrados para: ${ keyword } ) ;
106+ }
107+ break ;
108+ }
109+
110+ const normalized = jobs . map ( normalizeJob ) ;
111+
112+ allJobs . push (
113+ ...normalized . map ( ( job ) => ( {
114+ palavra : keyword ,
115+ ...job
116+ } ) )
117+ ) ;
118+
119+ jobsCountForKeyword += jobs . length ;
49120
50- try {
51- await page . waitForSelector ( ".job-search-card" , { timeout : config . pageTimeoutMs } ) ;
52- } catch {
53- logWarn ( `Elementos de vaga nao encontrados para: ${ keyword } ) ;
121+ await new Promise ( ( resolve ) => setTimeout ( resolve , config . waitBetweenSearchesMs ) ) ;
54122 }
55123
56- const jobs = await page
57- . $$eval ( ".job-search-card" , ( items ) =>
58- items . map ( ( jobCard ) => {
59- const titulo =
60- jobCard . querySelector ( "h3.base-search-card__title" ) ?. innerText ?. trim ( ) ||
61- jobCard
62- . querySelector ( "a[data-tracking-control-name='public_jobs_jserp-result_search-card']" )
63- ?. innerText ?. trim ( ) ||
64- "" ;
65-
66- const empresa =
67- jobCard . querySelector ( "h4.base-search-card__subtitle" ) ?. innerText ?. trim ( ) ||
68- jobCard
69- . querySelector (
70- "a[data-tracking-control-name='public_jobs_jserp-result_job-search-card-subtitle']"
71- )
72- ?. innerText ?. trim ( ) ||
73- "" ;
74-
75- const local =
76- jobCard . querySelector ( "span.job-search-card__location" ) ?. innerText ?. trim ( ) || "" ;
77-
78- const link =
79- jobCard . querySelector ( "h3.base-search-card__title a" ) ?. href ||
80- jobCard . querySelector (
81- "a[data-tracking-control-name='public_jobs_jserp-result_search-card']"
82- ) ?. href ||
83- "" ;
84-
85- return { titulo, empresa, local, link } ;
86- } )
87- )
88- . catch ( ( ) => [ ] ) ;
89-
90- const normalized = jobs . map ( normalizeJob ) ;
91-
92- allJobs . push (
93- ...normalized . map ( ( job ) => ( {
94- palavra : keyword ,
95- ...job
96- } ) )
97- ) ;
98-
99- logInfo ( `${ jobs . length } ${ keyword } ) ;
124+ logInfo ( `${ jobsCountForKeyword } ${ keyword } ) ;
100125 }
101126
102127 return dedupeJobs ( allJobs ) ;
0 commit comments