-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathLinkedlin.py
More file actions
149 lines (137 loc) · 9.44 KB
/
Linkedlin.py
File metadata and controls
149 lines (137 loc) · 9.44 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
# -*- coding: utf-8 -*-
import json
from DrissionPage import WebPage
import pandas as pd
from datetime import datetime
import time
import traceback
import requests
current_date = datetime.now().strftime('%Y-%m-%d')
bro = WebPage()
def get_company_name(data):
try:
return data['data']['companyDetails']['companyName']
except KeyError:
try:
return data['included'][-1]['name']
except (KeyError, IndexError, TypeError):
return ''
# 检测登录状态
try:
bro.get('https://www.linkedin.com/?trk=seo-authwall-base_nav-header-logo')
if bro.ele('@data-test-id=home-hero-sign-in-cta'):
input('发现登录窗口，请登录后按回车继续本程序')
except Exception as e:
print(e)
pass
headers = {
'accept': 'application/vnd.linkedin.normalized+json+2.1',
'accept-language': 'zh-CN,zh;q=0.9',
# Requests sorts cookies= alphabetically
# 'cookie': 'bcookie="v=2&4efb692e-850f-41fe-8e42-60513e407c48"; bscookie="v=1&20240724082206f503bb8a-446b-445c-8b2d-f9e3357357d8AQEGZ_GHoXkoTYS0kwqiib-K8a4p_LKm"; _gcl_au=1.1.1129714245.1721809328; _uetsid=dad3f160498411efa21b75acde05f844; _uetvid=dad41980498411efa7c5bd563921af0c; aam_uuid=87273921066518620232088361781689752310; li_rm=AQH3ccj2AYHIAwAAAZDj1V9he1SEpXqQfCEs4PQMC_Dqm1BRuoTaZEn7twjsx2bzRECT2HBIe6eSFe83OBuEvXT_lvSE2EEXqxgwjpob7GA9RcixSRC0SmQU; fid=AQHBHhMdzaSI9gAAAZDj1cctUGI893PPXXw5kXLRfeOZ-v9Wv-m0ggTE0JMYHwIW4YQjm64bcsVMlg; fcookie=AQEFigE4Ag26KAAAAZDj3xpPOO4u4myb_sM7f9qzzPe75grN-XCpBd4m26v-50-qvPqS8CpTgQEu7HR_Sj9jmFo6I5Sujmw2X7eUd8ZI6PE8ncx2d_6ZW6H6w6jZHxpzRa9dh6fywvBc38hKx7NXLU4S6BWrfJEZI4_TIeISPiZgfWgGK4iaUPuhJsOaRCmEdJin3RjbiT51matmy33FQcXe3pWV4eCZBu25-nhnoojpaQd2_IvhQk3xavV-hpcmGUmdbKSwrZWmaUq1p44GfMrBWlIQwtv18D3vl/IBfARFvD8cPTDFWuiq5FIZacaA4SGUb8YPrQfOwRXOEv4gaxuqO9PZRFnGSnt+w07B8fTbEjzFA==; li_at=AQEDAVELYs0D64sYAAABkOPiB_kAAAGRB-6L-VYA0sa3YMLjl1OG2wF2S0zD22SGBRT-xC9iDUQnS9y8pZLrZ62V0DtyrJZ4rQuZKqvQOzt0G-oAF2EU0BMFxJn0f7WKSh1fw6ripssi4JOqILzQwPv-; liap=true; JSESSIONID="ajax:8194873957528181257"; timezone=Asia/Shanghai; li_theme=light; li_theme_set=app; li_sugr=275ee335-362f-4592-9026-5293e9fd7a00; AnalyticsSyncHistory=AQIWKDlQU0QFVgAAAZDj5HPUbT5PB-XvfQ3X4x0-0wLKdqfCnjOYb7HGkGfGqH0MlQMOTgFQh0f3XkmHKFcKWg; lms_ads=AQFCrXkAYZpN8AAAAZDj5HVRS-ziy7_NbBkaFkRjou7LCw2WamwfMyHIZW4ib-ZVw3-DpUX13rLgJTHJUzsB3jXueE2BxyRi; lms_analytics=AQFCrXkAYZpN8AAAAZDj5HVRS-ziy7_NbBkaFkRjou7LCw2WamwfMyHIZW4ib-ZVw3-DpUX13rLgJTHJUzsB3jXueE2BxyRi; dfpfpt=c173e8b97d4e405db906d223efd32ed7; lang=v=2&lang=zh-cn; AMCVS_14215E3D5995C57C0A495C55%40AdobeOrg=1; AMCV_14215E3D5995C57C0A495C55%40AdobeOrg=-637568504%7CMCIDTS%7C19929%7CMCMID%7C87782939781986201082032379922159003965%7CMCAAMLH-1722470928%7C3%7CMCAAMB-1722470928%7C6G1ynYcLPuiQxYZrsz_pkqfLG9yMXBpb2zX5dvJdYQJzPXImdj0y%7CMCOPTOUT-1721873328s%7CNONE%7CvVersion%7C5.1.1%7CMCCIDH%7C667803310; fptctx2=taBcrIH61PuCVH7eNCyH0AHEYHVht29NHm46S5qgUjb2ukO%252fUw%252fonkY0srSkYcf%252fghhmqB%252fQDHPUQVLSA8KZv2qZJvtN6zLa5n2uVmYhcvicLNu0yk7%252fXKKj3zacMdilpae2ERQuYpfZfvdvz9IyNnViNsyj551BLWYoha3NmpyBtDpbsXGVUIlPvMc2gKwSfHo3tURBtLo%252bW2Y82ArDvaYp78xZc1bqhRRRbnAjer7F8fK1hszoQ7uDLxstViMYSmJ40Qg%252fFIJ2qSSwrI94w1dRHWlTezsIaT1%252frt1GBFKHmTE7Oempw7lxCtzi5SMeBAaR2p9TcF57%252fGkH7SOt%252bN0jO3ukqbTIoIh5RtRfWuU%253d; UserMatchHistory=AQLSpnkrkn2XaQAAAZDnOe7on68q6UJBffAnu77n6LP3jOEPAhIORf-nzVCisDid3VJaFMOmHpiUg96Yl470yqJ41d_IRiMu_4Rhs8oJoOnaaWZi_kIp9w8IgTP3GeN4MJf249j-d24mU7wVmcxhMQsxYeahV-4vgVNPS02wo1iYZ0e0ivCJu8f_srDqOSbJ-lV9UXWe3vKsZGIeNebbf0det-VObQ3knJeQOV3hCWwiQrX9YcRvrI6AjP6RJGdbBQGJ06_kCuUAUjLvbJWfUq7lpk0hLFcUF02HDnKHDRgz1CerAjTe6y5AzwWEMgdSWvxZXjosCbasFe-uvH6P6MOY1uqxhE3PYVclv7oykwE6V3g9ug; lidc="b=OB85:s=O:r=O:a=O:p=O:g=4998:u=2:x=1:i=1721866254:t=1721952504:v=2:sig=AQH4wsDthsfC9SzS4VxMr_ijktR_2oWg"',
'csrf-token': 'ajax:8194873957528181257',
'priority': 'u=1, i',
'referer': 'https://www.linkedin.com/jobs/search/?currentJobId=3964406119&distance=25.0&geoId=91000015&keywords=sustainability&origin=HISTORY',
'sec-ch-ua': '"Not/A)Brand";v="8", "Chromium";v="126", "Google Chrome";v="126"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Windows"',
'sec-fetch-dest': 'empty',
'sec-fetch-mode': 'cors',
'sec-fetch-site': 'same-origin',
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36',
'x-li-lang': 'zh_CN',
'x-li-page-instance': 'urn:li:page:d_flagship3_search_srp_jobs;NZuf40jfRN6zchx6V5IdWQ==',
'x-li-pem-metadata': 'Voyager - Careers - Job Details=job-posting',
'x-li-track': '{"clientVersion":"1.13.20377","mpVersion":"1.13.20377","osName":"web","timezoneOffset":8,"timezone":"Asia/Shanghai","deviceFormFactor":"DESKTOP","mpName":"voyager-web","displayDensity":1,"displayWidth":1920,"displayHeight":1080}',
'x-restli-protocol-version': '2.0.0',
}
geturl='https://www.linkedin.com/jobs/search/?currentJobId=3961526898&distance=25.0&geoId=91000015&keywords=sustainability&origin=HISTORY'
bro.get(geturl)
print(bro.url)
data = []
end_page = 5
datastart=0
cookies =bro.cookies()
session = requests.Session()
for cookie in cookies:
session.cookies.set(cookie['name'], cookie['value'])
for page in range(1, end_page + 1): # 爬取1到40页的数据
if page>1:
datastart += 25
bro.get(f'{geturl}&start={datastart}')
time.sleep(3)
print(f"正在爬取第 {page} 页...")
try:
title_elements=bro.ele('.scaffold-layout__list-container').eles('tag:li')
for title_element in title_elements:
if title_element.attr('data-occludable-job-id'):
job_data={}
try:
occludableid=title_element.attr('data-occludable-job-id')
response = session.get(
f'https://www.linkedin.com/voyager/api/jobs/jobPostings/{occludableid}?decorationId=com.linkedin.voyager.deco.jobs.web.shared.WebFullJobPosting-65&topN=1&topNRequestedFlavors=List(TOP_APPLICANT,IN_NETWORK,COMPANY_RECRUIT,SCHOOL_RECRUIT,HIDDEN_GEM,ACTIVELY_HIRING_COMPANY)',
headers=headers)
response.encoding = 'utf-8' # 设置响应编码
jsonstr = response.text.encode('gbk', errors='replace').decode('gbk')
jsonstr = json.loads(jsonstr)
#职位名称
job_name=jsonstr['data']['title']
#公司名称
company_name = get_company_name(jsonstr)
#公司地址
company_location=jsonstr['data']['formattedLocation']
#公司规模
try:
comstart=jsonstr['included'][-1]['staffCountRange']['start']
try:
comstend=jsonstr['included'][-1]['staffCountRange']['end']
except Exception as e:
comstend='+'
company_scale=f'{comstart}-{comstend}'
if comstend=='+':
company_scale=f'{comstart}+'
except:
comstart=''
#行业分类
industry=jsonstr['data']['formattedIndustries']
#发布时间
datatime=int(jsonstr['data']['listedAt'])/1000
datatime = datetime.fromtimestamp(datatime)
# 格式化日期
datatime = datatime.strftime('%Y-%m-%d')
#申请人数
applicants=jsonstr['data']['applies']
#职位介绍
job_description=jsonstr['data']['description']['text']
#职位链接
job_href=jsonstr['data']['jobPostingUrl']
print(f'职位名称：{job_name}')
print(f'公司名称：{company_name}')
print(f'公司地址：{company_location}')
print(f'公司规模：{company_scale }')
print(f'行业分类：{industry}')
print(f'发布时间：{datatime}')
print(f'申请人数：{applicants}')
print(f'职位介绍：{job_description}')
print('-' * 100)
# 将数据添加到列表中
job_data['职位名称'] = job_name
job_data['公司名称'] = company_name
job_data['公司地址'] = company_location
job_data['发布时间'] = datatime
job_data['申请人数'] = applicants
job_data['职位介绍'] = job_description
job_data['职位链接'] = job_href
data.append(job_data)
except Exception as e :
pass;
except Exception as e:
print(f"爬取数据时出现异常: {e}")
traceback.print_exc() # 输出完整的异常信息，包括错误行号
print('爬取完成！')
df = pd.DataFrame(data)
df = df.drop_duplicates(subset=['职位名称', '公司名称', '公司地址', '发布时间', '申请人数', '职位介绍'])
print(f'总共获取了 {len(df)} 条信息.')
df.to_excel(f'Linkedln_{current_date}.xlsx', index=False)
print(f'数据已保存为 Linkedln_{current_date}.xlsx 文件.')
bro.quit()