猎聘爬虫(附源码)

废话不多说直接附源码

cookies需要替换成自己的 , 该网站在不登录的情况下只能请求到10页数据 , 想要获得完整数据需要携带登录后的cookies

import requests
import json
from lxml import etree
import os
import openpyxl




headers = {
    "Accept": "application/json, text/plain, */*",
    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6",
    "Cache-Control": "no-cache",
    "Connection": "keep-alive",
    "Content-Type": "application/json;charset=UTF-8",
    "Origin": "https://www.liepin.com",
    "Pragma": "no-cache",
    "Referer": "https://www.liepin.com/",
    "Sec-Fetch-Dest": "empty",
    "Sec-Fetch-Mode": "cors",
    "Sec-Fetch-Site": "same-site",
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36 Edg/122.0.0.0",
    "X-Client-Type": "web",
    "X-Fscp-Bi-Stat": "{\"location\": \"https://www.liepin.com/zhaopin/?inputFrom=head_navigation&scene=init&workYearCode=0&ckId=jrkiappybgyczm7c2sk5zmfzwgpqpqia\"}",
    "X-Fscp-Fe-Version": "",
    "X-Fscp-Std-Info": "{\"client_id\": \"40108\"}",
    "X-Fscp-Trace-Id": "f22eb671-3c8f-4f94-8b14-e5e7d176be52",
    "X-Fscp-Version": "1.1",
    "X-Requested-With": "XMLHttpRequest",
    "X-XSRF-TOKEN": "hCnGTNiJQfe47qu4x2OChA",
    "sec-ch-ua": "\"Chromium\";v=\"122\", \"Not(A:Brand\";v=\"24\", \"Microsoft Edge\";v=\"122\"",
    "sec-ch-ua-mobile": "?0",
    "sec-ch-ua-platform": "\"Windows\""
}


def spiderData():
    # 循环每一页
    for i in range(1, 21):
        # 配置账号参数
        cookies =  ''
        # 网页链接
        url = "https://api-c.liepin.com/api/com.liepin.searchfront4c.pc-search-job"
        # 参数
        data = {
            "data": {
                "mainSearchPcConditionForm": {
                    "city": "410",
                    "dq": "410",
                    "pubTime": "",
                    "currentPage": f"{i}",
                    "pageSize": 40,
                    "key": "",
                    "suggestTag": "",
                    "workYearCode": "0",
                    "compId": "",
                    "compName": "",
                    "compTag": "",
                    "industry": "H01$H01",
                    "salary": "",
                    "jobKind": 2,
                    "compScale": "",
                    "compKind": "",
                    "compStage": "",
                    "eduLevel": ""
                },
                "passThroughForm": {
                    "scene": "init",
                    "ckId": "0nbwaavz2gngc40f8xmfp59in6ymulua",
                    "skId": "pf8wezdo0ezilzl4tyd1g4tcoyh43qe9",
                    "fkId": "0nbwaavz2gngc40f8xmfp59in6ymulua",
                    "suggest": None
                }
            }
        }
        data = json.dumps(data, separators=(',', ':'))
        response = requests.post(url, headers=headers, cookies=cookies, data=data).json()

        print(f"正在爬取第{i + 1}页")
        praseData(response)


# 这段代码主要用于解析和保存来自招聘网站的职位信息。下面是加上注释后的代码：
def praseData(data):
    for z in range(0,40):
        job_card_list = data.get('data', {}).get('data', {}).get('jobCardList', [])
        if 0 <= z < len(job_card_list):
            res_json_item = job_card_list[z]

            # 公司名称
            comp_name = res_json_item.get('comp', {}).get('compName')

            # 职位链接
            job_link = res_json_item.get('job', {}).get('link')

            # 工作地点
            place = res_json_item.get('job', {}).get('dq')

            # 薪资
            salary = res_json_item.get('job', {}).get('salary')

            # 职位名称
            job = res_json_item.get('job', {}).get('title')
        else:
            # 如果z不是有效索引或job_card_list为空，则处理错误或设置默认值
            comp_name = None
            job_link = None
            place = None
            salary = None
            job = None


            # 解析职位详情页面
        sub_data = requests.get(job_link, headers=headers).text
        # 使用 etree 解析 HTML 数据
        xml = etree.HTML(sub_data)
        # 尝试从详情页面中提取公司简介
        try:
            details = xml.xpath('//dl[@class="paragraph"]/dd/text()')[0]
        except:
            details = None

        # 公司简介
        companyProfile = xml.xpath("//div[@class='paragraph-box']/div/text()")
        company_profile = ','.join(companyProfile)

        # 公司信息

        try:
            intorduct = details.split('截止日期')[0].split()
            intorducts = ','.join(intorduct)
        except:
            intorducts = None

        # 保存到 excle 表格
        job_list = [job,place,salary,comp_name,company_profile,intorducts]
        print(job_list)
        save_data_to_xlsx(job_list)

# 保存到excle表格
def save_data_to_xlsx(data ):

    filename = f'job.xlsx'
    name_headers = ['职位', '地点', '薪资', '公司名称', '公司简介','描述']
    if os.path.exists(filename):
        workbook = openpyxl.load_workbook(filename)
        sheet = workbook.active
        sheet.append(data)
    else:
        workbook = openpyxl.Workbook()
        sheet = workbook.active
        # 添加表头
        sheet.append(name_headers)
        sheet.append(data)
    # 保存 Excel 文件
    workbook.save(filename)



if __name__ == '__main__':
    spiderData()


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163