python爬虫练手项目之获取某地企业名录

因为很多网站都增加了登录验证，所以需要添加一段利用cookies跳过登陆验证码的操作

import pandas as pd
import requests
from lxml import etree
# 通过Chrome浏览器F12来获取cookies，agent，headers
cookies ={'ssxmod_itna2':'eqfx0DgQGQ0QG=DC8DXxxxxx',
	  'ssxmod_itna':'euitGKD5iIgGxxxxx'}
agent ='Mozilla/5.0 (Windows NT 10.0; Win64; x64)xxxxxxx'
 
headers = {
    'User-Agent' : agent,
    'Host':'www.xxx.com',
    'Referer':'https://www.xxx.com/'
}
#建立会话
session = requests.session()
session.headers = headers

cookies获取方式

chrmoe浏览器，F12，把name和value填入cookies

agent获取方式

任意点击一条网络资源，右侧headers往下翻到底

测试访问是否成功

#↓此处测试访问是否成功，成功的话返回码200
requests.utils.add_dict_to_cookiejar(session.cookies, cookies)
url = 'https://www.xxx.com/search-prov/36/3604/p1' 
response=session.get(url)
print(response)

访问成功的话进入下一步

一般翻页后查看网址变化就能得出网址规则

#初始化df数据
df = pd.DataFrame(columns = ['企业名称'])
 
#观察翻页后网址变化规律，取10页数据
for k in range(10): 
	url = 'https://www.xxx.com/search-prov/36/3604/p' + str(k+1) + '/' 
	cookies_dict = requests.utils.add_dict_to_cookiejar(session.cookies, cookies)
	page_text = requests.get(url, headers = headers, cookies = cookies_dict).text # GET
	#print(page_text)
	tree = etree.HTML(page_text) #数据解析
	#取到企业名对应xpath
	name = [i for i in tree.xpath("//div[@class='company-title font-18 font-f6']/a/text()")]
	dic = {'企业名称':name}
	df1 = pd.DataFrame(dic)
	df = pd.concat([df,df1], axis=0)
	#print(df)
print('全部数据爬取成功')
print(df)

最后将结果导入csv文件；编码格式utf-8-sig防止乱码

#将df数据写入csv文件
df.to_csv('xx企业名录.csv',index=None,encoding = 'utf-8-sig')