目的

需要收集京东的地址信息省市区

# encoding:utf-8
# FileName: AreaCity
# Date:     2022/07/22 11:55
# Author:   数据闲逛人
# CSDN：   数据闲逛人
# Description: 爬取网页版京东地址填写的省市区
import pandas as pd
import numpy as np 
from selenium import webdriver
import psycopg2
from tqdm import tqdm
import pdb     
import os
import pdb

import warnings
warnings.filterwarnings("ignore")
from selenium.webdriver.common.by import By
import time

options = webdriver.ChromeOptions()
options.add_experimental_option('useAutomationExtension', False)
options.add_experimental_option('excludeSwitches', ['enable-automation'])
driver = webdriver.Chrome(chrome_options=options)
driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
    "source": """
    Object.defineProperty(navigator, 'webdriver', {
      get: () => undefined
    })
  """
})

driver.get('https://trade.jd.com/shopping/order/getOrderInfo.action?rid=1658384700764#none')
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33

爬取数据

def get_city_count():
    """
    获取省份下的城市数量
    """
    # 获取城市数量
    city_count = len(driver.find_element(By.XPATH,'//*[@id="jd_area"]/div[2]/div[2]/div[2]/ul').text.split('\n'))
    time.sleep(0.2)
    return city_count

def get_county_imformation():
    """
    获取区|县信息
    """
    # 获取区|县所属的城市
    province = driver.find_element(By.XPATH,'//*[@id="jd_area"]/div[2]/div[1]/a[1]').text
    print(province)
    # 获取区|县所属的城市
    city = driver.find_element(By.XPATH,'//*[@id="jd_area"]/div[2]/div[1]/a[2]/em').text
    print(city)
    # 获取区|县信息
    county_list = driver.find_element(By.XPATH,'//*[@id="jd_area"]/div[2]/div[2]/div[3]/ul').text.split('\n')
    time.sleep(0.1)
    # 计算区|县数据
    county_count = len(county_list) 
    city_all_county_data = pd.DataFrame({'province':[province]*county_count,'city':[city]*county_count,\
                                         'county':county_list})
    return city_all_county_data

def judge_province(province):
    """
    判断该省份是否继续爬取
    @param province:传进来的省份名称
    """
    #数据探索
    #1 获取所有文件的名称和路径
    csv_name = ""
    data_path = '../output/京东数据/'  #数据存储路径     ！！！！！按需修改这里
    data_names = os.listdir(data_path)  #列举出data_path里面所有的文件名称
    data = pd.DataFrame([])
    for data_name in data_names:
        # 删除不是的文件
        if data_name.endswith('.csv'): #  ！！！！不同各格式  按需修改
            csv_name = csv_name+data_name[:-4]+"|"
    csv_name = csv_name[:-1]
    if province not in csv_name:
        return True    
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46

iframe = driver.find_element_by_xpath('//*[@id="dialogIframe"]')
# 切换到iframe
driver.switch_to.frame(iframe)
1
2
3

hover_element=driver.find_element_by_xpath('//*[@id="jd_area"]/div[1]/div')
#移动鼠标到元素上触发事件
actions = ActionChains(driver)
actions.move_to_element(hover_element)
actions.perform()
#ActionChains(driver).move_to_element(hover_element).perform()
1
2
3
4
5
6

for i in range(1,34):
    data = pd.DataFrame({'province':[],'city':[],'county':[]})
    # 重新点击一下省份这里位置
    driver.find_element(By.XPATH,'//*[@id="jd_area"]/div[2]/div[1]/a[1]/em').click()
    time.sleep(0.2)
    # 点击省份
    driver.find_element(By.XPATH,'//*[@id="jd_area"]/div[2]/div[2]/div[1]/ul/li[{}]/a'.format(i)).click()
    time.sleep(0.2)
    province = driver.find_element(By.XPATH,'//*[@id="jd_area"]/div[2]/div[1]/a[1]').text
    time.sleep(0.2)
    # 获取某省份下面的数量
    city_count = get_city_count()
    if judge_province(province):
        for j in range(1,city_count+1):
            # 点击城市
            time.sleep(0.2)
            driver.find_element(By.XPATH,'//*[@id="jd_area"]/div[2]/div[2]/div[2]/ul/li[{}]/a'.format(j)).click()
            time.sleep(0.2)
            # 获取区|县列表
            city_all_county_data = get_county_imformation()
            data = data.append(city_all_county_data)
            driver.find_element(By.XPATH,'//*[@id="jd_area"]/div[2]/div[1]/a[2]'.format(j)).click()
        data.to_csv('../output/京东数据/'+province+'.csv',index=False)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

重庆
万州区
重庆
涪陵区
重庆
梁平区
重庆
南川区
重庆
潼南区
1
2
3
4
5
6
7
8
9
10

最后收集到的数据

里面的文件内容