爬虫常用句法

无头浏览器

options = webdriver.ChromeOptions()
    options.add_argument('--headless')
    options.add_argument('--disable-gpu')
    driver = webdriver.Chrome(
        executable_path='C:\\Program Files\\Google\\Chrome\\Application\\105.0.5195.127\\chromedriver',chrome_options=options)#,chrome_options=options
    
1
2
3
4
5
6

等到某元素可见

# 一直等待某元素可见，默认超时10秒
def is_visible(driver,locator, timeout=10):
    try:
        ui.WebDriverWait(driver, timeout).until(EC.visibility_of_element_located((By.XPATH, locator)))
        return True
    except TimeoutException:
        return False
        
1
2
3
4
5
6
7
8

 if is_visible(driver,'//*[@id="copyText"]'):
        driver.find_element_by_xpath('//*[@id="copyText"]').click()
1
2

多个池-多线程

# 遍历文件夹，找到文件所在路径
class findPath:
    def __init__(self):
        self.fileList = []

    def gci(self, filepath):
        files = os.listdir(filepath)
        for fi in files:
            fi_d = os.path.join(filepath, fi)
            if os.path.isdir(fi_d):
                self.gci(fi_d)
            else:
                self.fileList.append(fi_d)

    def getAllPaths(self, filepath):
        self.gci(filepath)
        return self.fileList
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17

from multiprocessing.dummy import Pool as ThreadPool
pool = ThreadPool(3)
    flist = findPath().getAllPaths(root_dir)

    results = pool.map(Get_singleimg, flist)
1
2
3
4
5