• 【自用存档】bs4爬虫代码

    只需更改相应的前端寻找代码部分(下为丐版)

    1. from bs4 import BeautifulSoup
    2. import urllib.request
    3.  
    4.  
    5. def getHtml(url):
    6.     resp = urllib.request.urlopen(url)
    7.     data = resp.read()
    8.     return data.decode("gbk")
    9.  
    10. def getOnePage(url):
    11.     nextUrl = ""
    12.     try:
    13.         html = getHtml(url)
    14.         soup = BeautifulSoup(html,"html.parser")
    15.         lis = soup.find("div",attrs = {"class":"con shoplist"}).find_all("li")
    16.   
    17.         for li in lis:
    18.             title = li.find("a")["title"]
    19.             author = li.find("p",attrs={"class":"search_book_author"}).find("a")['title']
    20.             time = li.find("p",attrs={"class":"search_book_author"}).find_all("span")[1].text
    21.             publisher = li.find("p",attrs={"class":"search_book_author"}).find_all("span")[2].text
    22.             
    23.             #author publisher puddate brief price
    24.             print("书名:",title)
    25.             print("作者:",author)
    26.             print("出版社：",publisher)
    27.             print("出版时间",time)
    28.             #print("简历")
    29.             #print("金额")
    30.             print("   ")
    31.         
    32.         
    33.     except Exception as err:
    34.         print(err)
    35.     return nextUrl
    36.  
    37.  
    38. url= "http://search.dangdang.com/?key=python&act=input&page_index=1"
    39. nextUrl = getOnePage(url)
    40.  
    41.  
    42.  
    43.  
    44.  
    45.    
    46.  
    47.

    以豆瓣top250为例子

    1. from shutil import move
    2. from bs4 import BeautifulSoup
    3. import urllib.request
    4. import time
    5. import pandas as pd 
    6. #import response
    7. '''
    8. 爬取单个电影需要信息内容
    9. '''
    10.  
    11. '''检查改网页是否可以爬取'''
    12. def getHtml(url):
    13.     proxy_ip='127.0.0.1:8888'
    14.     try:
    15.         req = urllib.request.Request(url)
    16.         # 对付网站反爬虫程序，有了以下的请求头文
    17.         req.add_header('User-Agent','Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0')
    18.         
    19.         # 此类是 URL 请求的抽象。
    20.         proxy=urllib.request.ProxyHandler({'http':proxy_ip})
    21.         # 返回一个OpenerDirector实例，它按照给定的顺序链接处理程序。handler可以是 的实例，也可以是 的BaseHandler子类
    22.         opener=urllib.request.build_opener(proxy,urllib.request.HTTPHandler)
    23.         # 安装一个OpenerDirector实例作为默认的全局开启器。
    24.         urllib.request.install_opener(opener)
    25.         # 打开 URL url，它可以是字符串或 Request对象。
    26.         data=urllib.request.urlopen(req).read().decode('utf-8','ignore')
    27.         
    28.         return data
    29.         '''
    30.         except urllib.error.URLError as e:
    31.             if hasattr(e,'code'):
    32.                 print(e.code)
    33.             if hasattr(e,'reason'):
    34.                 print(e.reason)
    35.             time.sleep(10) #出现异常延时10s
    36.         '''
    37.     except Exception as e:
    38.         print('exception:'+str(e))
    39.         time.sleep(1)
    40.  
    41.  
    42.  
    43. '''获取每个电影页面的内容【名称+简介】'''
    44. def getOnePage(url):
    45.     try:
    46.         # 调用可否爬取函数
    47.         html = getHtml(url)
    48.  
    49.         # 调用bs对象来实现爬取 
    50.         soup = BeautifulSoup(html,"lxml")
    51.  
    52.         '''
    53.         1. 爬取内容 
    54.             html嵌套如：related-info -> indent->all hidden
    55.             判错处理：有些简介太长，需要展开显示，
    56.                 a. 先爬起隐藏内容，
    57.                 b. 如果为空，则直接爬取页面未被隐藏内容
    58.         '''
    59.         # 前缀太长，所以分段爬取
    60.         movie_content_pre = soup.find("div", attrs={"class":"related-info"}).find("div",attrs={"class":"indent"})
    61.       
    62.         # 简介的判错处理
    63.         try:
    64.             # movie_content = movie_content_pre.find("span",attrs={"class":"all hidden"}).text
    65.             movie_content = movie_content_pre.find("span",attrs={"class":"all hidden"}).text
    66.         except:
    67.             # movie_content = movie_content_pre.find("span",attrs={"property":"v:summary"}).text
    68.             movie_content = movie_content_pre.find("span",attrs={"property":"v:summary"}).text
    69.         print("all",movie_content)
    70.     
    71.         
    72.         movie_content = movie_content.strip()
    73.         movie_content = movie_content.replace('\u3000', '').replace('\r', '')
    74.         movie_content = movie_content.replace("\n","").replace("  ","")
    75.         print("     1.movie content done")
    76.         
    77.       
    78.         '''
    79.         2. 爬取内容+类型
    80.             因为两者在同一个 
      下，
    81.             变量: information -> 存储content的内容
    82.         '''
    83.         '''
    84.         information = soup.find("div",attrs={"id":"content"})
    85.         # 一部电影包含多个标签，先用movie_type_all 包含所有标签内容
    86.         movie_type_all = information.find("div",attrs={"id":"info"}).find_all("span",attrs={"property":"v:genre"})
    87.         # 创建列表存储 类别 内容
    88.         movie_type = []
    89.         # 按照所有标签内容，遍历得到单个标签
    90.         for i in range(len(movie_type_all)):
    91.             types = movie_type_all[i].text
    92.             movie_type.append(types)
    93.         
    94.         # 为写入文件而转换格式str
    95.         movie_type = str(movie_type)
    96.         
    97.         print("     2.movie type done")
    98.         '''
    99.       
    100.         # 查找电影名称
    101.         movie_title = soup.find("div",attrs={"id":"content"}).find("span",attrs={"property":"v:itemreviewed"}).text
    102.         print("     2.movie title done")
    103.       
    104.         # print("电影名称\n",movie_title)
    105.         # print("电影简介\n",movie_type)
    106.         # print("电影内容\n",movie_content)    
    107.     
    108.  
    109.     # 判错 遇错处理
    110.     except Exception as err:
    111.         print(err)
    112.  
    113.  
    114.     # 返回数据
    115.     return movie_title,movie_content
    116.  
    117.  
    118. # # 检查某个电影页面内容是否爬取成功
    119. url = "https://movie.douban.com/subject/1292001/"
    120. getOnePage(url)
    121.  
    122. data = getOnePage(url)
    123. with open("check.txt","w") as f:
    124.     f.write(str(data))

    1. #   排行榜爬取下一个网页的内容（跳转爬取）
    2. '''爬取下一个页面的所有链接内容'''
    3. from one_page_film import a_page_to_all_links
    4.  
    5. def get_All_Pages_Links():
    6.     try:
    7.         # 参考学习链接：https://zhuanlan.zhihu.com/p/62601606
    8.         # 自定义url实现网址爬取
    9.         url_style = "https://movie.douban.com/top250?start={index}&filter="
    10.         url_lst = []
    11.         # 0-255 步长为25 
    12.         for i in range(0, 250, 25):
    13.             url = url_style.format(index=i)
    14.             url_lst.append(url)
    15.        
    16.     except Exception as err:
    17.         print(err)
    18.     return url_lst
    19.     
    20. # 存储10页的内容链接
    21. url_list = get_All_Pages_Links()
    22. # 环环嵌套 由页码链接得到每一页的电影链接
    23. for i in url_list:  
    24.     # flag  
    25.     print("No: pages",i)
    26.     a_page_to_all_links(i)
    27.     
    1. # 在某一页排行榜爬取每个电影的链接
    2. from bs4 import BeautifulSoup
    3. import requests
    4. # 从spider文件调用函数
    5. from one_page_content import getHtml,getOnePage
    6.  
    7. '''对该页面的所有电影链接进行爬取'''
    8. def getPageLink(url):
    9.     try:
    10.         # 调用函数
    11.         html = getHtml(url)
    12.         # 调用bs对象来实现爬取 
    13.         soup = BeautifulSoup(html,"lxml")
    14.  
    15.         # 一个电影名称在一个li标签里面
    16.         all_links = soup.find("ol",attrs={"class":"grid_view"}).find_all("li")
    17.         
    18.         # 变量movie_link_list用来存储各个电影链接
    19.         movie_link_list = []
    20.         # 循环遍历li标签得到各个电影的链接
    21.         for i in range(len(all_links)):
    22.             link = all_links[i].find("div",attrs={"class":"pic"}).find("a")['href']
    23.             movie_link_list.append(link)
    24.             
    25.         # print("存储在list的链接:\n",movie_link_list)
    26.        
    27.     except Exception as err:
    28.         print(err)
    29.  
    30.     return movie_link_list
    31.  
    32.  
    33.  
    34. def a_page_to_all_links(url):
    35.     # 一个页面的所有内容 存储在one_pages_content
    36.     one_pages_content = []
    37.     # 一个页面中所有的电影链接存储在one_page_link
    38.     one_page_link = getPageLink(url)
    39.     # flag 
    40.     epoch = 0
    41.     for i in range(len(one_page_link)):
    42.         epoch = epoch + 1
    43.         print(" epoch:",epoch)
    44.  
    45.         # 调用函数得到单个电影的内容
    46.         one_movie_content = getOnePage(one_page_link[i])
    47.         
    48.         # print(one_movie_content)
    49.         print("name",one_movie_content[0])
    50.         print("content",one_movie_content[1])
    51.         
    52.         # 写入语料库
    53.         with open("data/all_content_corpus.txt",'a',encoding="utf-8") as f:  
    54.             f.write(one_movie_content[0])
    55.             f.write(",")     
    56.             f.write(one_movie_content[1])
    57.             f.write("\n")
    58.             
    59.        
    60. # url = "https://movie.douban.com/top250?start=0&filter="   
    61. # a_page_to_all_links(url)
    62.  
    63.

  • 相关阅读:
    通过git命令查询某个用户提交信息
    dwg如何转换成pdf，cad快速转pdf
    iNFTnews | 喝茶送虚拟股票？浅析奈雪的茶“发币”
    linux权限详解
    docker pull、docker load、docker run使用方法
    海康相机SDK二次开发C++程序
    D. Boris and His Amazing Haircut
    C#，人工智能，深度学习，OpenCV开发，入门教程——Visual Studio 2022，OpenCvSharp环境搭建与可视化
    2022河南萌新联赛第（七）场：南阳理工学院 H-防风台
    SWC ports and port interface
  • 原文地址：https://blog.csdn.net/dannnnnnnnnnnn/article/details/126367927