• 爬虫-打包整个小说网站

    使用Python爬取小说章节并存储到MongoDB数据库

    1. 背景介绍

    在本篇文章中，我们将展示如何使用Python爬取小说网站的章节内容并将其存储到MongoDB数据库中。我们会使用requests进行网页请求，lxml处理HTML，re进行正则表达式匹配，threading实现并发处理，以及pymongo连接MongoDB。

    真实情况是写课设没数据，获取取一点

    2. 环境和依赖

    请确保你的环境已经安装了以下依赖包：

    pip install requests lxml pymongo

    3. 代码详解

    下面是完整的代码实现。我们将按照功能模块进行解释。

    3.1 引入必要的库

    首先，我们需要引入所需的库：

    1. import requests
    2. from lxml import etree
    3. import re
    4. import threading
    5. import time
    6. from pymongo import MongoClient
    3.2 设置请求头

    为了防止被网站封禁，我们设置了请求头：

    1. headers = {
    2.     'User-Agent': '写你自己的'
    3. }
    3.3 初始化全局变量和信号量

    为了控制并发和存储章节内容，我们设置了全局变量和信号量：

    1. zhangjie_content = []  # 存储章节内容
    2. semaphore = threading.Semaphore(20)  # 限制并发数量为20
    3.4 MongoDB连接设置

    连接到MongoDB数据库

    1. client = MongoClient('mongodb://localhost:27017/')
    2. db = client.novel_database
    3. collection = db.novels
    3.5 插入数据到MongoDB

    定义一个辅助函数，将数据插入到MongoDB：

    1. def insert_to_mongodb(title, novel_type, author, update_time, chapters):
    2.     data = {
    3.         "title": title,
    4.         "novel_type": novel_type,
    5.         "author": author,
    6.         "update_time": update_time,
    7.         "zhangjie": chapters
    8.     }
    9.     collection.insert_one(data)
    10.     print(f"插入 {len(chapters)} 章成功：{title}")
    3.6 爬取章节内容

    定义爬取章节内容的函数：

    1. def neirong(ur, url, s, retries=3):
    2.     while retries > 0:
    3.         try:
    4.             reps = requests.get(rf'{ur}{url}', headers=headers)
    5.             reps.raise_for_status()
    6.             html = etree.HTML(reps.text)
    7.             if html is None:
    8.                 print(f"解析 HTML 内容错误，URL: {ur}/{url}")
    9.                 return
    10.  
    11.             chapter = html.xpath('//*[@id="content"]/h1/text()')
    12.             if not chapter:
    13.                 print(f"未找到章节标题，URL: {ur}/{url}")
    14.                 return
    15.  
    16.             chapter = chapter[0].strip()
    17.             text = html.xpath('//*[@id="htmlContent"]/text()')
    18.             if not text:
    19.                 print(f"未找到章节内容，URL: {ur}/{url}")
    20.                 return
    21.  
    22.             text = ''.join(text[1:])  # 连接文本内容
    23.             zhangjie_content.append({"chapter": chapter, "text": text})
    24.  
    25.             return
    26.         except requests.RequestException as e:
    27.             print(f"请求错误，URL: {ur}{url}, 错误: {e}")
    28.             retries -= 1
    29.             time.sleep(1)  # 等待一段时间后重试
    30.     print(f"重试次数过多，放弃 URL: {ur}{url}")
    3.7 爬取章节列表

    定义爬取章节列表的函数：

    1. def zhangjie(url, retries=3):
    2.     while retries > 0:
    3.         try:
    4.             reps = requests.get(url, headers=headers, timeout=10)
    5.             reps.raise_for_status()
    6.             html = etree.HTML(reps.text)
    7.             if html is None:
    8.                 print(f"解析 HTML 内容错误，URL: {url}")
    9.                 return
    10.  
    11.             title = html.xpath('//*[@id="info"]/h1/text()')
    12.             title = title[0].strip() if title else "未知书名"
    13.             novel_type = html.xpath('//*[@id="maininfo"]/div[1]/a[2]/text()')
    14.             novel_type = novel_type[0].strip() if novel_type else "未知类型"
    15.             author = html.xpath('//*[@id="info"]/p[1]/a/text()')
    16.             author = author[0].strip() if author else "未知作者"
    17.             update_time = html.xpath('//*[@id="info"]/p[3]/text()')
    18.             update_time = update_time[0].strip() if update_time else "未知时间"
    19.  
    20.             option_texts = html.xpath('/html/body/div[4]/div/div/select/option/text()')
    21.             if not option_texts:
    22.                 print(f"未找到页码信息，URL: {url}")
    23.                 return
    24.  
    25.             zhang = re.findall(r'第\s*(\d+)\s*页\(末页\)', option_texts[-1])
    26.             if not zhang:
    27.                 print(f"未找到页码匹配，URL: {url}")
    28.                 return
    29.             zhang = int(zhang[0])
    30.             print('开始爬取:', title)
    31.  
    32.             s = 0  # 设置爬取多少章
    33.             for i in range(1, zhang + 1):
    34.                 if s >= 100:
    35.                     break  # 已经爬取100章，跳出循环
    36.  
    37.                 zhangjie_url = f'{url}/index_{i}.html'
    38.                 zhangjie_reps = requests.get(zhangjie_url, headers=headers, timeout=10)
    39.                 zhangjie_reps.raise_for_status()
    40.                 zhangjie_html = etree.HTML(zhangjie_reps.text)
    41.                 if zhangjie_html is None:
    42.                     print(f"解析 HTML 内容错误，URL: {zhangjie_url}")
    43.                     break
    44.  
    45.                 zhangjieLis = zhangjie_html.xpath('/html/body/div[4]/div/ul/li/a/@href')
    46.                 if not zhangjieLis:
    47.                     print(f"未找到章节列表，URL: {zhangjie_url}")
    48.                     break
    49.  
    50.                 threads = []
    51.  
    52.                 for j in zhangjieLis:
    53.                     if s >= 100:
    54.                         break  # 已经爬取100章，跳出循环
    55.                     thread = threading.Thread(target=crawl_with_semaphore, args=(neirong, url, j, s))
    56.                     threads.append(thread)
    57.                     thread.start()
    58.                     time.sleep(0.1)
    59.                     s += 1  # 统计章节数目
    60.  
    61.                 for thread in threads:
    62.                     thread.join()
    63.  
    64.             # 插入所有爬取的章节内容到MongoDB
    65.             insert_to_mongodb(title, novel_type, author, update_time, list(zhangjie_content))
    66.             zhangjie_content.clear()  # 清空章节内容列表
    67.  
    68.             print(f"已成功记录数据：{title}")
    69.  
    70.             return
    71.         except requests.RequestException as e:
    72.             print(f"请求错误，URL: {url}, 错误: {e}")
    73.             retries -= 1
    74.             time.sleep(1)  # 等待一段时间后重试
    75.     print(f"重试次数过多，放弃 URL: {url}")
    3.8 使用信号量控制并发

    定义一个辅助函数，使用信号量控制并发数量：

    1. def crawl_with_semaphore(target, *args):
    2.     with semaphore:  # 使用信号量来控制并发数量
    3.         target(*args)
    3.9 主函数

    定义主函数，从主页爬取小说列表并调用爬取章节的函数：

    1. def main(i):
    2.     main_url = rf'http://www.biqule.net/top/monthvisit/{i}.html'  # 月访问榜主页链接
    3.     try:
    4.         reps = requests.get(main_url, headers=headers, timeout=10)
    5.         reps.raise_for_status()
    6.         html = etree.HTML(reps.text)
    7.         if html is None:
    8.             print("解析 HTML 内容错误，主页 URL")
    9.             return
    10.  
    11.         novels = html.xpath('//div/ul/li/span[@class="sp_2"]/a')
    12.         novel_urls = [{"title": novel.text.strip(), "url": novel.attrib['href']} for novel in novels if
    13.                       novel.text and 'href' in novel.attrib]
    14.         for i in novel_urls:
    15.             global lis
    16.             lis = []
    17.             zhangjie(i['url'])
    18.  
    19.     except requests.RequestException as e:
    20.         print(f"请求错误，URL: {main_url}, 错误: {e}")
    3.10 程序入口

    定义程序入口，并调用主函数：

    1. if __name__ == "__main__":
    2.     for i in range(1, 51): # 1，51是从第一页爬到第五十页
    3.         main(i)

    4. 总结

    通过本文的示例，我们展示了如何使用Python爬取小说网站的章节内容并将其存储到Mongo

    5.完整代码

    1. import requests
    2. from lxml import etree
    3. import re
    4. import threading
    5. import time
    6. from pymongo import MongoClient
    7.  
    8. # 设置请求头部，防止被网站封禁
    9. headers = {
    10.     'User-Agent': '改成你自己的'
    11. }
    12.  
    13. # 全局变量和信号量用于控制并发和存储章节内容
    14. zhangjie_content = []  # 存储章节内容
    15. semaphore = threading.Semaphore(20)  # 限制并发数量为20
    16.  
    17. # MongoDB连接设置
    18. client = MongoClient('mongodb://localhost:27017/')
    19. db = client.novel_database
    20. collection = db.novels
    21.  
    22. def insert_to_mongodb(title, novel_type, author, update_time, chapters):
    23.     """
    24.     辅助函数，用于将数据插入到MongoDB中
    25.     """
    26.     data = {
    27.         "title": title,
    28.         "novel_type": novel_type,
    29.         "author": author,
    30.         "update_time": update_time,
    31.         "zhangjie": chapters
    32.     }
    33.     collection.insert_one(data)
    34.     print(f"插入 {len(chapters)} 章成功：{title}")
    35.  
    36. def neirong(ur, url, s, retries=3):
    37.     while retries > 0:
    38.         try:
    39.             reps = requests.get(rf'{ur}{url}', headers=headers)
    40.             reps.raise_for_status()
    41.             html = etree.HTML(reps.text)
    42.             if html is None:
    43.                 print(f"解析 HTML 内容错误，URL: {ur}/{url}")
    44.                 return
    45.  
    46.             chapter = html.xpath('//*[@id="content"]/h1/text()')
    47.             if not chapter:
    48.                 print(f"未找到章节标题，URL: {ur}/{url}")
    49.                 return
    50.  
    51.             chapter = chapter[0].strip()
    52.             text = html.xpath('//*[@id="htmlContent"]/text()')
    53.             if not text:
    54.                 print(f"未找到章节内容，URL: {ur}/{url}")
    55.                 return
    56.  
    57.             text = ''.join(text[1:])  # 连接文本内容
    58.             zhangjie_content.append({"chapter": chapter, "text": text})
    59.  
    60.             return
    61.         except requests.RequestException as e:
    62.             print(f"请求错误，URL: {ur}{url}, 错误: {e}")
    63.             retries -= 1
    64.             time.sleep(1)  # 等待一段时间后重试
    65.     print(f"重试次数过多，放弃 URL: {ur}{url}")
    66.  
    67.  
    68. def zhangjie(url, retries=3):
    69.     while retries > 0:
    70.         try:
    71.             reps = requests.get(url, headers=headers, timeout=10)
    72.             reps.raise_for_status()
    73.             html = etree.HTML(reps.text)
    74.             if html is None:
    75.                 print(f"解析 HTML 内容错误，URL: {url}")
    76.                 return
    77.  
    78.             title = html.xpath('//*[@id="info"]/h1/text()')
    79.             title = title[0].strip() if title else "未知书名"
    80.             novel_type = html.xpath('//*[@id="maininfo"]/div[1]/a[2]/text()')
    81.             novel_type = novel_type[0].strip() if novel_type else "未知类型"
    82.             author = html.xpath('//*[@id="info"]/p[1]/a/text()')
    83.             author = author[0].strip() if author else "未知作者"
    84.             update_time = html.xpath('//*[@id="info"]/p[3]/text()')
    85.             update_time = update_time[0].strip() if update_time else "未知时间"
    86.  
    87.             option_texts = html.xpath('/html/body/div[4]/div/div/select/option/text()')
    88.             if not option_texts:
    89.                 print(f"未找到页码信息，URL: {url}")
    90.                 return
    91.  
    92.             zhang = re.findall(r'第\s*(\d+)\s*页\(末页\)', option_texts[-1])
    93.             if not zhang:
    94.                 print(f"未找到页码匹配，URL: {url}")
    95.                 return
    96.             zhang = int(zhang[0])
    97.             print('开始爬取:', title)
    98.  
    99.             s = 0  # 设置爬取多少章
    100.             for i in range(1, zhang + 1):
    101.                 if s >= 100:
    102.                     break  # 已经爬取100章，跳出循环
    103.  
    104.                 zhangjie_url = f'{url}/index_{i}.html'
    105.                 zhangjie_reps = requests.get(zhangjie_url, headers=headers, timeout=10)
    106.                 zhangjie_reps.raise_for_status()
    107.                 zhangjie_html = etree.HTML(zhangjie_reps.text)
    108.                 if zhangjie_html is None:
    109.                     print(f"解析 HTML 内容错误，URL: {zhangjie_url}")
    110.                     break
    111.  
    112.                 zhangjieLis = zhangjie_html.xpath('/html/body/div[4]/div/ul/li/a/@href')
    113.                 if not zhangjieLis:
    114.                     print(f"未找到章节列表，URL: {zhangjie_url}")
    115.                     break
    116.  
    117.                 threads = []
    118.  
    119.                 for j in zhangjieLis:
    120.                     if s >= 100:
    121.                         break  # 已经爬取100章，跳出循环
    122.                     thread = threading.Thread(target=crawl_with_semaphore, args=(neirong, url, j, s))
    123.                     threads.append(thread)
    124.                     thread.start()
    125.                     time.sleep(0.1)
    126.                     s += 1  # 统计章节数目
    127.  
    128.                 for thread in threads:
    129.                     thread.join()
    130.  
    131.             # 插入所有爬取的章节内容到MongoDB
    132.             insert_to_mongodb(title, novel_type, author, update_time, list(zhangjie_content))
    133.             zhangjie_content.clear()  # 清空章节内容列表
    134.  
    135.             print(f"已成功记录数据：{title}")
    136.  
    137.             return
    138.         except requests.RequestException as e:
    139.             print(f"请求错误，URL: {url}, 错误: {e}")
    140.             retries -= 1
    141.             time.sleep(1)  # 等待一段时间后重试
    142.     print(f"重试次数过多，放弃 URL: {url}")
    143.  
    144. def crawl_with_semaphore(target, *args):
    145.     with semaphore:  # 使用信号量来控制并发数量
    146.         target(*args)
    147.  
    148.  
    149.  
    150. # 主函数
    151. def main(i):
    152.     main_url = rf'http://www.biqule.net/top/monthvisit/{i}.html'  # 月访问榜主页链接
    153.     try:
    154.         reps = requests.get(main_url, headers=headers, timeout=10)
    155.         reps.raise_for_status()
    156.         html = etree.HTML(reps.text)
    157.         if html is None:
    158.             print("解析 HTML 内容错误，主页 URL")
    159.             return
    160.  
    161.         novels = html.xpath('//div/ul/li/span[@class="sp_2"]/a')
    162.         novel_urls = [{"title": novel.text.strip(), "url": novel.attrib['href']} for novel in novels if
    163.                       novel.text and 'href' in novel.attrib]
    164.         for i in novel_urls:
    165.             global lis
    166.             lis = []
    167.             zhangjie(i['url'])
    168.  
    169.     except requests.RequestException as e:
    170.         print(f"请求错误，URL: {main_url}, 错误: {e}")
    171.  
    172. if __name__ == "__main__":
    173.     for i in range(1, 51):
    174.         main(i)

    5.2.升级版

    1. 使用异步 I/O：
      使用异步库如 aiohttp 和 asyncio 可以显著提高网络请求的效率。

    2. 减少等待时间：
      减少每次请求之间的等待时间。

    3. 使用连接池：
      使用连接池可以重用连接，减少建立新连接的开销。

    说明：

    1. 改用了 aiohttp 和 asyncio，以实现异步 I/O 操作。
    2. 使用信号量（semaphore）仍然控制并发数量，以避免过多请求导致的封禁。
    3. 将 fetch 函数封装请求逻辑，并且所有的网络请求都使用这个函数。
    4. 采用 asyncio.gather 同时处理多个任务。

    这样，可以显著提升爬取速度，并且享受到异步 I/O 带来的性能提升。

    1. import aiohttp
    2. import asyncio
    3. from lxml import etree
    4. import re
    5. from pymongo import MongoClient
    6.  
    7. # 全局变量和信号量用于控制并发和存储章节内容
    8. zhangjie_content = []  # 存储章节内容
    9. semaphore = asyncio.Semaphore(50)  # 限制并发数量为50
    10.  
    11. # MongoDB连接设置
    12. client = MongoClient('mongodb://localhost:27017/')
    13. db = client.novel_database
    14. collection = db.novels
    15.  
    16.  
    17. def insert_to_mongodb(title, novel_type, author, update_time, chapters, img_url, jianjie):
    18.     """
    19.     辅助函数，用于将数据插入到MongoDB中
    20.     """
    21.     data = {
    22.         "title": title,
    23.         "novel_type": novel_type,
    24.         "author": author,
    25.         "update_time": update_time,
    26.         "zhangjie": chapters,
    27.         'img_url': img_url,
    28.         'jianjie': jianjie
    29.     }
    30.     collection.insert_one(data)
    31.     print(f"插入 {len(chapters)} 章成功：{title}")
    32.  
    33.  
    34. async def fetch(session, url):
    35.     async with semaphore:  # 使用信号量来控制并发数量
    36.         try:
    37.             async with session.get(url) as response:
    38.                 return await response.text()
    39.         except Exception as e:
    40.             print(f"请求错误，URL: {url}, 错误: {e}")
    41.  
    42.  
    43. async def neirong(session, base_url, url):
    44.     try:
    45.         html_str = await fetch(session, f'{base_url}{url}')
    46.         html = etree.HTML(html_str)
    47.         if html is None:
    48.             print(f"解析 HTML 内容错误，URL: {base_url}{url}")
    49.             return
    50.  
    51.         chapter = html.xpath('//*[@id="content"]/h1/text()')
    52.         if not chapter:
    53.             print(f"未找到章节标题，URL: {base_url}{url}")
    54.             return
    55.  
    56.         chapter = chapter[0].strip()
    57.         text = html.xpath('//*[@id="htmlContent"]/text()')
    58.         if not text:
    59.             print(f"未找到章节内容，URL: {base_url}{url}")
    60.             return
    61.  
    62.         text = ''.join(text[1:])  # 连接文本内容
    63.         zhangjie_content.append({"chapter": chapter, "text": text})
    64.  
    65.     except Exception as e:
    66.         print(f"处理章节内容错误，URL: {base_url}{url}, 错误: {e}")
    67.  
    68.  
    69. async def zhangjie(session, url):
    70.     try:
    71.         html_str = await fetch(session, url)
    72.         html = etree.HTML(html_str)
    73.         if html is None:
    74.             print(f"解析 HTML 内容错误，URL: {url}")
    75.             return
    76.  
    77.         title = html.xpath('//*[@id="info"]/h1/text()')
    78.         title = title[0].strip() if title else "未知书名"
    79.         novel_type = html.xpath('//*[@id="maininfo"]/div[1]/a[2]/text()')
    80.         novel_type = novel_type[0].strip() if novel_type else "未知类型"
    81.         author = html.xpath('//*[@id="info"]/p[1]/a/text()')
    82.         author = author[0].strip() if author else "未知作者"
    83.         update_time = html.xpath('//*[@id="info"]/p[3]/text()')
    84.         update_time = update_time[0].strip() if update_time else "未知时间"
    85.         img_url = html.xpath('//*[@id="fmimg"]/img/@src')
    86.         img_url = img_url[0].strip() if img_url else "未知图片"
    87.         jianjie = ''.join(html.xpath('//*[@id="intro"]//text()')).strip() if html.xpath(
    88.             '//*[@id="intro"]//text()') else "未知简介"
    89.  
    90.         option_texts = html.xpath('/html/body/div[4]/div/div/select/option/text()')
    91.         if not option_texts:
    92.             print(f"未找到页码信息，URL: {url}")
    93.             return
    94.  
    95.         zhang = re.findall(r'第\s*(\d+)\s*页\(末页\)', option_texts[-1])
    96.         if not zhang:
    97.             print(f"未找到页码匹配，URL: {url}")
    98.             return
    99.         zhang = int(zhang[0])
    100.         print('开始爬取:', title)
    101.  
    102.         for i in range(1, zhang + 1):
    103.             if len(zhangjie_content) >= 100:
    104.                 break  # 已经爬取100章，跳出循环
    105.  
    106.             zhangjie_url = f'{url}/index_{i}.html'
    107.             zhangjie_html_str = await fetch(session, zhangjie_url)
    108.             zhangjie_html = etree.HTML(zhangjie_html_str)
    109.             if zhangjie_html is None:
    110.                 print(f"解析 HTML 内容错误，URL: {zhangjie_url}")
    111.                 break
    112.  
    113.             zhangjieLis = zhangjie_html.xpath('/html/body/div[4]/div/ul/li/a/@href')
    114.             if not zhangjieLis:
    115.                 print(f"未找到章节列表，URL: {zhangjie_url}")
    116.                 break
    117.  
    118.             tasks = []
    119.             for j in zhangjieLis:
    120.                 if len(zhangjie_content) >= 100:
    121.                     break  # 已经爬取100章，跳出循环
    122.                 task = asyncio.create_task(neirong(session, url, j))
    123.                 tasks.append(task)
    124.  
    125.             await asyncio.gather(*tasks)
    126.  
    127.         # 插入所有爬取的章节内容到MongoDB
    128.         insert_to_mongodb(title, novel_type, author, update_time, list(zhangjie_content), img_url, jianjie)
    129.         zhangjie_content.clear()  # 清空章节内容列表
    130.  
    131.         print(f"已成功记录数据：{title}")
    132.  
    133.     except Exception as e:
    134.         print(f"处理章节信息错误，URL: {url}, 错误: {e}")
    135.  
    136.  
    137. async def main():
    138.     async with aiohttp.ClientSession() as session:
    139.         tasks = []
    140.         for i in range(1, 51):
    141.             main_url = f'http://www.biqule.net/top/monthvisit/{i}.html'  # 月访问榜主页链接
    142.             task = asyncio.create_task(process_main_page(session, main_url))
    143.             tasks.append(task)
    144.         await asyncio.gather(*tasks)
    145.  
    146.  
    147. async def process_main_page(session, main_url):
    148.     try:
    149.         html_str = await fetch(session, main_url)
    150.         html = etree.HTML(html_str)
    151.         if html is None:
    152.             print("解析 HTML 内容错误，主页 URL")
    153.             return
    154.  
    155.         novels = html.xpath('//div/ul/li/span[@class="sp_2"]/a')
    156.         novel_urls = [{"title": novel.text.strip(), "url": novel.attrib['href']} for novel in novels if
    157.                       novel.text and 'href' in novel.attrib]
    158.         tasks = []
    159.         for novel in novel_urls:
    160.             task = asyncio.create_task(zhangjie(session, novel['url']))
    161.             tasks.append(task)
    162.         await asyncio.gather(*tasks)
    163.  
    164.     except Exception as e:
    165.         print(f"处理主页面错误，URL: {main_url}, 错误: {e}")
    166.  
    167.  
    168. if __name__ == "__main__":
    169.     asyncio.run(main())

  • 相关阅读:
    Docker 常用命令
    Mysql加锁流程详解
    技术分享 | MySQL Shell 运行 SQL 的两种内置方法概述
    【JavaWeb笔记】Servlet入门—获取参数
    Cadence OrCAD Capture 设置线相对延迟约束的方法
    kubernetespod控制器详解2与service详解1
    LrC 13 & ACR 16：镜头模糊
    JUnit进行单元测试
    [ICCV-23] DeformToon3D: Deformable Neural Radiance Fields for 3D Toonification
    六.schema设计
  • 原文地址：https://blog.csdn.net/iku_n/article/details/139509931