python爬虫之feapder.AirSpider轻量爬虫案例：豆瓣

# # MYSQL
MYSQL_IP = "localhost"
MYSQL_PORT = 3306
MYSQL_DB = "video"
MYSQL_USER_NAME = "root"
MYSQL_USER_PASS = "root"

# # REDIS
# # ip:port 多个可写为列表或者逗号隔开 如 ip1:port1,ip2:port2 或 ["ip1:port1", "ip2:port2"]
REDISDB_IP_PORTS = "localhost:6379"
REDISDB_USER_PASS = ""
REDISDB_DB = 0
# 连接redis时携带的其他参数，如ssl=True
REDISDB_KWARGS = dict()
# 适用于redis哨兵模式
REDISDB_SERVICE_NAME = ""

# # 去重
ITEM_FILTER_ENABLE = True # item 去重
ITEM_FILTER_SETTING = dict(
    filter_type=1,  # 永久去重（BloomFilter） = 1 、内存去重（MemoryFilter） = 2、 临时去重（ExpireFilter）= 3、轻量去重（LiteFilter）= 4
    name="douban"
)

# # 邮件报警
EMAIL_SENDER = "123123123@163.com"  # 发件人
EMAIL_PASSWORD = "EYNXMBWJKMLZFTKQ"  # 授权码
EMAIL_RECEIVER = ["123123123@163.com"]  # 收件人 支持列表，可指定多个
EMAIL_SMTPSERVER = "smtp.163.com"  # 邮件服务器 默认为163邮箱
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

CREATE TABLE IF NOT EXISTS douban(
        id INT AUTO_INCREMENT,
        title VARCHAR(255),
        rating FLOAT,
        quote VARCHAR(255),
        intro TEXT,
        PRIMARY KEY(id)
        )
1
2
3
4
5
6
7
8

# -*- coding: utf-8 -*-
"""
Created on 2023-10-08 16:17:51
---------
@summary:
---------
@author: Administrator
"""

from feapder import Item


class DoubanItem(Item):
    """
    This class was generated by feapder
    command: feapder create -i douban
    """

    __table_name__ = "douban"
    __unique_key__ = ["title","quote","rating","title"] # 指定去重的key为 title、quote，最后的指纹为title与quote值联合计算的md5

    def __init__(self, *args, **kwargs):
        super().__init__(**kwargs)
        # self.id = None
        self.intro = None
        self.quote = None
        self.rating = None
        self.title = None

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29

# -*- coding: utf-8 -*-
"""
Created on 2023-10-06 15:36:09
---------
@summary:
---------
@author: Administrator
"""

import feapder
from items.douban_item import DoubanItem
from feapder.network.user_agent import get as get_ua
from requests.exceptions import ConnectTimeout,ProxyError
from feapder.utils.email_sender import EmailSender
import feapder.setting as setting

class AirSpiderDouban(feapder.AirSpider):
    def __init__(self, thread_count=None):
        super().__init__(thread_count)
        self.request_url = 'https://movie.douban.com/top250'

    def start_requests(self):
        yield feapder.Request(self.request_url)

    def download_midware(self, request):
        request.headers = {
            'User-Agent': get_ua()
        }
        return request

    def parse(self, request, response):
        video_list = response.xpath('//ol[@class="grid_view"]/li')
        for li in video_list:
            item = DoubanItem()
            item['title'] = li.xpath('.//div[@class="hd"]/a/span[1]/text()').extract_first()
            item['rating'] = li.xpath('.//div[@class="bd"]//span[@class="rating_num"]/text()').extract_first()
            item['quote'] = li.xpath('.//div[@class="bd"]//p[@class="quote"]/span/text()').extract_first()
            detail_url = li.xpath('.//div[@class="hd"]/a/@href').extract_first()
            if detail_url:
                yield feapder.Request(detail_url, callback=self.get_detail_info, item=item)
        # 获取下一页数据
        next_page_url = response.xpath('//div[@class="paginator"]//link[@rel="next"]/@href').extract_first()
        if next_page_url:
            yield feapder.Request(next_page_url,callback=self.parse)

    def get_detail_info(self, request, response):
        item = request.item
        detail = response.xpath('//span[@class="all hidden"]/text()').extract_first() or ''
        if not detail:
            detail = response.xpath('//div[@id="link-report-intra"]/span[1]/text()').extract_first() or ''
        item['intro'] = detail.strip()
        yield item

    def exception_request(self, request, response, e):
        prox_err = [ConnectTimeout,ProxyError]
        if type(e) in prox_err:
            request.del_proxy()

    def end_callback(self):
        with EmailSender(setting.EMAIL_SENDER,setting.EMAIL_PASSWORD) as email_sender:
            email_sender.send(setting.EMAIL_RECEIVER, title='python',content="爬虫结束")


if __name__ == "__main__":
    AirSpiderDouban(thread_count=5).start()

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66

from feapder import ArgumentParser
from spiders import *

def crawl_air_spider_douban():
    """
    AirSpider爬虫
    """
    spider = air_spider_douban.AirSpiderDouban()
    spider.start()

if __name__ == "__main__":
    parser = ArgumentParser(description="爬虫练习")


    parser.add_argument(
        "--crawl_air_spider_douban", action="store_true", help="豆瓣AirSpide", function=crawl_air_spider_douban
    )


    parser.run("crawl_air_spider_douban")
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20