Python 网络请求工具库

retry 见链接 ：https://blog.csdn.net/CXY00000/article/details/132482838

# !/usr/bin/python3
# -*- coding:utf-8 -*-
"""
@author: JHC000abc@gmail.com
@file: util_network.py
@time: 2023/7/30 22:37
@desc:

"""
import requests
from .util_decorate import retry


class NetWorkRequests():

    @retry()
    def _requests(self, url, headers=None, params=None, cookies=None, timeout=5,
                  data=None, json=None, method="POST", stream=False, proxies=None):
        """

        :param url:
        :param headers:
        :param params:
        :param cookies:
        :param timeout:
        :param data:
        :param json:
        :param method:
        :param stream:
        :param proxies:
        :return:
        """
        if method == "POST":
            response = requests.post(
                url=url,
                data=data,
                json=json,
                stream=stream,
                cookies=cookies,
                headers=headers,
                proxies=proxies,
                timeout=timeout)
        elif method == "GET":
            response = requests.get(
                url=url,
                params=params,
                stream=stream,
                cookies=cookies,
                headers=headers,
                proxies=proxies,
                timeout=timeout)
        else:
            raise ValueError("ERROR Methods {}".format(method))
        return response

    def download_videos(self, url, file, max_size=102400, method="GET"):
        """

        :param url:
        :param file:
        :param max_size:
        :param method:
        :return:
        """
        res = {
            "status":-1,
            "url":url,
            "msg":"Failed",
            "result":[]
        }
        response = self._requests(url, method=method)
        if response["status"] == -1:
            msg = "Failed"
        else:
            download_status_all = True
            content_len = int(response["msg"].headers["Content-Length"])
            nums = content_len // max_size if content_len % max_size == 0 else content_len // max_size + 1
            res["content_len"] = content_len
            res["split_nums"] = nums
            with open(file, "ab+")as fp:
                for i in range(nums):
                    start, end = i * max_size, (i + 1) * max_size
                    headers = {
                        "Range": "bytes={}-{}".format(start, end)
                    }
                    response = self._requests(
                        url, headers=headers, stream=True, method=method)
                    chunk_status = response["status"]
                    if chunk_status == -1:
                        download_status_all = False
                    else:
                        for chunk in response["msg"].iter_content(chunk_size=max_size):
                            fp.seek(start)
                            fp.write(chunk)
                    res["result"].append({
                        "status":chunk_status,
                        "during":[start,end],
                    })
            if download_status_all:
                msg = "Success"
                res["status"] = 0
            else:
                msg = "Failed"

        res["msg"] = msg

        return res

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108