• 小黑准备收拾东西领物资的日常积累：线程与进程

    课程来源:https://www.bilibili.com/video/BV1ma411q7me/?spm_id_from=333.337.top_right_bar_window_custom_collection.content.click&vd_source=0b1383084f626b5cf37be3e82e883106

    多线程实例

    '''
基于多线程对上述串行示例进行优化：

一个工厂，创建一个车间，这个车间中创建 3个工人，并行处理任务。
一个程序，创建一个进程，这个进程中创建 3个线程，并行处理任务。
'''
import time
import requests
import threading

url_list = [
    ("东北F4模仿秀.mp4", "https://aweme.snssdk.com/aweme/v1/playwm/?video_id=v0300f570000bvbmace0gvch7lo53oog"),
    ("卡特扣篮.mp4", "https://aweme.snssdk.com/aweme/v1/playwm/?video_id=v0200f3e0000bv52fpn5t6p007e34q1g"),
    ("罗斯mvp.mp4", "https://aweme.snssdk.com/aweme/v1/playwm/?video_id=v0200f240000buuer5aa4tij4gv6ajqg")
]

def task(file_name,video_url):
    res = requests.get(video_url)
    with open(file_name,mode='wb') as f:
        f.write(res.content)
    print(time.time())

for name,url in url_list:
    # 创建线程,让每个线程都执行task函数
    t = threading.Thread(target=task,args=(name,url))
    t.start()
    

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21
    • 22
    • 23
    • 24
    • 25
    • 26
    • 27

    1667533467.525886
    1667533467.565978
    1667533467.565978

    多进程实例

    import time
import requests
import threading
import multiprocessing

url_list = [
    ("东北F4模仿秀.mp4", "https://aweme.snssdk.com/aweme/v1/playwm/?video_id=v0300f570000bvbmace0gvch7lo53oog"),
    ("卡特扣篮.mp4", "https://aweme.snssdk.com/aweme/v1/playwm/?video_id=v0200f3e0000bv52fpn5t6p007e34q1g"),
    ("罗斯mvp.mp4", "https://aweme.snssdk.com/aweme/v1/playwm/?video_id=v0200f240000buuer5aa4tij4gv6ajqg")
]

def task(file_name,video_url):
    res = requests.get(video_url)
    with open(file_name,mode='wb') as f:
        f.write(res.content)
    print(time.time())
if __name__ == '__main__':
    for name,url in url_list:
        # 创建进程,让每个进程都执行task函数
        t = multiprocessing.Process(target=task,args=(name,url))
        t.start()

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21

    1667533771.786621
    1667533771.786621
    1667533771.786621

    得出结论:在这个任务中多进程比多线程开销大

    串行

    import time
if __name__ == '__main__':
    start = time.time()
    result = 0
    for i in range(100000000):
        result += i
    print(result)
    end = time.time()
    print('耗时:',end-start)

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9

    4999999950000000
    耗时: 6.52695894241333

    多进程处理

    import time
import multiprocessing

def task(start,end,queue):
    result = 0
    for i in range(start,end):
        result += i
    queue.put(result)

if __name__ == '__main__':
    queue = multiprocessing.Queue()

    start_time = time.time()

    p1 = multiprocessing.Process(target = task,args = (0,50000000,queue))
    p1.start()

    p2 = multiprocessing.Process(target=task,args=(50000000,100000000,queue))
    p2.start()

    v1 = queue.get(block = True)
    v2 = queue.get(block = True)
    print(v1 + v2)

    end_time = time.time()
    print('耗时:',end_time - start_time)

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21
    • 22
    • 23
    • 24
    • 25
    • 26

    4999999950000000
    耗时: 2.2204787731170654

    Process finished with exit code 0

    多进程与多线程同时使用

    import multiprocessing
import threading

def thread_task():
    pass

def task(start,end):
    t1 = threading.Thread(target = thread_task)
    t1.start()

    t2 = threading.Thread(target = thread_task)
    t2.start()

    t3 = threading.Thread(target = thread_task)
    t3.start()

if __name__ == '__main__':
    p1 = multiprocessing.Process(target = task,args=(0,50000000))
    p1.start()
    p2 = multiprocessing.Process(target = task,args = (50000000,100000000))
    p2.start()

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21

    多线程开发

    import threading

number = 0

def _add():
    global number
    for i in range(10000000):
        number += 1

t = threading.Thread(target = _add)
t.start()
t.join()    # 主进程等待中...
print(number)

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13

    10000000

    import threading

number = 0

def _add():
    global number
    for i in range(10000000):
        number += 1

def _sub():
    global number
    for i in range(10000000):
        number -= 1

t1 = threading.Thread(target = _add)
t2 = threading.Thread(target = _sub)
t1.start()
t1.join()  # t1线程执行完毕,才能继续往后走
t2.start()
t2.join()    # t2线程执行完毕,才能继续往后走
print(number)

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21

    0

    import threading

number = 0

def _add():
    global number
    for i in range(10000000):
        number += 1

def _sub():
    global number
    for i in range(10000000):
        number -= 1

t1 = threading.Thread(target = _add)
t2 = threading.Thread(target = _sub)
t1.start()
t2.start()
t1.join()  # t1线程执行完毕,才能继续往后走
t2.join()    # t2线程执行完毕,才能继续往后走
print(number)

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21

    -756842

    守护进程

    import threading
import time

def task(arg):
    time.sleep(5)
    print('任务')

if __name__ == '__main__':
    t = threading.Thread(target=task,args=(11,))
    t.setDaemon(True)    # 设置为守护2进程
    t.start()

    print('END')

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13

    END

    线程名称设置

    import threading

def task(arg):
    # 获取当前执行此代码的线程
    name = threading.current_thread().getName()
    print(name)

for i in range(10):
    t = threading.Thread(target = task,args=(11,))
    t.setName('小黑无敌酒量')
    t.start()

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11

    小黑无敌酒量
    小黑无敌酒量
    小黑无敌酒量
    小黑无敌酒量
    小黑无敌酒量
    小黑无敌酒量
    小黑无敌酒量
    小黑无敌酒量
    小黑无敌酒量
    小黑无敌酒量

    自定义线程类，写在run里

    import threading
class MyThread(threading.Thread):
    def run(self):
        print('执行此线程',self._args)
if __name__ == '__main__':
    t = MyThread(args=(100,))
    t.start()

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7

    执行此线程 (100,)

    import requests
import threading

class DouYinThread(threading.Thread):
    def run(self):
        file_name,video_url = self._args
        res = requests.get(video_url)
        with open(file_name,mode='wb') as f:
            f.write(res.content)

url_list = [
    ("东北F4模仿秀.mp4", "https://aweme.snssdk.com/aweme/v1/playwm/?video_id=v0300f570000bvbmace0gvch7lo53oog"),
    ("卡特扣篮.mp4", "https://aweme.snssdk.com/aweme/v1/playwm/?video_id=v0200f3e0000bv52fpn5t6p007e34q1g"),
    ("罗斯mvp.mp4", "https://aweme.snssdk.com/aweme/v1/playwm/?video_id=v0200f240000buuer5aa4tij4gv6ajqg")
]

for item in url_list:
    t = DouYinThread(args=(item[0],item[1]))
    t.start()

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19

    线程安全

    import threading

lock_object = threading.RLock()

loop = 10000000
number = 0

def _add(count):
    lock_object.acquire()    # 申请锁
    global number
    for i in range(count):
        number += 1
    lock_object.release()    # 释放锁

def _sub(count):
    lock_object.acquire()
    global number
    for i in range(count):
        number -= 1
    lock_object.release()

t1 = threading.Thread(target=_add,args=(loop,))
t2 = threading.Thread(target=_sub,args=(loop,))
t1.start()
t2.start()

t1.join()
t2.join()
print(number)

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21
    • 22
    • 23
    • 24
    • 25
    • 26
    • 27
    • 28
    • 29

    0

    import threading

num = 0

def task():
    global num
    for i in range(1000000):
        num += 1
    print(num)

if __name__ == '__main__':
    for i in range(2):
        t = threading.Thread(target=task)
        t.start()

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14

    1552302
    1971094

    import threading

num = 0
lock_object = threading.RLock()

def task():
    print('开始')
    lock_object.acquire()    # 第一个抵达的线程进入并上锁，其他线程就需要在此等待。
    global num
    for i in range(1000000):
        num += 1
    lock_object.release()    # 线程出去,并解开锁
    print(num)

for i in range(2):
    t = threading.Thread(target=task)
    t.start()

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17

    开始
    开始
    1000000
    2000000

    Lock同步锁

    import threading

num = 0
lock_object = threading.Lock()

def task():
    print('开始')
    global num
    lock_object.acquire()
    for i in range(1000000):
        num += 1
    lock_object.release()
    print(num)

for i in range(2):
    t = threading.Thread(target = task)
    t.start()

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17

    开始
    开始
    1000000
    2000000

    RLock递归锁

    import threading

num = 0
lock_object = threading.RLock()

def task():
    print('开始')
    global num
    lock_object.acquire()
    for i in range(1000000):
        num += 1
    lock_object.release()
    print(num)

for i in range(2):
    t = threading.Thread(target = task)
    t.start()

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17

    开始
    开始
    1000000
    2000000

    递归锁多次申请锁和多次释放

    import threading
import time

lock_object = threading.RLock()

def task():
    print('开始')
    lock_object.acquire()
    lock_object.acquire()
    print(123)
    lock_object.release()
    lock_object.release()

for i in range(3):
    t = threading.Thread(target=task)
    t.start()

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16

    开始
    123
    开始开始
    123

    123

    递归锁实例

    import threading
lock = threading.RLock()

# 程序员A开发了一个函数，函数可以被其他开发者调用，内部需要基于锁保证数据安全
def func():
    with lock:
        pass

# 程序员B想用那个程序
def run():
    print('其他功能')
    func()
    print('其他功能')

# 程序员C也想用，但是得加锁
def process():
    with lock:
        print('其他功能')
        func()    # 此时会出现多次锁的情况,只有Rlock支持
        print('其他功能')

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20

    死锁案例

    import threading

num = 0
lock_object = threading.Lock()

def task():
    print('开始')
    lock_object.acquire()
    lock_object.acquire()
    global num
    for i in range(100000):
        num += 1
    lock_object.release()
    lock_object.release()
    print(2)

for i in range(2):
    t = threading.Thread(target = task)
    t.start()

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    import threading
import time

lock_1 = threading.Lock()
lock_2 = threading.Lock()

def task1():
    lock_1.acquire()
    time.sleep(1)
    lock_2.acquire()
    print(11)
    lock_2.release()
    print(111)
    lock_1.release()
    print(11111)

def task2():
    lock_2.acquire()
    time.sleep(1)
    lock_1.acquire()
    print(22)
    lock_1.release()
    print(3333)
    lock_2.releasr()
    print(2222222)

if __name__ == '__main__':
    t1 = threading.Thread(target=task1)
    t1.start()

    t2 = threading.Thread(target=task2)
    t2.start()

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21
    • 22
    • 23
    • 24
    • 25
    • 26
    • 27
    • 28
    • 29
    • 30
    • 31
    • 32

    线程池

    import time
from concurrent.futures import ThreadPoolExecutor

def task(video_url):
    print('开始执行任务',video_url)
    time.sleep(5)

# 创建线程池，最多维护10个进程
pool = ThreadPoolExecutor(10)

url_list = ['www.{}'.format(i) for i in range(300)]

for url in url_list:
    pool.submit(task,url)
print('执行中...')
pool.shutdown(True)
print('继续走下去')

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    import time
import random
from concurrent.futures import ThreadPoolExecutor,Future

def task(video_url):
    print('开始执行任务',video_url)
    time.sleep(2)
    return random.randint(0,10)

def done(response):
    print('任务执行后的返回值',response.result())

if __name__ == '__main__':
    # 创建线程池,最多维护10个线程
    pool = ThreadPoolExecutor(10)
    url_list = ['www.xxxx-{}.com'.format(i) for i in range(15)]
    for url in url_list:
        # 在线程池里提交一个任务，线程池中如果有空闲线程，则分配一个线程去执行，执行完毕再将线程还给线程池；如果没有空闲线程则等待。
        future = pool.submit(task,url)
        future.add_done_callback(done)

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20

    最终统一的效果

    import time
import random
from concurrent.futures import ThreadPoolExecutor,Future

def task(video_url):
    print('开始执行任务',video_url)
    time.sleep(2)
    return random.randint(0,10)

if __name__ == '__main__':
    pool = ThreadPoolExecutor(10)
    future_list = []
    url_list = ['www.xxxx-{}.com'.format(i) for i in range(15)]
    for url in url_list:
        future = pool.submit(task,url)
        future_list.append(future)
    pool.shutdown(True)
    for fu in future_list:
        print(fu.result())

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19

    线程池下载图片案例（需要自己写一遍）

    import os
import requests
from concurrent.futures import ThreadPoolExecutor

def download(image_url):
    res = requests.get(
        url = image_url,
        headers = {
            'User-Agent':"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36"
        }
    )
    return res

def outer(file_name):
    def save(response):
        res = response.result()
        # 写入本地
        # 检查images目录是否存在？不存在，则创建一个images目录
        if not os.path.exists('images'):
            # 创建images目录
            os.makedirs('images')
        file_path = os.path.join('images',file_name)
        # 将图片写入文件
        with open(file_path,mode='wb') as img_object:
            img_object.write(res.content)
    return save

if __name__ == '__main__':
    pool = ThreadPoolExecutor(10)
    with open('mv.csv',mode='r',encoding='utf-8') as file_object:
        for line in file_object:
            nid,name,url = line.strip().split(',')
            file_name = '{}.png'.format(name)
            fur = pool.submit(download,url)
            fur.add_done_callback(outer(file_name))

    • 1
    • 2
    • 3
    • 4
    • 5
    • 6
    • 7
    • 8
    • 9
    • 10
    • 11
    • 12
    • 13
    • 14
    • 15
    • 16
    • 17
    • 18
    • 19
    • 20
    • 21
    • 22
    • 23
    • 24
    • 25
    • 26
    • 27
    • 28
    • 29
    • 30
    • 31
    • 32
    • 33
    • 34
    • 35

    在这里插入图片描述

    小黑生活

    请添加图片描述

    请添加图片描述
    请添加图片描述
    请添加图片描述
    请添加图片描述
    请添加图片描述
    请添加图片描述
    请添加图片描述

    明天北马目标：安全回家

  • 相关阅读:
    微服务项目：尚融宝（37）（核心业务流程：用户绑定（2））
    软考高级-系统架构师-软件架构设计练习题
    Mock.js的使用api讲解
    Win11磁盘分区后在恢复之前分区的方法介绍
    《大数据项目实战之搜索引擎用户行为分析》
    [学习笔记]TypeScript查缺补漏（一）：类
    含文档+PPT+源码等]精品微信小程序旅游服务平台+后台管理系统|前后分离VUE[包运行成功]微信小程序项目源码Java毕业设计
    2022最新总结一线大厂Java八股文合集，堪称历史最强
    Spring之依赖注入源码解析
    utc时间怎么转换北京时间?
  • 原文地址：https://blog.csdn.net/qq_37418807/article/details/127686023