Python concurrent 包
1、concurrent.futures
异步并行任务编程模块，提供一个高级的异步可执行的便利接口。
提供了两个池执行器：
ThreadPoolExecutor异步调用的线程池的ExecutorProcessPoolExecutor异步调用的进程池的Executor
2、ThreadPoolExecutor
ThreadPoolExecutor(max_workers=None)
# 池中至多创建max-workers个线程的池来同时异步执行，返回Executor实例
submit(*args, **kwargs)
# 提交执行的函数及其参数，返回Future类的实例
shutdown(wait=True)
# 清理池
# concurrent.futures
# ThreadPoolExecutor
from concurrent.futures import ThreadPoolExecutor
import time
import logging
import threading
FORMAT = "%(asctime)s %(threadName)22s %(thread)8d %(message)s"
logging.basicConfig(level=logging.INFO, format=FORMAT)
def worker(n):
logging.info('enter thread ~~~~~~~~~~~~~~')
time.sleep(n)
logging.info('finished ~~~~~~~~~~~~')
return n
executor = ThreadPoolExecutor(max_workers=3)
fs = []
for i in range(6):
future = executor.submit(worker, i) # 异步，非阻塞，执行完毕后立马执行下一条命令
logging.info(future)
fs.append(future)
# logging.info('-' * 55)
# for i in range(3):
# future = executor.submit(worker, 5) # 异步，非阻塞，执行完毕后立马执行下一条命令
# logging.info(future)
# logging.info('=' * 55)
# executor.shutdown()
# logging.info('*' * 55)
while True:
flag = True
for f in fs:
logging.info(f.done())
flag = flag and f.done()
if flag:
executor.shutdown()
logging.info(threading.enumerate())
break
# if threading.active_count() == 1:
# logging.info(threading.enumerate())
# logging.info(future)
# break
time.sleep(1)
logging.info(threading.enumerate())
for f in fs:
logging.info(f.result())
2022-04-21 16:01:17,590 ThreadPoolExecutor-0_0 5212 enter thread ~~~~~~~~~~~~~~
2022-04-21 16:01:17,590 MainThread 7416 <Future at 0x1fb22ca8730 state=running>
2022-04-21 16:01:17,590 ThreadPoolExecutor-0_0 5212 finished ~~~~~~~~~~~~
2022-04-21 16:01:17,590 ThreadPoolExecutor-0_0 5212 enter thread ~~~~~~~~~~~~~~
2022-04-21 16:01:17,590 MainThread 7416 <Future at 0x1fb22ca8c70 state=running>
2022-04-21 16:01:17,590 MainThread 7416 <Future at 0x1fb22ca8fa0 state=pending>
2022-04-21 16:01:17,591 ThreadPoolExecutor-0_1 31892 enter thread ~~~~~~~~~~~~~~
2022-04-21 16:01:17,591 ThreadPoolExecutor-0_2 49012 enter thread ~~~~~~~~~~~~~~
2022-04-21 16:01:17,591 MainThread 7416 <Future at 0x1fb22cb6040 state=running>
2022-04-21 16:01:17,591 MainThread 7416 <Future at 0x1fb22cb6400 state=pending>
2022-04-21 16:01:17,591 MainThread 7416 <Future at 0x1fb22cb64f0 state=pending>
2022-04-21 16:01:17,591 MainThread 7416 True
2022-04-21 16:01:17,591 MainThread 7416 False
2022-04-21 16:01:17,591 MainThread 7416 False
2022-04-21 16:01:17,591 MainThread 7416 False
2022-04-21 16:01:17,591 MainThread 7416 False
2022-04-21 16:01:17,591 MainThread 7416 False
2022-04-21 16:01:18,593 ThreadPoolExecutor-0_0 5212 finished ~~~~~~~~~~~~
2022-04-21 16:01:18,593 MainThread 7416 [<_MainThread(MainThread, started 7416)>, <Thread(ThreadPoolExecutor-0_0, started daemon 5212)>, <Thread(ThreadPoolExecutor-0_1, started daemon 31892)>, <Thread(ThreadPoolExecutor-0_2, started daemon 49012)>]
2022-04-21 16:01:18,593 ThreadPoolExecutor-0_0 5212 enter thread ~~~~~~~~~~~~~~
2022-04-21 16:01:18,593 MainThread 7416 True
2022-04-21 16:01:18,594 MainThread 7416 True
2022-04-21 16:01:18,594 MainThread 7416 False
2022-04-21 16:01:18,594 MainThread 7416 False
2022-04-21 16:01:18,594 MainThread 7416 False
2022-04-21 16:01:18,594 MainThread 7416 False
2022-04-21 16:01:19,599 ThreadPoolExecutor-0_1 31892 finished ~~~~~~~~~~~~
2022-04-21 16:01:19,599 MainThread 7416 [<_MainThread(MainThread, started 7416)>, <Thread(ThreadPoolExecutor-0_0, started daemon 5212)>, <Thread(ThreadPoolExecutor-0_1, started daemon 31892)>, <Thread(ThreadPoolExecutor-0_2, started daemon 49012)>]
2022-04-21 16:01:19,599 MainThread 7416 True
2022-04-21 16:01:19,599 ThreadPoolExecutor-0_1 31892 enter thread ~~~~~~~~~~~~~~
2022-04-21 16:01:19,599 MainThread 7416 True
2022-04-21 16:01:19,600 MainThread 7416 True
2022-04-21 16:01:19,600 MainThread 7416 False
2022-04-21 16:01:19,600 MainThread 7416 False
2022-04-21 16:01:19,600 MainThread 7416 False
2022-04-21 16:01:20,594 ThreadPoolExecutor-0_2 49012 finished ~~~~~~~~~~~~
2022-04-21 16:01:20,610 MainThread 7416 [<_MainThread(MainThread, started 7416)>, <Thread(ThreadPoolExecutor-0_0, started daemon 5212)>, <Thread(ThreadPoolExecutor-0_1, started daemon 31892)>, <Thread(ThreadPoolExecutor-0_2, started daemon 49012)>]
2022-04-21 16:01:20,610 MainThread 7416 True
2022-04-21 16:01:20,610 MainThread 7416 True
2022-04-21 16:01:20,610 MainThread 7416 True
2022-04-21 16:01:20,610 MainThread 7416 True
2022-04-21 16:01:20,610 MainThread 7416 False
2022-04-21 16:01:20,610 MainThread 7416 False
2022-04-21 16:01:21,611 MainThread 7416 [<_MainThread(MainThread, started 7416)>, <Thread(ThreadPoolExecutor-0_0, started daemon 5212)>, <Thread(ThreadPoolExecutor-0_1, started daemon 31892)>, <Thread(ThreadPoolExecutor-0_2, started daemon 49012)>]
2022-04-21 16:01:21,611 MainThread 7416 True
2022-04-21 16:01:21,611 MainThread 7416 True
2022-04-21 16:01:21,611 MainThread 7416 True
2022-04-21 16:01:21,611 MainThread 7416 True
2022-04-21 16:01:21,611 MainThread 7416 False
2022-04-21 16:01:21,611 MainThread 7416 False
2022-04-21 16:01:22,604 ThreadPoolExecutor-0_0 5212 finished ~~~~~~~~~~~~
2022-04-21 16:01:22,620 MainThread 7416 [<_MainThread(MainThread, started 7416)>, <Thread(ThreadPoolExecutor-0_0, started daemon 5212)>, <Thread(ThreadPoolExecutor-0_1, started daemon 31892)>, <Thread(ThreadPoolExecutor-0_2, started daemon 49012)>]
2022-04-21 16:01:22,620 MainThread 7416 True
2022-04-21 16:01:22,620 MainThread 7416 True
2022-04-21 16:01:22,620 MainThread 7416 True
2022-04-21 16:01:22,620 MainThread 7416 True
2022-04-21 16:01:22,620 MainThread 7416 True
2022-04-21 16:01:22,620 MainThread 7416 False
2022-04-21 16:01:23,626 MainThread 7416 [<_MainThread(MainThread, started 7416)>, <Thread(ThreadPoolExecutor-0_0, started daemon 5212)>, <Thread(ThreadPoolExecutor-0_1, started daemon 31892)>, <Thread(ThreadPoolExecutor-0_2, started daemon 49012)>]
2022-04-21 16:01:23,626 MainThread 7416 True
2022-04-21 16:01:23,626 MainThread 7416 True
2022-04-21 16:01:23,626 MainThread 7416 True
2022-04-21 16:01:23,626 MainThread 7416 True
2022-04-21 16:01:23,626 MainThread 7416 True
2022-04-21 16:01:23,626 MainThread 7416 False
2022-04-21 16:01:24,603 ThreadPoolExecutor-0_1 31892 finished ~~~~~~~~~~~~
2022-04-21 16:01:24,634 MainThread 7416 [<_MainThread(MainThread, started 7416)>, <Thread(ThreadPoolExecutor-0_0, started daemon 5212)>, <Thread(ThreadPoolExecutor-0_1, started daemon 31892)>, <Thread(ThreadPoolExecutor-0_2, started daemon 49012)>]
2022-04-21 16:01:24,634 MainThread 7416 True
2022-04-21 16:01:24,634 MainThread 7416 True
2022-04-21 16:01:24,634 MainThread 7416 True
2022-04-21 16:01:24,634 MainThread 7416 True
2022-04-21 16:01:24,634 MainThread 7416 True
2022-04-21 16:01:24,634 MainThread 7416 True
2022-04-21 16:01:24,635 MainThread 7416 [<_MainThread(MainThread, started 7416)>]
2022-04-21 16:01:24,635 MainThread 7416 0
2022-04-21 16:01:24,635 MainThread 7416 1
2022-04-21 16:01:24,635 MainThread 7416 2
2022-04-21 16:01:24,635 MainThread 7416 3
2022-04-21 16:01:24,635 MainThread 7416 4
2022-04-21 16:01:24,635 MainThread 7416 5
Process finished with exit code 0
3、ProcessPoolExecutor
# concurrent.futures
# ThreadPoolExecutor
from concurrent.futures import ProcessPoolExecutor
import time
import logging
import threading
FORMAT = "%(process)8s %(processName)16s %(thread)8d %(message)s"
logging.basicConfig(level=logging.INFO, format=FORMAT)
def worker(n):
logging.info('enter process ~~~~~~~~~~~~~~')
time.sleep(n)
logging.info('finished ~~~~~~~~~~~~')
return n
if __name__ == '__main__':
executor = ProcessPoolExecutor(max_workers=3)
fs = []
for i in range(6):
future = executor.submit(worker, i) # 异步，非阻塞，执行完毕后立马执行下一条命令
logging.info(future)
fs.append(future)
# logging.info('-' * 55)
# for i in range(3):
# future = executor.submit(worker, 5) # 异步，非阻塞，执行完毕后立马执行下一条命令
# logging.info(future)
# logging.info('=' * 55)
# executor.shutdown()
# logging.info('*' * 55)
while True:
flag = True
for f in fs:
logging.info(f.done())
flag = flag and f.done()
if flag:
executor.shutdown()
logging.info(threading.enumerate())
break
time.sleep(1)
logging.info(threading.enumerate())
for f in fs:
logging.info(f.result())
19532 MainProcess 31296 <Future at 0x19cf1028940 state=running>
19532 MainProcess 31296 <Future at 0x19cf1032bb0 state=pending>
19532 MainProcess 31296 <Future at 0x19cf1042ca0 state=pending>
19532 MainProcess 31296 <Future at 0x19cf1042d90 state=pending>
19532 MainProcess 31296 <Future at 0x19cf1042910 state=pending>
19532 MainProcess 31296 <Future at 0x19cf1042e50 state=pending>
19532 MainProcess 31296 False
19532 MainProcess 31296 False
19532 MainProcess 31296 False
19532 MainProcess 31296 False
19532 MainProcess 31296 False
19532 MainProcess 31296 False
46924 SpawnProcess-1 49624 enter process ~~~~~~~~~~~~~~
46924 SpawnProcess-1 49624 finished ~~~~~~~~~~~~
46924 SpawnProcess-1 49624 enter process ~~~~~~~~~~~~~~
18288 SpawnProcess-2 41360 enter process ~~~~~~~~~~~~~~
34664 SpawnProcess-3 50828 enter process ~~~~~~~~~~~~~~
19532 MainProcess 31296 [<_MainThread(MainThread, started 31296)>, <Thread(QueueManagerThread, started daemon 38880)>, <Thread(QueueFeederThread, started daemon 49672)>]
19532 MainProcess 31296 True
19532 MainProcess 31296 False
19532 MainProcess 31296 False
19532 MainProcess 31296 False
19532 MainProcess 31296 False
19532 MainProcess 31296 False
46924 SpawnProcess-1 49624 finished ~~~~~~~~~~~~
46924 SpawnProcess-1 49624 enter process ~~~~~~~~~~~~~~
19532 MainProcess 31296 [<_MainThread(MainThread, started 31296)>, <Thread(QueueManagerThread, started daemon 38880)>, <Thread(QueueFeederThread, started daemon 49672)>]
19532 MainProcess 31296 True
19532 MainProcess 31296 True
19532 MainProcess 31296 False
19532 MainProcess 31296 False
19532 MainProcess 31296 False
19532 MainProcess 31296 False
18288 SpawnProcess-2 41360 finished ~~~~~~~~~~~~
18288 SpawnProcess-2 41360 enter process ~~~~~~~~~~~~~~
19532 MainProcess 31296 [<_MainThread(MainThread, started 31296)>, <Thread(QueueManagerThread, started daemon 38880)>, <Thread(QueueFeederThread, started daemon 49672)>]
19532 MainProcess 31296 True
19532 MainProcess 31296 True
19532 MainProcess 31296 True
19532 MainProcess 31296 False
19532 MainProcess 31296 False
19532 MainProcess 31296 False
34664 SpawnProcess-3 50828 finished ~~~~~~~~~~~~
19532 MainProcess 31296 [<_MainThread(MainThread, started 31296)>, <Thread(QueueManagerThread, started daemon 38880)>, <Thread(QueueFeederThread, started daemon 49672)>]
19532 MainProcess 31296 True
19532 MainProcess 31296 True
19532 MainProcess 31296 True
19532 MainProcess 31296 True
19532 MainProcess 31296 False
19532 MainProcess 31296 False
19532 MainProcess 31296 [<_MainThread(MainThread, started 31296)>, <Thread(QueueManagerThread, started daemon 38880)>, <Thread(QueueFeederThread, started daemon 49672)>]
19532 MainProcess 31296 True
19532 MainProcess 31296 True
19532 MainProcess 31296 True
19532 MainProcess 31296 True
19532 MainProcess 31296 False
19532 MainProcess 31296 False
46924 SpawnProcess-1 49624 finished ~~~~~~~~~~~~
19532 MainProcess 31296 [<_MainThread(MainThread, started 31296)>, <Thread(QueueManagerThread, started daemon 38880)>, <Thread(QueueFeederThread, started daemon 49672)>]
19532 MainProcess 31296 True
19532 MainProcess 31296 True
19532 MainProcess 31296 True
19532 MainProcess 31296 True
19532 MainProcess 31296 True
19532 MainProcess 31296 False
19532 MainProcess 31296 [<_MainThread(MainThread, started 31296)>, <Thread(QueueManagerThread, started daemon 38880)>, <Thread(QueueFeederThread, started daemon 49672)>]
19532 MainProcess 31296 True
19532 MainProcess 31296 True
19532 MainProcess 31296 True
19532 MainProcess 31296 True
19532 MainProcess 31296 True
19532 MainProcess 31296 False
18288 SpawnProcess-2 41360 finished ~~~~~~~~~~~~
19532 MainProcess 31296 [<_MainThread(MainThread, started 31296)>, <Thread(QueueManagerThread, started daemon 38880)>, <Thread(QueueFeederThread, started daemon 49672)>]
19532 MainProcess 31296 True
19532 MainProcess 31296 True
19532 MainProcess 31296 True
19532 MainProcess 31296 True
19532 MainProcess 31296 True
19532 MainProcess 31296 True
19532 MainProcess 31296 [<_MainThread(MainThread, started 31296)>]
19532 MainProcess 31296 0
19532 MainProcess 31296 1
19532 MainProcess 31296 2
19532 MainProcess 31296 3
19532 MainProcess 31296 4
19532 MainProcess 31296 5
Process finished with exit code 0
4、ProcessPoolExecutor 支持上下文管理
# concurrent.futures
# ThreadPoolExecutor
from concurrent.futures import ProcessPoolExecutor
import time
import logging
import threading
FORMAT = "%(process)8s %(processName)16s %(thread)8d %(message)s"
logging.basicConfig(level=logging.INFO, format=FORMAT)
def worker(n):
logging.info('enter process ~~~~~~~~~~~~~~')
time.sleep(n)
logging.info('finished ~~~~~~~~~~~~')
return n
if __name__ == '__main__':
executor = ProcessPoolExecutor(max_workers=3)
with executor:
fs = []
for i in range(6):
future = executor.submit(worker, i) # 异步，非阻塞，执行完毕后立马执行下一条命令
logging.info(future)
fs.append(future)
while True:
time.sleep(2)
logging.info(threading.enumerate())
flag = True
for f in fs:
logging.info(f.done())
flag = flag and f.done()
if flag:
break
for f in fs:
logging.info(f.result())
logging.info(executor)
37256 MainProcess 42568 <Future at 0x2622cb99940 state=running>
37256 MainProcess 42568 <Future at 0x2622cba3bb0 state=running>
37256 MainProcess 42568 <Future at 0x2622cbb2d30 state=running>
37256 MainProcess 42568 <Future at 0x2622cbb2cd0 state=pending>
37256 MainProcess 42568 <Future at 0x2622cbb2fd0 state=pending>
37256 MainProcess 42568 <Future at 0x2622cbc7040 state=pending>
2600 SpawnProcess-1 8480 enter process ~~~~~~~~~~~~~~
2600 SpawnProcess-1 8480 finished ~~~~~~~~~~~~
25636 SpawnProcess-2 46040 enter process ~~~~~~~~~~~~~~
2600 SpawnProcess-1 8480 enter process ~~~~~~~~~~~~~~
10336 SpawnProcess-3 42296 enter process ~~~~~~~~~~~~~~
25636 SpawnProcess-2 46040 finished ~~~~~~~~~~~~
25636 SpawnProcess-2 46040 enter process ~~~~~~~~~~~~~~
37256 MainProcess 42568 [<_MainThread(MainThread, started 42568)>, <Thread(QueueManagerThread, started daemon 10416)>, <Thread(QueueFeederThread, started daemon 39908)>]
37256 MainProcess 42568 True
37256 MainProcess 42568 True
37256 MainProcess 42568 False
37256 MainProcess 42568 False
37256 MainProcess 42568 False
37256 MainProcess 42568 False
2600 SpawnProcess-1 8480 finished ~~~~~~~~~~~~
2600 SpawnProcess-1 8480 enter process ~~~~~~~~~~~~~~
10336 SpawnProcess-3 42296 finished ~~~~~~~~~~~~
37256 MainProcess 42568 [<_MainThread(MainThread, started 42568)>, <Thread(QueueManagerThread, started daemon 10416)>, <Thread(QueueFeederThread, started daemon 39908)>]
37256 MainProcess 42568 True
37256 MainProcess 42568 True
37256 MainProcess 42568 True
37256 MainProcess 42568 True
37256 MainProcess 42568 False
37256 MainProcess 42568 False
25636 SpawnProcess-2 46040 finished ~~~~~~~~~~~~
37256 MainProcess 42568 [<_MainThread(MainThread, started 42568)>, <Thread(QueueManagerThread, started daemon 10416)>, <Thread(QueueFeederThread, started daemon 39908)>]
37256 MainProcess 42568 True
37256 MainProcess 42568 True
37256 MainProcess 42568 True
37256 MainProcess 42568 True
37256 MainProcess 42568 True
37256 MainProcess 42568 False
2600 SpawnProcess-1 8480 finished ~~~~~~~~~~~~
37256 MainProcess 42568 [<_MainThread(MainThread, started 42568)>, <Thread(QueueManagerThread, started daemon 10416)>, <Thread(QueueFeederThread, started daemon 39908)>]
37256 MainProcess 42568 True
37256 MainProcess 42568 True
37256 MainProcess 42568 True
37256 MainProcess 42568 True
37256 MainProcess 42568 True
37256 MainProcess 42568 True
37256 MainProcess 42568 0
37256 MainProcess 42568 1
37256 MainProcess 42568 2
37256 MainProcess 42568 3
37256 MainProcess 42568 4
37256 MainProcess 42568 5
37256 MainProcess 42568 <concurrent.futures.process.ProcessPoolExecutor object at 0x000002622C864D90>
Process finished with exit code 0
版权声明：本文为weixin_44983653原创文章，遵循CC 4.0 BY-SA版权协议，转载请附上原文出处链接和本声明。