Python中运行并行任务技巧
示例
标准线程多进程,生产者/消费者示例:
worker越多,问题越大
# -*- coding: utf8 -*-
import os
import time
import queue
import threading
from pil import image
def create_thumbnail(filename, size=(128, 128)):
try:
fp, fmt = filename.rsplit('.', 1)
im = image.open(filename)
im.thumbnail(size, image.antialias)
im.save((fp + '_'+'x'.join(str(i) for i in size) + '.'+fmt), im.format)
return '%s thumbnail success!' % filename
except exception:
return '%s thumbnail failed!' % filename
def get_image_paths(folder):
return [os.path.join(folder, f) for f in os.listdir(folder) if 'png' in f]
class consumer(threading.thread):
def __init__(self, queue):
threading.thread.__init__(self)
self._queue = queue
def run(self):
while true:
content = self._queue.get()
if isinstance(content, str) and content == 'quit':
break
respone = create_thumbnail(content)
print 'bye bye!'
def producer():
filenames = get_image_paths('images')
queue = queue.queue()
worker_threads = build_worker_pool(queue, 4)
start_time = time.time()
for filename in filenames:
queue.put(filename)
for worker in worker_threads:
queue.put('quit')
for worker in worker_threads:
worker.join()
print time.time() - start_time
def build_worker_pool(queue, size):
workers = []
for _ in range(size):
worker = consumer(queue)
worker.start()
workers.append(worker)
return workers
if __name__ == '__main__':
producer()
map
map能够处理集合按顺序遍历,最终将调用产生的结果保存在一个简单的集合当中。
# -*- coding: utf8 -*-
import os
import time
from multiprocessing import pool
from pil import image
def create_thumbnail(filename, size=(128, 128)):
try:
fp, fmt = filename.rsplit('.', 1)
im = image.open(filename)
im.thumbnail(size, image.antialias)
im.save((fp + '_'+'x'.join(str(i) for i in size) + '.'+fmt), im.format)
return '%s thumbnail success!' % filename
except exception:
return '%s thumbnail failed!' % filename
def get_image_paths(folder):
return [os.path.join(folder, f) for f in os.listdir(folder) if 'png' in f]
def main():
filenames = get_image_paths('images')
start_time = time.time()
pool = pool(4)
pool.map(create_thumbnail, filenames)
pool.close()
pool.join()
print time.time() - start_time
if __name__ == '__main__':
main()
推荐阅读
-
vscode如何运行python? vs code新建并运行python代码的技巧
-
「译」在 python 中,如果 x 是 list,为什么 x += "ha" 可以运行,而 x = x + "ha" 却抛出异常呢?
-
laravel 在正式环境运行队列命令,为什么要过1分钟才会去消费队列中的任务?
-
bat文件中运行python脚本方法
-
提升Python运行速度的5个小技巧
-
python开启多个子进程并行运行的方法
-
Python的Django框架中URLconf相关的一些技巧整理
-
使用python将大量数据导出到Excel中的小技巧分享
-
Windows 配置Apache以便在浏览器中运行Python script的CGI模式
-
分享Python中list的各项操作技巧