Python 3 线程模型,进程模型记录
最近需要使用 python3 多线程处理大型数据,顺道探究了一下,python3 的线程模型的情况,下面进行简要记录;
多线程运行的优点:
- 使用线程可以把程序中占用时间较长的任务放到后台去处理;
- 用户界面可以更加吸引人,并且不阻塞界面的运行;
- 程序运行的速度可以更快;
- 充分利用cpu多核的特征进行处理;
内核线程:由操作系统内核创建和撤销;
用户线程:不需要内核支持在用户程序中实现的线程;
python3 中的多线程:
- _thread 提供了一些原始的api 用于写多线程程序;
- threading 提供了更加便利的接口
- 两者都是python3内置的线程模块
#!/usr/bin/env python import _thread def print_time( threadname, delay): print (threadname) count = 0 while 1: pass count += 1 try: _thread.start_new_thread( print_time, ("thread-1", 1, ) ) _thread.start_new_thread( print_time, ("thread-2", 2, ) ) _thread.start_new_thread( print_time, ("thread-3", 2, ) ) _thread.start_new_thread( print_time, ("thread-4", 2, ) ) _thread.start_new_thread( print_time, ("thread-5", 2, ) ) _thread.start_new_thread( print_time, ("thread-6", 2, ) ) _thread.start_new_thread( print_time, ("thread-7", 2, ) ) _thread.start_new_thread( print_time, ("thread-8", 2, ) ) _thread.start_new_thread( print_time, ("thread-9", 2, ) ) _thread.start_new_thread( print_time, ("thread-10", 2, ) ) _thread.start_new_thread( print_time, ("thread-11", 2, ) ) _thread.start_new_thread( print_time, ("thread-12", 2, ) ) _thread.start_new_thread( print_time, ("thread-13", 2, ) ) _thread.start_new_thread( print_time, ("thread-14", 2, ) ) _thread.start_new_thread( print_time, ("thread-15", 2, ) ) except: print ("error: can't start thread!") while 1: pass
#!/usr/bin/env python3 import threading import time exitflag = 0 class mythread (threading.thread): def __init__(self, threadid, name, counter): threading.thread.__init__(self) self.threadid = threadid self.name = name self.counter = counter def run(self): print ("start" + self.name) print_time(self.name, self.counter, 5) print ("exit" + self.name) def print_time(threadname, delay, counter): while counter: if exitflag: threadname.exit() time.sleep(delay) print ("%s: %s" % (threadname, time.ctime(time.time()))) counter -= 1 thread1 = mythread(1, "thread-1", 1) thread2 = mythread(2, "thread-2", 2) thread1.start() thread2.start() thread1.join() thread2.join() print ("exit!")
python 的多线程 threading 有时候并不是特别理想. 最主要的原因是就是, python 的设计上, 有一个必要的环节, 就是 global interpreter lock (gil). 这个东西让 python 还是一次性只能处理一个东西:
尽管python完全支持多线程编程, 但是解释器的c语言实现部分在完全并行执行时并不是线程安全的。 实际上,解释器被一个全局解释器锁保护着,它确保任何时候都只有一个python线程执行。 gil最大的问题就是python的多线程程序并不能利用多核cpu的优势 (比如一个使用了多个线程的计算密集型程序只会在一个单cpu上面运行);
gil只会影响到那些严重依赖cpu的程序(比如计算型的)。 如果你的程序大部分只会涉及到i/o,比如网络交互,那么使用多线程就很合适, 因为它们大部分时间都在等待;
import threading from queue import queue import copy import time def job(l, q): res = sum(l) q.put(res) def multithreading(l): q = queue() threads = [] for i in range(4): t = threading.thread(target=job, args=(copy.copy(l), q), name='t%i' % i) t.start() threads.append(t) [t.join() for t in threads] total = 0 for _ in range(4): total += q.get() print(total) def normal(l): total = sum(l) print(total) if __name__ == '__main__': l = list(range(1000000)) s_t = time.time() normal(l*4) print('normal: ',time.time()-s_t) s_t = time.time() multithreading(l) print('multithreading: ', time.time()-s_t)
#!/usr/bin/env python import multiprocessing as mp import threading as td def job(a,b): while 1: pass t1 = td.thread(target=job,args=(1,2)) t2 = td.thread(target=job,args=(1,2)) t3 = td.thread(target=job,args=(1,2)) t4 = td.thread(target=job,args=(1,2)) t5 = td.thread(target=job,args=(1,2)) t6 = td.thread(target=job,args=(1,2)) t7 = td.thread(target=job,args=(1,2)) t8 = td.thread(target=job,args=(1,2)) t9 = td.thread(target=job,args=(1,2)) t10 = td.thread(target=job,args=(1,2)) t11 = td.thread(target=job,args=(1,2)) t12 = td.thread(target=job,args=(1,2)) t13 = td.thread(target=job,args=(1,2)) t14 = td.thread(target=job,args=(1,2)) t15 = td.thread(target=job,args=(1,2)) t16 = td.thread(target=job,args=(1,2)) # p1 = mp.process(target=job,args=(1,2)) # p2 = mp.process(target=job,args=(1,2)) # p3 = mp.process(target=job,args=(1,2)) # p4 = mp.process(target=job,args=(1,2)) # p5 = mp.process(target=job,args=(1,2)) # p6 = mp.process(target=job,args=(1,2)) # p7 = mp.process(target=job,args=(1,2)) # p8 = mp.process(target=job,args=(1,2)) # p9 = mp.process(target=job,args=(1,2)) # p10 = mp.process(target=job,args=(1,2)) # p11 = mp.process(target=job,args=(1,2)) # p12 = mp.process(target=job,args=(1,2)) # p13 = mp.process(target=job,args=(1,2)) # p14 = mp.process(target=job,args=(1,2)) # p15 = mp.process(target=job,args=(1,2)) # p16 = mp.process(target=job,args=(1,2)) t1.start() t2.start() t3.start() t4.start() t5.start() t6.start() t7.start() t8.start() t9.start() t10.start() t11.start() t12.start() t13.start() t14.start() t15.start() t16.start() # p1.start() # p2.start() # p3.start() # p4.start() # p5.start() # p6.start() # p7.start() # p8.start() # p9.start() # p10.start() # p11.start() # p12.start() # p13.start() # p14.start() # p15.start() # p16.start() t1.join() t2.join() t3.join() t4.join() t5.join() t6.join() t7.join() t8.join() t9.join() t10.join() t11.join() t12.join() t13.join() t14.join() t15.join() t16.join() # p1.join() # p2.join() # p3.join() # p4.join() # p5.join() # p6.join() # p7.join() # p8.join() # p9.join() # p10.join() # p11.join() # p12.join() # p13.join() # p14.join() # p15.join() # p16.join()
使用python multiprocess 包能够发挥多核cpu并行处理能力:
- multiprocess 接口和threading 使用的接口一样;
保持更新,转载请注明出处,更多内容请关注cnblogs.com/xuyaowen;
参考链接:
https://morvanzhou.github.io/tutorials/python-basic/threading/5-gil/
上一篇: python 序列化
下一篇: layer 防止重复弹出的办法