CheungSSHPool的调用会导致系统线程资源耗尽
第一次提issue,不知道怎么贴代码
问题都写在代码注释里了,这次将就看吧
后续我会就提出的bug修改部分代码
--- 文件 cheungssh_auto_thread.py
class AutoGetThreadNum:
def init(self):
self.cpu=100 ### 1.cpu数目最好动态检测,不要写死,或者在写一个方法修改cpu数目
self.thread=5
def auto_thread(self): #### 2.cpu数目不动态检测的话,这个方法其实是没有用的,进程数始终是200,每次申请200而且有些情况下不释放,例如执行top命令,就会导致系统线程耗尽,影响整个系统
if self.cpu>90:
self.thread=200
elif self.cpu>80:
self.thread=150
elif self.cpu>70:
self.thread=100
elif self.cpu>60:
self.thread=50
elif self.cpu>50:
self.thread=40
elif self.cpu>30:
self.thread=20
elif self.cpu>20:
self.thread=5
return self.thread
--- 文件 cheungssh_thread_queue.py
import threading,Queue,time
from cheungssh_auto_thread import AutoGetThreadNum
from cheungssh_modul_controler import CheungSSHControler
from cheungssh_error import CheungSSHError
import os,sys,json
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mysite.settings")
sys.path.append('/home/cheungssh/mysite')
from django.core.cache import cache
class CheungSSHThreadAdmin(object):
def __init__(self):
self.REDIS=cache.master_client
def run(self,parameter={}):
cheungssh_info={"status":True,"content":""}
try:
task_type=parameter["task_type"]
tid=parameter["tid"]
multi_thread=parameter["multi_thread"]
if not type(multi_thread) ==type(False):raise CheungSSHError("CHB0000000010")
if task_type=="cmd":
cmd=parameter["cmd"]
servers=parameter["servers"]
if not type(servers)==type([]):raise CheungSSHError("CHB0000000011")
total="total.%s" % tid
current="current.%s" %tid
self.REDIS.set(total,len(servers))
self.REDIS.set(current,0)
#CheungSSHConnector.progress[total]=len(servers)
#CheungSSHConnector.progress[current]=0
if multi_thread:
pool=CheungSSHPool()
for s in servers:
controler=CheungSSHControler()
param={"cmd":cmd,"sid":s,"tid":tid}
pool.add_task(controler.command_controler,param)
else:
for s in servers:
controler=CheungSSHControler()
controler.command_controler(cmd=cmd,sid=s,tid=tid)
elif task_type=="file":
pass
else:
raise CheungSSHError("CHB0000000009")
except Exception,e:
cheungssh_info["content"]=str(e)
cheungssh_info["status"]=False
print cheungssh_info
return cheungssh_info
class CheungSSHThread(threading.Thread):
def init(self,queue):
threading.Thread.init(self)
self.queue=queue
self.daemon=True
self.start() ### 5. start自动调用run()方法
def run(self):
while True:### 4. 这里,每起一个thread,这个thread都不会自动退出,最后导致系统线程被耗尽
try:
func,kws=self.queue.get()
func(**kws)
except Exception,e:
print e
pass
self.queue.task_done()
class CheungSSHPool(AutoGetThreadNum):
def __init__(self):
AutoGetThreadNum.__init__(self)
self.thread_num=self.auto_thread()
self.queue=Queue.Queue(self.thread_num)
for i in range(self.thread_num):
CheungSSHThread(self.queue) ### 3. 每次会申请200个线程,而每一个的thread的run方法是一个没有break的死循环,线程永远不会退出。详见 ### 4.
def add_task(self,func,dict):
self.queue.put((func,dict))
def all_complete(self):
self.queue.join()
def test_func(**kws):
time.sleep(0.5)
print "哈哈,这里是你穿的参数",kws
if name=='main':
p=CheungSSHPool()
for i in range(20):
p.add_task(test_func,{"username":i})
p.all_complete()