Python ThreadPool.add_job方法代码示例

# 需要导入模块: from threadpool import ThreadPool [as 别名]
# 或者: from threadpool.ThreadPool import add_job [as 别名]
def deskWriteJob( arglist, kwargs):
    info = arglist[0]
    #print "in the thread is:" + str(info)
    webservername = str(info["dev_name"])
    if info["dev_name"] not in g_client_list:
        g_lock.acquire()
        print "new webserver"
        print info["dev_name"]
        g_client_list[info["dev_name"]] = info["dev_name"]
        g_lock_dict[info["dev_name"]] = thread.allocate_lock()
        newwebserver = WebServer()
        newwebserver.SetDevname(info["dev_name"])
        g_websever[info["dev_name"]] = newwebserver
        g_lock.release()
    temp_lock = g_lock_dict[info["dev_name"]]
    temp_lock.acquire()
    temp_webserver = g_websever[info["dev_name"]]
    temp_webserver.JobDespatcher(info)
    temp_lock.release()


while True:
    data,addr = udp_server.readmeg()
    parsertest = Parser()
    info = parsertest.parserMeg(data)
    #print "info json = "
    #print info
    tdpool.add_job( deskWriteJob, info, info)
    #print "out the thread is:" + str(info)

# 需要导入模块: from threadpool import ThreadPool [as 别名]
# 或者: from threadpool.ThreadPool import add_job [as 别名]
class HttpDeamon(object):
    def __init__(self, ip, port, threadnum=100):
        self.ip = ip
        self.port = port
        self.threadnum = threadnum
        self.tp = ThreadPool(self.threadnum)
        self.id = 0 
        self.server_socket = startup(self.ip, self.port)
        
        #bug fix, cannot use local startup, why?
        #self.server_socket = self.startup()
        
        #for epoll, async, later consider how to imple
        self.inQueue = Queue.Queue()
        self.outQueue  = Queue.Queue()
        
    def run(self):
        """block version, odd enough, this is a multi thread version"""
        while True:
            client_socket, (ip, port) = self.server_socket.accept()
            logger.debug('receive request from {0}:{1}'.format(ip, port))
            self.tp.add_job(task, self.id, client_socket)
            self.id += 1
    
    def epoll_run(self):
        """async version"""
        try:
            epoll_fd = select.epoll()
            epoll_fd.register(self.server_socket.fileno(), select.EPOLLIN)
        except select.error, msg:
            exit_error(msg)

        connections = {}
        addresses = {}
        datalist = {}
        while True:
            epoll_list = epoll_fd.poll()
            for fd, events in epoll_list:
                if fd == self.server_socket.fileno():
                    conn, addr = self.server_socket.accept()
                    logger.debug("accept connection from {0}:{1} fd={2}".format(addr[0], addr[1], conn.fileno()))
                    conn.setblocking(0)
                    epoll_fd.register(conn.fileno(), select.EPOLLIN|select.EPOLLET) 
                    connections[conn.fileno()] = conn
                    addresses[conn.fileno()] = addr
                elif select.EPOLLIN & events:
                    datas = ''
                    while True:
                        try:
                            data = connections[fd].recv(10)
                            if not data and not datas:
                                epoll_fd.unregister(fd)
                                connections[fd].close()
                                logger.debug("{0}:{1} closed".format(addresses[fd][0], addresses[fd][1]))
                                break
                            else:
                                datas += data
                        except socket.error, msg:
                            if msg.errno == errno.EAGAIN:
                                logger.debug("{0} receive {1}".format(fd, datas))
                                datalist[fd] = datas
                                epoll_fd.modify(fd, select.EPOLLET|select.EPOLLOUT)
                                break
                            else:
                                epoll_fd.unregister(fd)
                                connections[fd].close()
                                logger.error(msg)
                                break
                elif select.EPOLLHUP & events:
                    epoll_fd.unregister(fd)
                    connections[fd].close()
                    logger.debug("{0}:{1} closed".format(addresses[fd][0], addresses[fd][1]))
                elif select.EPOLLOUT & events:
                    send_len = 0
                    while True:
                        send_len += connections[fd].send(datalist[fd][send_len:])
                        if send_len == len(datalist[fd]):
                            break
                    epoll_fd.modify(fd, select.EPOLLIN|select.EPOLLET)

# 需要导入模块: from threadpool import ThreadPool [as 别名]
# 或者: from threadpool.ThreadPool import add_job [as 别名]
			print('error')
		else:
			item_curpage=itemprog['friendList'].findall(html_content)
			print('friendList: renrenId={}, curpage={}, items={}, timecost1={},timecost2={}'.format(rid,curpage,len(item_curpage),pause-fstart,fstop-pause))
			if len(item_curpage) < 1:#all pages request$
				break
			else:
				friendList.update(item_curpage)
				n=len(item_curpage)
	return friendList,curpage,n

if __name__=='__main__':
	login()
	nThread=10
	tp=ThreadPool(nThread)
	start=time.time()
	for i in range(1):
		tp.add_job(friendList,'230760442')#wei
		tp.add_job(friendList,'250068531')#ju
		#tp.add_job(friendList,'242934804')#wei
		#tp.add_job(friendList,'233960464')#wei
	#tp.add_job(friendList,'287286312')
	#tp.add_job(friendList,'240303471')
	tstop=time.time()
	tp.wait_for_complete()
	stop=time.time()
	res=tp.get_res(friendList)
	for key,meth_res in res.items():
		print('{} has {} items'.format(key,len(meth_res[0])))
	print("{} items,timecost={}".format(len(res.keys()),stop-start))