Python ThreadPool.ThreadPool类代码示例

def main():
	appName = sys.argv[0]
	logging.basicConfig(
#		filename = './log/' + appName + '_' + time.strftime("%Y%m%d_%H%M%S") + '.log',
		datefmt = '%Y-%m%d %H:%M:%S',
		format = '%(asctime)s | %(levelname)s | %(name)s | %(message)s',
		level = logging.INFO
	)
	func_name = sys._getframe().f_code.co_name
	logger = logging.getLogger(func_name)
	logger.info('Started')

	parseArgs()

	logger.debug('requests.session')
	session = requests.session()
	# `mount` a custom adapter that retries failed connections for HTTP and HTTPS requests.
	session.mount("http://", requests.adapters.HTTPAdapter(max_retries=10))

	logger.debug('session.post')
	r = session.post(LOGIN_URL, data = LOGIN_PARAM)

	idResultEnd = idResult
	idResultBegin = idResult - quantResult
	# 1) Init a Thread pool with the desired number of threads
	logger.debug('ThreadPool')
	pool = ThreadPool(10)
	logger.debug('for idAtual in xrange(%d, %d, -1)' % (idResultEnd, idResultBegin))
	for idAtual in xrange(idResultEnd, idResultBegin, -1):
		# 2) Add the task to the queue
		pool.add_task(downloadResult, session, idAtual)
	# 3) Wait for completion
	pool.wait_completion()
	###
	logger.info('Finished')

 def initializeThreadPool(self,statusEvent):
     ThreadPool.clear()
     ThreadPool.mapPackageToCycle=self.mapPackageToCycle
     ThreadPool.listAvailableCyclicPackages=self.listAvailableCyclicPackages
     ThreadPool.listBuildOptionPackages=self.listBuildOptionPackages
     ThreadPool.pkgBuildOptionFile=self.pkgBuildOptionFile
     ThreadPool.logger=self.logger
     ThreadPool.statusEvent=statusEvent

    def test_connectivity(self,  timeout):
        """
        Tests connectivity with each server of the target_list and returns 
        the list of online servers.
        """
        
        # Use a thread pool to connect to each server
        thread_pool = ThreadPool()
        for target_str in self._target_list:
            thread_pool.add_job((self._test_server,
                                (target_str, timeout)))
            
        nb_threads = min(len(self._target_list), self.MAX_THREADS)
        thread_pool.start(nb_threads)
        
        # Recover valid targets
        for (job, target) in thread_pool.get_result():
            self._targets_OK.append(target)
            yield target
                        
        # Store invvalid targets
        for (job, exception) in thread_pool.get_error():
            self._targets_ERR.append(exception)

        thread_pool.join()
        return   

    def test_server_list(cls, target_list, shared_settings):
        """
        Tests connectivity with each server of the target_list and returns
        the list of online servers.
        """

        # Use a thread pool to connect to each server
        thread_pool = ThreadPool()
        for target_str in target_list:
            thread_pool.add_job((cls._test_server, (target_str, shared_settings)))

        nb_threads = min(len(target_list), cls.MAX_THREADS)
        thread_pool.start(nb_threads)

        # Return valid targets
        for (job, target) in thread_pool.get_result():
            yield target

        # Use None as a sentinel
        yield None

        # Return invalid targets
        for (job, exception) in thread_pool.get_error():
            yield exception

        thread_pool.join()
        return

def test():
	print('start testing')
	tp = ThreadPool()
	for i in range(15):
		time.sleep(0.1)
		tp.add_job(test_job, i)
	
	t = time.time()
	tp.wait_for_complete()
	
	print('s:'+str(time.time()-t))
	print('end testing')

 def buildGivenPackages (self, listPackages):
     returnVal=self.calculateParams(listPackages)
     if not returnVal:
         self.logger.error("Unable to set paramaters. Terminating the package manager.")
         return False
     
     statusEvent=threading.Event()
     numWorkerThreads=self.calculatePossibleNumWorkerThreads()
     if numWorkerThreads > 8:
         numWorkerThreads = 8
     if numWorkerThreads == 0:
         return False
      
     self.initializeScheduler(statusEvent)
     self.initializeThreadPool(statusEvent)
     
     i=0
     while i < numWorkerThreads:
         workerName="WorkerThread"+str(i)
         ThreadPool.addWorkerThread(workerName)
         ThreadPool.startWorkerThread(workerName)
         i = i + 1
     
     statusEvent.wait()
     Scheduler.stopScheduling=True
     self.logger.info("Waiting for all remaining worker threads")
     listWorkerObjs=ThreadPool.getAllWorkerObjects()
     for w in listWorkerObjs:
         w.join()
         
     setFailFlag=False
     allPackagesBuilt=False
     
     if Scheduler.isAnyPackagesFailedToBuild():
         setFailFlag=True
     
     if Scheduler.isAllPackagesBuilt():
         allPackagesBuilt=True
     
     if setFailFlag:
         self.logger.error("Some of the packages failed:")
         self.logger.error(Scheduler.listOfFailedPackages)
         return False
     
     if not setFailFlag:
         if allPackagesBuilt:
             self.logger.info("All packages built successfully")
         else:
             self.logger.error("Build stopped unexpectedly.Unknown error.")
             return False
     
     self.logger.info("Terminated")
     return True

    def buildGivenPackages (self, listPackages, buildThreads):
        if constants.rpmCheck:
            alreadyBuiltRPMS=self.readAlreadyAvailablePackages()
            listPackages=list(set(listPackages)|(set(constants.listMakeCheckRPMPkgtoInstall)-set(alreadyBuiltRPMS)))

        returnVal=self.calculateParams(listPackages)
        if not returnVal:
            self.logger.error("Unable to set paramaters. Terminating the package manager.")
            raise Exception("Unable to set paramaters")

        statusEvent=threading.Event()
        self.initializeScheduler(statusEvent)
        self.initializeThreadPool(statusEvent)

        i=0
        while i < buildThreads:
            workerName="WorkerThread"+str(i)
            ThreadPool.addWorkerThread(workerName)
            ThreadPool.startWorkerThread(workerName)
            i = i + 1

        statusEvent.wait()
        Scheduler.stopScheduling=True
        self.logger.info("Waiting for all remaining worker threads")
        listWorkerObjs=ThreadPool.getAllWorkerObjects()
        for w in listWorkerObjs:
            w.join()

        setFailFlag=False
        allPackagesBuilt=False
        if Scheduler.isAnyPackagesFailedToBuild():
            setFailFlag=True

        if Scheduler.isAllPackagesBuilt():
            allPackagesBuilt=True

        if setFailFlag:
            self.logger.error("Some of the packages failed:")
            self.logger.error(Scheduler.listOfFailedPackages)
            raise Exception("Failed during building package")

        if not setFailFlag:
            if allPackagesBuilt:
                self.logger.info("All packages built successfully")
            else:
                self.logger.error("Build stopped unexpectedly.Unknown error.")
                raise Exception("Unknown error")

        self.logger.info("Terminated")

class Spider(object):

    unvisited_urls = dict()    # 存储未访问列表
    visited_urls = dict()
    task_queue = Queue()       # 任务队列

    def __init__(self, seed):
        self.unvisited_urls = seed
        self.gather = Gather()
        self.url = ''
        self.pool = ThreadPool(self.task_queue, 2)   # 启动4个线程
        self.lock = threading.Lock()                 # 初始化锁

    '''
    start  从未访问url列表中取出一条未访问过的url，与run一起打包放到任务队列中
    '''
    def start(self):
        # 在这里读取配置文件
        for key in self.unvisited_urls:
            self.task_queue.put((self.run, key))
        self.pool.wait_all_complete()
        # 在这里把已完成队列和未完成队列写入文件中。

    def run(self, url, *, save_dir, name):           # 工作线程，下载网页，保存网页，提取新的链接,线程需要提供编号，以便存储(命名参数)
        # time.sleep(5)   # 防止ip被封
        result = self.gather.download_webpage(url)   # 下载网页
        if forbidden_count >= 50:                    # 结束整个程序
            print("exit function")
            return False
        if result[0] and self.gather.doc_save(result[1], save_dir, name):      # 保存网页
            self.visited_urls[url] = True            # 标记为已经访问过的网址
            new_url = self.gather.extra_urls(result[1])       # 提取新的url
            self.lock.acquire()
            for u in new_url:                        # 如果该链接既没有访问过，也不存在于待访问列表中，添加网址,这里在多线程中不安全
                if not self.visited_urls.get(u) and not self.unvisited_urls.get(u):
                    # print("%s add" % u)
                    self.unvisited_urls[u] = True
                    self.task_queue.put((self.run, u))
            print("任务队列大小：%d" % self.task_queue.qsize())
            print("已完成任务量：%d" % len(self.visited_urls))
            self.lock.release()

        self.lock.acquire()
        self.unvisited_urls.pop(url)                  # 在未访问列表中删除该网址
        self.lock.release()  # 释放
        print()
        return True

def GetMatchPage(season, league, matches):
	'''
	获取指定联赛的Match页面
	'''
	tp = ThreadPool()
	print('Start updating ' + season + ' ' + league + ' matches...')
	dirLeague = Global.Dir_Root + season + '\\' + league
	
	for matchID in sorted(matches.keys()):
		urlMatch = matches.get(matchID)
		tp.add_job(GetPageText, dirLeague, matchID, urlMatch, len(matches))
	
	tp.wait_for_complete()
	
	if len(matches) > 0:
		print('\n' + season + ' ' +league + ' updating complete!\n')
	else:
		print(season + ' ' + league + ' updating complete!\n')

    def OnExecute(self, button_desc):
        if "run_detect_button" == button_desc:
            desc_fail = "运行检查程序返回失败\r\n"
            desc_success = "所有设备运行检查程序返回成功\r\n"
            cmds_txt = "./cmds/cmds_detect.txt"
        
        if "scan_result_button" == button_desc:
            desc_fail = "查看检查结果返回失败\r\n"
            desc_processexit = "设备检测程序异常退出\r\n"
            desc_success = "所有设备查看检查结果返回成功\r\n"
            cmds_txt = "./cmds/cmds_scan.txt"

        ips = self.ParseIp(self.ip.GetValue().encode('utf-8').split(';'))
        if False == ips:
            return
        cmds = getContent(cmds_txt,"lines")
        if False == cmds:
            return
        pool = ThreadPool( _THREAD_POOL_SIZE )
        for ip in ips: 
            pool.addTask( telnetTask, ip = ip, cmds = cmds)

        #必须放在pool.addFinishCb( finish_event,pool=pool)之前
        def finish_event( *args, **kwargs ):
            pool = kwargs['pool']
            result = pool.show()
            # [[ip,[ret1,ret2]],...]
            result_to_show = ""
            for ip_retlist in result:
                ip = ip_retlist[0]
                retlist = ip_retlist[1]
                if ["fail"] == retlist:
                    result_to_show += ip + " Telnet时抛出了异常\r\n"
                else:
                    t = retlist[len(retlist)-1].split("\r\n")
                    status = t[len(t)-2]
                    if "0" == status:
                        pass 
                    elif "1" == status:                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           
                        result_to_show += ip + desc_fail#desc_fail能访问到
                    elif "2" == status:
                        result_to_show += ip + desc_processexit
                    else:
                        result_to_show += ip + "  脚本返回意料之外的值\r\n"
                        last_cmd_result = retlist[len(retlist)-1]
                        logging.warning(toUnicode(last_cmd_result))
                        
            if "" == result_to_show:
                result_to_show = desc_success

            logging.warning(result_to_show.decode('utf-8'))
            result_to_show = self.result_text.GetValue()+getCurTime().decode("ascii")+"    ".decode("utf-8")+result_to_show.decode("utf-8")
            self.result_text.SetValue(result_to_show)
            self.ButtonStatus("Enable")

        pool.addFinishCb( finish_event,pool=pool)
        pool.start()

        self.ButtonStatus("Disable")

def test_thread_pool():
    pool = ThreadPool(2, 0.5)
    for i in range(2):
        handle = login.login('ssh', 'root', 'n', '10.20.60.23')
        task = (handle.execute_cmd, ('uname -a',), {})
        pool.add_task(task)
    print pool.get_result()
    pool.task_join()

 def __init__(self, is_debug=False):
     self.prev_day = {}
     self.thread_pool = ThreadPool(
         is_debug,
         int(cfg.read('max_threads')),
         int(cfg.read('max_buf')),
     )
     # 注意先清空先前留下的下载任务
     # TODO: 改为继续执行先前未完成的任务
     db.Execute("DELETE FROM `CurrentTask`")

class Worker(QThread):
    update_ui_signal = pyqtSignal()

    def __init__(self, parent=None):
        QThread.__init__(self, parent)
        self.toDisplay = Queue()
        self.threadpool = ThreadPool(max_workers=cpu_count())

    def __del__(self):
        self.threadpool.shutdown()

    def compress_file(self, images, showapp, verbose, imagelist):
        """Start the worker thread."""
        for image in images:
            #FIXME:http://code.google.com/p/pythonthreadpool/issues/detail?id=5
            time.sleep(0.05)
            self.threadpool.add_job(image.compress, None,
                                    return_callback=self.toDisplay.put)
        self.showapp = showapp
        self.verbose = verbose
        self.imagelist = imagelist
        self.start()

    def run(self):
        """Compress the given file, get data from it and call update_table."""
        tp = self.threadpool
        while self.showapp or not (tp._ThreadPool__active_worker_count == 0 and
                                   tp._ThreadPool__jobs.empty()):
            image = self.toDisplay.get()

            self.update_ui_signal.emit()

            if not self.showapp and self.verbose: # we work via the commandline
                if image.retcode == 0:
                    ir = ImageRow(image)
                    print("File: " + ir['fullpath'] + ", Old Size: "
                        + ir['oldfilesizestr'] + ", New Size: "
                        + ir['newfilesizestr'] + ", Ratio: " + ir['ratiostr'])
                else:
                    print("[error] {} could not be compressed".format(image.fullpath), file=sys.stderr)

    def getNextPackageToBuild():
        with Scheduler.lock:
            if Scheduler.stopScheduling:
                return None

            if not Scheduler.listOfPackagesToBuild:
                if Scheduler.event is not None:
                    Scheduler.event.set()

            if Scheduler.listOfPackagesNextToBuild.empty():
                Scheduler._getListNextPackagesReadyToBuild()

            if Scheduler.listOfPackagesNextToBuild.empty():
                return None

            packageTup = Scheduler.listOfPackagesNextToBuild.get()

            package = packageTup[1]
            if Scheduler.listOfPackagesNextToBuild.qsize() > 0:
                ThreadPool.activateWorkerThreads(
                    Scheduler.listOfPackagesNextToBuild.qsize())
            Scheduler.listOfPackagesCurrentlyBuilding.add(package)
            Scheduler.listOfPackagesToBuild.remove(package)
            return package

    def __init__(self, workDir, setup, esPath, token, experiment, userid, sitename, outputDir=None, yodaToOS=False, threads=10, isDaemon=False, process=0, totalProcess=1):
        self.__workDir = workDir
        self.__updateEventRangesDir = os.path.join(self.__workDir, 'updateEventRanges_%s' % process)
        if not os.path.exists(self.__updateEventRangesDir):
            os.makedirs(self.__updateEventRangesDir)
        self.__logFile = os.path.join(workDir, 'EventStager.log')
        self.__setup = setup
        self.__siteMover = S3ObjectstoreSiteMover(setup, useTimerCommand=False)
        self.__esPath = esPath
        self.__token = token
        self.__experiment = experiment
        self.__outputDir = outputDir

        self.__userid = userid
        self.__sitename = sitename

        self.__report =  getInitialTracingReport(userid=self.__userid, sitename=self.__sitename, dsname=None, eventType="objectstore", analysisJob=False, jobId=None, jobDefId=None, dn=self.__userid)

        self.__num_stagingFile = 0
        self.__eventRanges = {}
        self.__eventRanges_staged = {}
        self.__eventRanges_faileStaged = {}

        self.__eventStager = None
        self.__canFinish = False
        self.__status = 'new'
        self.__threads = threads
        self.__isDaemon = isDaemon
        self.__startTime = time.time()

        self.__processedJobs = []
        self.__handlingOthers = 0
        self.__otherProcesses = []
        self.__startWait = None
        self.__waitTime = 15 * 60 # 15 minutes

        self.__yodaToOS = yodaToOS

        if not os.environ.has_key('PilotHomeDir'):
            os.environ['PilotHomeDir'] = os.path.dirname(__file__)

        self.__process = process
        self.__totalProcess = totalProcess

        self.__siteMover.setup(experiment)

        self.__threadpool = ThreadPool(self.__threads)
        logging.info("Init EventStager workDir %s setup %s esPath %s token %s experiment %s userid %s sitename %s threads %s outputDir %s isDaemond %s" % (self.__workDir, self.__setup, self.__esPath, self.__token, self.__experiment, self.__userid, self.__sitename, self.__threads, self.__outputDir, self.__isDaemon))