Python ThreadPool.wait_completion方法代码示例

# 需要导入模块: from ThreadPool import ThreadPool [as 别名]
# 或者: from ThreadPool.ThreadPool import wait_completion [as 别名]
def main():
	appName = sys.argv[0]
	logging.basicConfig(
#		filename = './log/' + appName + '_' + time.strftime("%Y%m%d_%H%M%S") + '.log',
		datefmt = '%Y-%m%d %H:%M:%S',
		format = '%(asctime)s | %(levelname)s | %(name)s | %(message)s',
		level = logging.INFO
	)
	func_name = sys._getframe().f_code.co_name
	logger = logging.getLogger(func_name)
	logger.info('Started')

	parseArgs()

	logger.debug('requests.session')
	session = requests.session()
	# `mount` a custom adapter that retries failed connections for HTTP and HTTPS requests.
	session.mount("http://", requests.adapters.HTTPAdapter(max_retries=10))

	logger.debug('session.post')
	r = session.post(LOGIN_URL, data = LOGIN_PARAM)

	idResultEnd = idResult
	idResultBegin = idResult - quantResult
	# 1) Init a Thread pool with the desired number of threads
	logger.debug('ThreadPool')
	pool = ThreadPool(10)
	logger.debug('for idAtual in xrange(%d, %d, -1)' % (idResultEnd, idResultBegin))
	for idAtual in xrange(idResultEnd, idResultBegin, -1):
		# 2) Add the task to the queue
		pool.add_task(downloadResult, session, idAtual)
	# 3) Wait for completion
	pool.wait_completion()
	###
	logger.info('Finished')

# 需要导入模块: from ThreadPool import ThreadPool [as 别名]
# 或者: from ThreadPool.ThreadPool import wait_completion [as 别名]
        def __scanfile(self, file):
            scanner = FileScanner(file,  self.outDir,  self.modules,  self.profile, self.pool)
            scanner.scan()
            
if __name__ == '__main__':

    pool = ThreadPool(MAX_THREADS)
    
    if options.inputDir:
        scanner = DirScanner(options.inputDir,  options.outputDir,  MODULES,  options.profile, pool)
        scanner.scan()
    else:
        scanner = FileScanner(options.inputFile,  options.outputDir,  MODULES,  options.profile, pool)
        scanner.scan()
        
    pool.wait_completion()
    
    #Search all results if required
    if options.search:
        if options.search == 'IP':
            pattern = re.compile(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b")
        else:
            pattern = re.compile(options.search)
        ofile = open (os.path.join(options.outputDir, 'SearchHits.txt'), 'w+')
        for root,  dirs,  files in walk(options.outputDir):
            for fname in files:
                if fname <> 'SearchHits.txt':
                    ifile = open(os.path.join(root, fname), 'r')
                    line = 0
                    for text in ifile.readlines():
                        line = line + 1

# 需要导入模块: from ThreadPool import ThreadPool [as 别名]
# 或者: from ThreadPool.ThreadPool import wait_completion [as 别名]
def matchAll(phase, suffix):
    global basics
    global rules
    global matches
    global failures
    global predicates
    global locators
    global noFiles
    global noFilesAffected
    global noUnits
    global noPatternConstraints
    global noPatternConstraintsOk
    global noContentConstraints
    global noContentConstraintsOk
    global noPredicateConstraints
    global noPredicateConstraintsOk
    global noFragments
    if (phase!="basics"):
        basics = tools101.getBasics()
    rules = json.load(open(const101.rulesDump, 'r'))["results"]["rules"]
    matches = list()
    failures = list()
    predicates = set()
    locators = set()
    noFiles = 0
    noUnits = 0
    noFilesAffected = 0
    noPatternConstraints = 0
    noPatternConstraintsOk = 0
    noContentConstraints = 0
    noContentConstraintsOk = 0
    noPredicateConstraints = 0
    noPredicateConstraintsOk = 0
    noFragments = 0

    pool = ThreadPool(4)

    print "Matching 101meta metadata on 101repo (phase \"" + str(phase)+ "\")."
    for root, dirs, files in os.walk(os.path.join(const101.sRoot, "contributions"), followlinks=True):
        if not root.startswith(os.path.join(const101.sRoot, ".git")+os.sep):
            for basename in files:
                noFiles += 1
                if not basename in [".gitignore"]:
                    dirname = root[len(const101.sRoot)+1:]
                    pool.add_task(handleFile, phase, dirname, basename, suffix)
                    #handleFile(phase, dirname, basename, suffix)

    sys.stdout.write('\n')

    pool.wait_completion()

    mr = dict()
    mr["matches"] = matches
    mr["failures"] = failures
    mr["rules"] = rules
    if phase=="predicates":
        mr["predicates"] = list(predicates)
    if phase=="fragments":
        mr["locators"] = list(locators)
    print str(noFiles) + " files examined."
    print str(noFilesAffected) + " files affected."
    print str(len(failures)) + " failures encountered."
    print str(noUnits) + " metadata units attached."
    print str(noContentConstraints) + " content constraints checked."
    print str(noContentConstraintsOk) + " content constraints succeeded."
    print str(noPatternConstraints) + " filename-pattern constraints checked."
    print str(noPatternConstraintsOk) + " filename-pattern constraints succeeded."
    if phase=="predicates":
        print str(noPredicateConstraints) + " predicate constraints checked."
        print str(noPredicateConstraintsOk) + " predicate constraints succeeded."
    if phase=="fragments":
        print str(len(locators)) + " fragment locators exercised."
        print str(noFragments) + " fragment descriptions checked."
    return mr

# 需要导入模块: from ThreadPool import ThreadPool [as 别名]
# 或者: from ThreadPool.ThreadPool import wait_completion [as 别名]

#.........这里部分代码省略.........

        self.__hpcManager = hpcManager
        self.HPCMode = "HPC_" + hpcManager.getMode(defRes)
        self.__job.setMode(self.HPCMode)
        self.__job.setHpcStatus('waitingResource')
        rt = RunJobUtilities.updatePilotServer(self.__job, self.getPilotServer(), self.getPilotPort())
        self.__JR.updatePandaServer(self.__job, self.__jobSite, self.__node, 25443)

        hpcManager.getFreeResources(defRes)
        self.__job.coreCount = hpcManager.getCoreCount()
        self.__job.setHpcStatus('gettingEvents')
        rt = RunJobUtilities.updatePilotServer(self.__job, self.getPilotServer(), self.getPilotPort())
        self.__JR.updatePandaServer(self.__job, self.__jobSite, self.__node, 25443)

        numRanges = hpcManager.getEventsNumber()
        tolog("HPC Manager needs events: %s, max_events: %s; use the smallest one." % (numRanges, defRes['max_events']))
        if numRanges > int(defRes['max_events']):
            numRanges = int(defRes['max_events'])
        eventRanges = self.getEventRanges(numRanges=numRanges)
        #tolog("Event Ranges: %s " % eventRanges)
        if len(eventRanges) == 0:
            tolog("Get no Event ranges. return")
            return
        for eventRange in eventRanges:
            self.__eventRanges[eventRange['eventRangeID']] = 'new'

        # setup stage out
        self.setupStageOutHPCEvent()

        hpcManager.initJob(hpcJob)
        hpcManager.initEventRanges(eventRanges)
        
        hpcManager.submit()
        threadpool = ThreadPool(defRes['stageout_threads'])

        old_state = None
        time_start = time.time()
        while not hpcManager.isFinished():
            state = hpcManager.poll()
            self.__job.setHpcStatus(state)
            if old_state is None or old_state != state or time.time() > (time_start + 60*10):
                old_state = state
                time_start = time.time()
                tolog("HPCManager Job stat: %s" % state)
                self.__JR.updateJobStateTest(self.__job, self.__jobSite, self.__node, mode="test")
                rt = RunJobUtilities.updatePilotServer(self.__job, self.getPilotServer(), self.getPilotPort())
                self.__JR.updatePandaServer(self.__job, self.__jobSite, self.__node, 25443)

            if state and state == 'Complete':
                break
            outputs = hpcManager.getOutputs()
            for output in outputs:
                #self.stageOutHPCEvent(output)
                threadpool.add_task(self.stageOutHPCEvent, output)

            time.sleep(30)
            self.updateHPCEventRanges()

        tolog("HPCManager Job Finished")
        self.__job.setHpcStatus('stagingOut')
        rt = RunJobUtilities.updatePilotServer(self.__job, self.getPilotServer(), self.getPilotPort())
        self.__JR.updatePandaServer(self.__job, self.__jobSite, self.__node, 25443)

        outputs = hpcManager.getOutputs()
        for output in outputs:
            #self.stageOutHPCEvent(output)
            threadpool.add_task(self.stageOutHPCEvent, output)

        self.updateHPCEventRanges()
        threadpool.wait_completion()
        self.updateHPCEventRanges()


        if len(self.__failedStageOuts) > 0:
            tolog("HPC Stage out retry 1")
            half_stageout_threads = defRes['stageout_threads'] / 2
            if half_stageout_threads < 1:
                half_stageout_threads = 1
            threadpool = ThreadPool(half_stageout_threads)
            failedStageOuts = self.__failedStageOuts
            self.__failedStageOuts = []
            for failedStageOut in failedStageOuts:
                threadpool.add_task(self.stageOutHPCEvent, failedStageOut)
            threadpool.wait_completion()
            self.updateHPCEventRanges()

        if len(self.__failedStageOuts) > 0:
            tolog("HPC Stage out retry 2")
            threadpool = ThreadPool(1)
            failedStageOuts = self.__failedStageOuts
            self.__failedStageOuts = []
            for failedStageOut in failedStageOuts:
                threadpool.add_task(self.stageOutHPCEvent, failedStageOut)
            threadpool.wait_completion()
            self.updateHPCEventRanges()

        self.__job.setHpcStatus('finished')
        self.__JR.updatePandaServer(self.__job, self.__jobSite, self.__node, 25443)
        self.__hpcStatus, self.__hpcLog = hpcManager.checkHPCJobLog()
        tolog("HPC job log status: %s, job log error: %s" % (self.__hpcStatus, self.__hpcLog))