Python ThreadPool.add_task方法代码示例

# 需要导入模块: from ThreadPool import ThreadPool [as 别名]
# 或者: from ThreadPool.ThreadPool import add_task [as 别名]
def main():
	appName = sys.argv[0]
	logging.basicConfig(
#		filename = './log/' + appName + '_' + time.strftime("%Y%m%d_%H%M%S") + '.log',
		datefmt = '%Y-%m%d %H:%M:%S',
		format = '%(asctime)s | %(levelname)s | %(name)s | %(message)s',
		level = logging.INFO
	)
	func_name = sys._getframe().f_code.co_name
	logger = logging.getLogger(func_name)
	logger.info('Started')

	parseArgs()

	logger.debug('requests.session')
	session = requests.session()
	# `mount` a custom adapter that retries failed connections for HTTP and HTTPS requests.
	session.mount("http://", requests.adapters.HTTPAdapter(max_retries=10))

	logger.debug('session.post')
	r = session.post(LOGIN_URL, data = LOGIN_PARAM)

	idResultEnd = idResult
	idResultBegin = idResult - quantResult
	# 1) Init a Thread pool with the desired number of threads
	logger.debug('ThreadPool')
	pool = ThreadPool(10)
	logger.debug('for idAtual in xrange(%d, %d, -1)' % (idResultEnd, idResultBegin))
	for idAtual in xrange(idResultEnd, idResultBegin, -1):
		# 2) Add the task to the queue
		pool.add_task(downloadResult, session, idAtual)
	# 3) Wait for completion
	pool.wait_completion()
	###
	logger.info('Finished')

# 需要导入模块: from ThreadPool import ThreadPool [as 别名]
# 或者: from ThreadPool.ThreadPool import add_task [as 别名]
def test_thread_pool():
    pool = ThreadPool(2, 0.5)
    for i in range(2):
        handle = login.login('ssh', 'root', 'n', '10.20.60.23')
        task = (handle.execute_cmd, ('uname -a',), {})
        pool.add_task(task)
    print pool.get_result()
    pool.task_join()

# 需要导入模块: from ThreadPool import ThreadPool [as 别名]
# 或者: from ThreadPool.ThreadPool import add_task [as 别名]

#.........这里部分代码省略.........
                        filepath = os.path.join(self.__workDir, file)
                        os.rename(filepath, filepath.replace("cmdcopying", "cmd"))
            else:
                if file.endswith("cmdcopying"):
                    present = time.time()
                    origin_file = file.replace("cmdcopying", "cmd")
                    if self.__process == 0:
                        filepath = os.path.join(self.__workDir, file)
                        if (present - os.path.getmtime(filepath)) > older:
                            os.rename(filepath, filepath.replace("cmdcopying", "cmd"))


    def getEventRanges(self):
        if len(self.__eventRanges.keys()) > 5:
            return
        outputFiles = self.getUnstagedOutputFiles()
        for file in outputFiles:
            if len(self.__eventRanges.keys()) > 5:
                return
            self.__startWait = None
            self.__eventRanges[file] = {}
            self.__eventRanges_staged[file] = []
            self.__eventRanges_faileStaged[file] = []

            filepath = os.path.join(self.__workDir, file)
            handle = open(filepath)
            for line in handle:
                if len(line.strip()) == 0:
                    continue
                line = line.replace("  ", " ")
                jobId, eventRange, status, output = line.split(" ")
                output = output.split(",")[0]
                self.__eventRanges[file][eventRange] = {'retry':0, 'event': (jobId, eventRange, status, output)}
                self.__threadpool.add_task(self.stageOutEvent, (file, jobId, eventRange, status, output))
                if jobId not in self.__processedJobs:
                    self.__processedJobs.append(jobId)
            handle.close()
            os.rename(filepath, filepath + ".staging")

    def checkMissedStagingFiles(self):
        all_files = os.listdir(self.__workDir)
        for file in all_files:
            try:
                if file.endswith(".dump.staged.reported"):
                    origin_file = file.replace(".dump.staged.reported", ".dump")
                    filepath = os.path.join(self.__workDir, file)
                    size = os.path.getsize(filepath)
                    if size == 0:
                        if (int(hashlib.sha1(origin_file).hexdigest(),16) % self.__totalProcess) == self.__process:
                            back_file = filepath.replace(".dump.staged.reported", ".dump.BAK")
                            origin_file = filepath.replace(".dump.staged.reported", ".dump")
                            staging_file = filepath.replace(".dump.staged.reported", ".dump.staging")
                            if not os.path.exists(back_file) and not os.path.exists(origin_file) and not os.path.exists(staging_file):
                                os.remove(filepath)
                            os.rename(back_file, origin_file)
            except:
                logging.warning("Failed to rename %s to %s: %s" % (back_file, origin_file, traceback.format_exc()))

    def checkFailedStagingFiles(self):
        for file in self.__eventRanges_faileStaged:
            while self.__eventRanges_faileStaged[file]:
                jobId, eventRangeID, status, output = self.__eventRanges_faileStaged[file].pop()
                if eventRangeID not in self.__eventRanges[file]:
                    logging.warning("checkFailedStagingFiles: %s is not in eventRanges" % eventRangeID)
                else:
                    if  self.__eventRanges[file][eventRangeID]['retry'] < 3:

# 需要导入模块: from ThreadPool import ThreadPool [as 别名]
# 或者: from ThreadPool.ThreadPool import add_task [as 别名]
def matchAll(phase, suffix):
    global basics
    global rules
    global matches
    global failures
    global predicates
    global locators
    global noFiles
    global noFilesAffected
    global noUnits
    global noPatternConstraints
    global noPatternConstraintsOk
    global noContentConstraints
    global noContentConstraintsOk
    global noPredicateConstraints
    global noPredicateConstraintsOk
    global noFragments
    if (phase!="basics"):
        basics = tools101.getBasics()
    rules = json.load(open(const101.rulesDump, 'r'))["results"]["rules"]
    matches = list()
    failures = list()
    predicates = set()
    locators = set()
    noFiles = 0
    noUnits = 0
    noFilesAffected = 0
    noPatternConstraints = 0
    noPatternConstraintsOk = 0
    noContentConstraints = 0
    noContentConstraintsOk = 0
    noPredicateConstraints = 0
    noPredicateConstraintsOk = 0
    noFragments = 0

    pool = ThreadPool(4)

    print "Matching 101meta metadata on 101repo (phase \"" + str(phase)+ "\")."
    for root, dirs, files in os.walk(os.path.join(const101.sRoot, "contributions"), followlinks=True):
        if not root.startswith(os.path.join(const101.sRoot, ".git")+os.sep):
            for basename in files:
                noFiles += 1
                if not basename in [".gitignore"]:
                    dirname = root[len(const101.sRoot)+1:]
                    pool.add_task(handleFile, phase, dirname, basename, suffix)
                    #handleFile(phase, dirname, basename, suffix)

    sys.stdout.write('\n')

    pool.wait_completion()

    mr = dict()
    mr["matches"] = matches
    mr["failures"] = failures
    mr["rules"] = rules
    if phase=="predicates":
        mr["predicates"] = list(predicates)
    if phase=="fragments":
        mr["locators"] = list(locators)
    print str(noFiles) + " files examined."
    print str(noFilesAffected) + " files affected."
    print str(len(failures)) + " failures encountered."
    print str(noUnits) + " metadata units attached."
    print str(noContentConstraints) + " content constraints checked."
    print str(noContentConstraintsOk) + " content constraints succeeded."
    print str(noPatternConstraints) + " filename-pattern constraints checked."
    print str(noPatternConstraintsOk) + " filename-pattern constraints succeeded."
    if phase=="predicates":
        print str(noPredicateConstraints) + " predicate constraints checked."
        print str(noPredicateConstraintsOk) + " predicate constraints succeeded."
    if phase=="fragments":
        print str(len(locators)) + " fragment locators exercised."
        print str(noFragments) + " fragment descriptions checked."
    return mr

# 需要导入模块: from ThreadPool import ThreadPool [as 别名]
# 或者: from ThreadPool.ThreadPool import add_task [as 别名]
    def runHPCEvent(self):
        tolog("runHPCEvent")
        self.__job.jobState = "running"
        self.__job.setState([self.__job.jobState, 0, 0])
        self.__job.pilotErrorDiag = None
        rt = RunJobUtilities.updatePilotServer(self.__job, self.getPilotServer(), self.getPilotPort())
        self.__JR.updateJobStateTest(self.__job, self.__jobSite, self.__node, mode="test")

        defRes = self.getDefaultResources()
        if defRes['copy_input_files'] == 'true':
            self.__copyInputFiles = True
        else:
            self.__copyInputFiles = False

        status, output, hpcJob = self.prepareHPCJob()
        if status == 0:
            tolog("HPC Job: %s " % hpcJob)
        else:
            tolog("failed to create the Tag file")
            self.failJob(0, PilotErrors.ERR_UNKNOWN, self.__job, pilotErrorDiag=output)
            return 


        self.__hpcStatus = None
        self.__hpcLog = None

        logFileName = None
        tolog("runJobHPCEvent.getPilotLogFilename=%s"% self.getPilotLogFilename())
        if self.getPilotLogFilename() != "":
            logFileName = self.getPilotLogFilename()
        hpcManager = HPCManager(globalWorkingDir=self.__job.workdir, logFileName=logFileName, poolFileCatalog=self.__poolFileCatalogTemp, inputFiles=self.__inputFilesGlobal, copyInputFiles=self.__copyInputFiles)

        self.__hpcManager = hpcManager
        self.HPCMode = "HPC_" + hpcManager.getMode(defRes)
        self.__job.setMode(self.HPCMode)
        self.__job.setHpcStatus('waitingResource')
        rt = RunJobUtilities.updatePilotServer(self.__job, self.getPilotServer(), self.getPilotPort())
        self.__JR.updatePandaServer(self.__job, self.__jobSite, self.__node, 25443)

        hpcManager.getFreeResources(defRes)
        self.__job.coreCount = hpcManager.getCoreCount()
        self.__job.setHpcStatus('gettingEvents')
        rt = RunJobUtilities.updatePilotServer(self.__job, self.getPilotServer(), self.getPilotPort())
        self.__JR.updatePandaServer(self.__job, self.__jobSite, self.__node, 25443)

        numRanges = hpcManager.getEventsNumber()
        tolog("HPC Manager needs events: %s, max_events: %s; use the smallest one." % (numRanges, defRes['max_events']))
        if numRanges > int(defRes['max_events']):
            numRanges = int(defRes['max_events'])
        eventRanges = self.getEventRanges(numRanges=numRanges)
        #tolog("Event Ranges: %s " % eventRanges)
        if len(eventRanges) == 0:
            tolog("Get no Event ranges. return")
            return
        for eventRange in eventRanges:
            self.__eventRanges[eventRange['eventRangeID']] = 'new'

        # setup stage out
        self.setupStageOutHPCEvent()

        hpcManager.initJob(hpcJob)
        hpcManager.initEventRanges(eventRanges)
        
        hpcManager.submit()
        threadpool = ThreadPool(defRes['stageout_threads'])

        old_state = None
        time_start = time.time()
        while not hpcManager.isFinished():
            state = hpcManager.poll()
            self.__job.setHpcStatus(state)
            if old_state is None or old_state != state or time.time() > (time_start + 60*10):
                old_state = state
                time_start = time.time()
                tolog("HPCManager Job stat: %s" % state)
                self.__JR.updateJobStateTest(self.__job, self.__jobSite, self.__node, mode="test")
                rt = RunJobUtilities.updatePilotServer(self.__job, self.getPilotServer(), self.getPilotPort())
                self.__JR.updatePandaServer(self.__job, self.__jobSite, self.__node, 25443)

            if state and state == 'Complete':
                break
            outputs = hpcManager.getOutputs()
            for output in outputs:
                #self.stageOutHPCEvent(output)
                threadpool.add_task(self.stageOutHPCEvent, output)

            time.sleep(30)
            self.updateHPCEventRanges()

        tolog("HPCManager Job Finished")
        self.__job.setHpcStatus('stagingOut')
        rt = RunJobUtilities.updatePilotServer(self.__job, self.getPilotServer(), self.getPilotPort())
        self.__JR.updatePandaServer(self.__job, self.__jobSite, self.__node, 25443)

        outputs = hpcManager.getOutputs()
        for output in outputs:
            #self.stageOutHPCEvent(output)
            threadpool.add_task(self.stageOutHPCEvent, output)

        self.updateHPCEventRanges()
#.........这里部分代码省略.........