本文整理汇总了Python中JediKnight.JediKnight.start方法的典型用法代码示例。如果您正苦于以下问题：Python JediKnight.start方法的具体用法？Python JediKnight.start怎么用？Python JediKnight.start使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类JediKnight.JediKnight的用法示例。
在下文中一共展示了JediKnight.start方法的5个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: start
# 需要导入模块: from JediKnight import JediKnight [as 别名]
# 或者: from JediKnight.JediKnight import start [as 别名]
def start(self):
# start base classes
JediKnight.start(self)
FactoryBase.initializeMods(self,self.taskBufferIF,self.ddmIF)
# go into main loop
while True:
startTime = datetime.datetime.utcnow()
try:
# get logger
tmpLog = MsgWrapper(logger)
tmpLog.info('start')
# loop over all vos
for vo in self.vos:
# loop over all sourceLabels
for prodSourceLabel in self.prodSourceLabels:
# prepare tasks to be finished
tmpLog.info('preparing tasks to be finished for vo={0} label={1}'.format(vo,prodSourceLabel))
tmpRet = self.taskBufferIF.prepareTasksToBeFinished_JEDI(vo,prodSourceLabel,
jedi_config.postprocessor.nTasks,
pid=self.pid)
if tmpRet == None:
# failed
tmpLog.error('failed to prepare tasks')
# get tasks to be finished
tmpLog.info('getting tasks to be finished')
tmpList = self.taskBufferIF.getTasksToBeFinished_JEDI(vo,prodSourceLabel,self.pid,
jedi_config.postprocessor.nTasks)
if tmpList == None:
# failed
tmpLog.error('failed to get tasks to be finished')
else:
tmpLog.info('got {0} tasks'.format(len(tmpList)))
# put to a locked list
taskList = ListWithLock(tmpList)
# make thread pool
threadPool = ThreadPool()
# make workers
nWorker = jedi_config.postprocessor.nWorkers
for iWorker in range(nWorker):
thr = PostProcessorThread(taskList,threadPool,
self.taskBufferIF,
self.ddmIF,
self)
thr.start()
# join
threadPool.join()
tmpLog.info('done')
except:
errtype,errvalue = sys.exc_info()[:2]
tmpLog.error('failed in {0}.start() with {1} {2}'.format(self.__class__.__name__,errtype.__name__,errvalue))
# sleep if needed
loopCycle = 60
timeDelta = datetime.datetime.utcnow() - startTime
sleepPeriod = loopCycle - timeDelta.seconds
if sleepPeriod > 0:
time.sleep(sleepPeriod)示例2: start
# 需要导入模块: from JediKnight import JediKnight [as 别名]
# 或者: from JediKnight.JediKnight import start [as 别名]
def start(self):
# start base classes
JediKnight.start(self)
FactoryBase.initializeMods(self, self.taskBufferIF, self.ddmIF)
# go into main loop
while True:
startTime = datetime.datetime.utcnow()
try:
# get logger
tmpLog = MsgWrapper(logger)
tmpLog.debug('start')
# loop over all vos
for vo in self.vos:
# loop over all sourceLabels
for prodSourceLabel in self.prodSourceLabels:
# get the list of tasks to refine
tmpList = self.taskBufferIF.getTasksToRefine_JEDI(vo, prodSourceLabel)
if tmpList == None:
# failed
tmpLog.error('failed to get the list of tasks to refine')
else:
tmpLog.debug('got {0} tasks'.format(len(tmpList)))
# put to a locked list
taskList = ListWithLock(tmpList)
# make thread pool
threadPool = ThreadPool()
# get work queue mapper
workQueueMapper = self.taskBufferIF.getWorkQueueMap()
# make workers
nWorker = jedi_config.taskrefine.nWorkers
for iWorker in range(nWorker):
thr = TaskRefinerThread(taskList, threadPool,
self.taskBufferIF,
self.ddmIF,
self, workQueueMapper)
thr.start()
# join
threadPool.join()
except:
errtype, errvalue = sys.exc_info()[:2]
tmpLog.error('failed in {0}.start() with {1} {2}'.format(self.__class__.__name__,
errtype.__name__, errvalue))
tmpLog.error('Traceback: {0}'.format(traceback.format_exc()))
# sleep if needed
loopCycle = jedi_config.taskrefine.loopCycle
timeDelta = datetime.datetime.utcnow() - startTime
sleepPeriod = loopCycle - timeDelta.seconds
if sleepPeriod > 0:
time.sleep(sleepPeriod)
# randomize cycle
self.randomSleep()示例3: start
# 需要导入模块: from JediKnight import JediKnight [as 别名]
# 或者: from JediKnight.JediKnight import start [as 别名]
def start(self):
# start base class
JediKnight.start(self)
# go into main loop
while True:
startTime = datetime.datetime.utcnow()
try:
# loop over all vos
for vo in self.vos:
# loop over all sourceLabels
for prodSourceLabel in self.prodSourceLabels:
# get the list of datasets to feed contents to DB
tmpList = self.taskBufferIF.getDatasetsToFeedContents_JEDI(vo,prodSourceLabel)
if tmpList == None:
# failed
logger.error('failed to get the list of datasets to feed contents')
else:
logger.debug('got %s datasets' % len(tmpList))
# put to a locked list
dsList = ListWithLock(tmpList)
# make thread pool
threadPool = ThreadPool()
# make workers
nWorker = jedi_config.confeeder.nWorkers
for iWorker in range(nWorker):
thr = ContentsFeederThread(dsList,threadPool,
self.taskBufferIF,self.ddmIF,
self.pid)
thr.start()
# join
threadPool.join()
except:
errtype,errvalue = sys.exc_info()[:2]
logger.error('failed in %s.start() with %s %s' % (self.__class__.__name__,errtype.__name__,errvalue))
# sleep if needed
loopCycle = jedi_config.confeeder.loopCycle
timeDelta = datetime.datetime.utcnow() - startTime
sleepPeriod = loopCycle - timeDelta.seconds
if sleepPeriod > 0:
time.sleep(sleepPeriod)
# randomize cycle
self.randomSleep()示例4: start
# 需要导入模块: from JediKnight import JediKnight [as 别名]
# 或者: from JediKnight.JediKnight import start [as 别名]
def start(self):
# start base classes
JediKnight.start(self)
FactoryBase.initializeMods(self,self.taskBufferIF,self.ddmIF)
# go into main loop
while True:
startTime = datetime.datetime.utcnow()
try:
# get logger
tmpLog = MsgWrapper(logger)
tmpLog.info('start')
# loop over all vos
for vo in self.vos:
# loop over all sourceLabels
for prodSourceLabel in self.prodSourceLabels:
# rescue picked files
tmpLog.info('rescue tasks with picked files for vo={0} label={1}'.format(vo,prodSourceLabel))
tmpRet = self.taskBufferIF.rescuePickedFiles_JEDI(vo,prodSourceLabel,
jedi_config.watchdog.waitForPicked)
if tmpRet == None:
# failed
tmpLog.error('failed to rescue')
else:
tmpLog.info('rescued {0} tasks'.format(tmpRet))
# reactivate pending tasks
tmpLog.info('reactivate pending tasks for vo={0} label={1}'.format(vo,prodSourceLabel))
timeoutForPending = None
if hasattr(jedi_config.watchdog,'timeoutForPendingVoLabel'):
timeoutForPending = JediCoreUtils.getConfigParam(jedi_config.watchdog.timeoutForPendingVoLabel,vo,prodSourceLabel)
if timeoutForPending == None:
timeoutForPending = jedi_config.watchdog.timeoutForPending
timeoutForPending = int(timeoutForPending)
tmpRet = self.taskBufferIF.reactivatePendingTasks_JEDI(vo,prodSourceLabel,
jedi_config.watchdog.waitForPending,
timeoutForPending)
if tmpRet == None:
# failed
tmpLog.error('failed to reactivate')
else:
tmpLog.info('reactivated {0} tasks'.format(tmpRet))
# unlock tasks
tmpLog.info('unlock tasks for vo={0} label={1}'.format(vo,prodSourceLabel))
tmpRet = self.taskBufferIF.unlockTasks_JEDI(vo,prodSourceLabel,
jedi_config.watchdog.waitForLocked)
if tmpRet == None:
# failed
tmpLog.error('failed to unlock')
else:
tmpLog.info('unlock {0} tasks'.format(tmpRet))
# restart contents update
tmpLog.info('restart contents update for vo={0} label={1}'.format(vo,prodSourceLabel))
tmpRet = self.taskBufferIF.restartTasksForContentsUpdate_JEDI(vo,prodSourceLabel)
if tmpRet == None:
# failed
tmpLog.error('failed to restart')
else:
tmpLog.info('restarted {0} tasks'.format(tmpRet))
# kick exhausted tasks
tmpLog.info('kick exhausted tasks for vo={0} label={1}'.format(vo,prodSourceLabel))
tmpRet = self.taskBufferIF.kickExhaustedTasks_JEDI(vo,prodSourceLabel,
jedi_config.watchdog.waitForExhausted)
if tmpRet == None:
# failed
tmpLog.error('failed to kick')
else:
tmpLog.info('kicked {0} tasks'.format(tmpRet))
# finish tasks when goal is reached
tmpLog.info('finish achieved tasks for vo={0} label={1}'.format(vo,prodSourceLabel))
tmpRet = self.taskBufferIF.getAchievedTasks_JEDI(vo,prodSourceLabel,
jedi_config.watchdog.waitForAchieved)
if tmpRet == None:
# failed
tmpLog.error('failed to finish')
else:
for jediTaskID in tmpRet:
self.taskBufferIF.sendCommandTaskPanda(jediTaskID,'JEDI. Goal reached',True,'finish',comQualifier='soft')
tmpLog.info('finished {0} tasks'.format(tmpRet))
# vo/prodSourceLabel specific action
impl = self.getImpl(vo,prodSourceLabel)
if impl != None:
tmpLog.info('special action for vo={0} label={1} with {2}'.format(vo,prodSourceLabel,impl.__class__.__name__))
tmpStat = impl.doAction()
if tmpStat != Interaction.SC_SUCCEEDED:
tmpLog.error('failed to run special acction for vo={0} label={1}'.format(vo,prodSourceLabel))
else:
tmpLog.info('done for vo={0} label={1}'.format(vo,prodSourceLabel))
tmpLog.info('done')
except:
errtype,errvalue = sys.exc_info()[:2]
tmpLog.error('failed in {0}.start() with {1} {2}'.format(self.__class__.__name__,errtype.__name__,errvalue))
# sleep if needed
loopCycle = jedi_config.watchdog.loopCycle
timeDelta = datetime.datetime.utcnow() - startTime
sleepPeriod = loopCycle - timeDelta.seconds
if sleepPeriod > 0:
time.sleep(sleepPeriod)
# randomize cycle
self.randomSleep()示例5: start
# 需要导入模块: from JediKnight import JediKnight [as 别名]
# 或者: from JediKnight.JediKnight import start [as 别名]
def start(self):
# start base classes
JediKnight.start(self)
FactoryBase.initializeMods(self,self.taskBufferIF,self.ddmIF)
# go into main loop
while True:
startTime = datetime.datetime.utcnow()
try:
# get logger
tmpLog = MsgWrapper(logger)
tmpLog.debug('start TaskBroker')
# get work queue mapper
workQueueMapper = self.taskBufferIF.getWorkQueueMap()
# loop over all vos
for vo in self.vos:
# loop over all sourceLabels
for prodSourceLabel in self.prodSourceLabels:
# loop over all work queues
for workQueue in workQueueMapper.getQueueListWithVoType(vo,prodSourceLabel):
msgLabel = 'vo={0} label={1} queue={2}: '.format(vo,prodSourceLabel,workQueue.queue_name)
tmpLog.debug(msgLabel+'start')
# get the list of tasks to check
tmpList = self.taskBufferIF.getTasksToCheckAssignment_JEDI(vo,prodSourceLabel,workQueue)
if tmpList == None:
# failed
tmpLog.error(msgLabel+'failed to get the list of tasks to check')
else:
tmpLog.debug(msgLabel+'got {0} tasks to check'.format(len(tmpList)))
# put to a locked list
taskList = ListWithLock(tmpList)
# make thread pool
threadPool = ThreadPool()
# make workers
nWorker = jedi_config.taskbroker.nWorkers
for iWorker in range(nWorker):
thr = TaskCheckerThread(taskList,threadPool,
self.taskBufferIF,
self.ddmIF,self,
vo,prodSourceLabel)
thr.start()
# join
threadPool.join()
# get the list of tasks to assign
tmpList = self.taskBufferIF.getTasksToAssign_JEDI(vo,prodSourceLabel,workQueue)
if tmpList == None:
# failed
tmpLog.error(msgLabel+'failed to get the list of tasks to assign')
else:
tmpLog.debug(msgLabel+'got {0} tasks to assign'.format(len(tmpList)))
# put to a locked list
taskList = ListWithLock(tmpList)
# make thread pool
threadPool = ThreadPool()
# make workers
nWorker = jedi_config.taskbroker.nWorkers
for iWorker in range(nWorker):
thr = TaskBrokerThread(taskList,threadPool,
self.taskBufferIF,
self.ddmIF,self,
vo,prodSourceLabel,
workQueue)
thr.start()
# join
threadPool.join()
tmpLog.debug(msgLabel+'done')
except:
errtype,errvalue = sys.exc_info()[:2]
tmpLog.error('failed in {0}.start() with {1} {2}'.format(self.__class__.__name__,
errtype.__name__,errvalue))
tmpLog.debug('done')
# sleep if needed
loopCycle = jedi_config.taskbroker.loopCycle
timeDelta = datetime.datetime.utcnow() - startTime
sleepPeriod = loopCycle - timeDelta.seconds
if sleepPeriod > 0:
time.sleep(sleepPeriod)
# randomize cycle
self.randomSleep()