Python JobDB.getAttributesForJobList方法代码示例

# 需要导入模块: from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB [as 别名]
# 或者: from DIRAC.WorkloadManagementSystem.DB.JobDB.JobDB import getAttributesForJobList [as 别名]
class MightyOptimizer( AgentModule ):
  """
      The specific agents must provide the following methods:
      - initialize() for initial settings
      - beginExecution()
      - execute() - the main method called in the agent cycle
      - endExecution()
      - finalize() - the graceful exit of the method, this one is usually used
                 for the agent restart
  """
  __jobStates = [ 'Received', 'Checking' ]

  def initialize( self ):
    """ Standard constructor
    """
    self.jobDB = JobDB()
    self.jobLoggingDB = JobLoggingDB()
    self._optimizers = {}
    self.am_setOption( "PollingTime", 30 )
    return S_OK()

  def execute( self ):
    """ The method call by AgentModule on each iteration
    """
    jobTypeCondition = self.am_getOption( "JobTypeRestriction", [] )
    jobCond = { 'Status': self.__jobStates  }
    if jobTypeCondition:
      jobCond[ 'JobType' ] = jobTypeCondition
    result = self.jobDB.selectJobs( jobCond )
    if not result[ 'OK' ]:
      return result
    jobsList = result[ 'Value' ]
    self.log.info( "Got %s jobs for this iteration" % len( jobsList ) )
    if not jobsList:
      return S_OK()
    result = self.jobDB.getAttributesForJobList( jobsList )
    if not result[ 'OK' ]:
      return result
    jobsToProcess = result[ 'Value' ]
    for jobId in jobsToProcess:
      self.log.info( "== Processing job %s == " % jobId )
      jobAttrs = jobsToProcess[ jobId ]
      jobDef = False
      jobOptimized = False
      jobOK = True
      while not jobOptimized:
        result = self.optimizeJob( jobId, jobAttrs, jobDef )
        if not result[ 'OK' ]:
          self.log.error( "Optimizer %s error" % jobAttrs[ 'MinorStatus' ], "Job %s: %s" % ( str(jobID), result[ 'Message' ] ) )
          jobOK = False
          break
        optResult = result[ 'Value' ]
        jobOptimized = optResult[ 'done' ]
        if 'jobDef' in optResult:
          jobDef = optResult[ 'jobDef' ]
      if jobOK:
        self.log.info( "Finished optimizing job %s" % jobId )
    return S_OK()


  def optimizeJob( self, jobId, jobAttrs, jobDef ):
    """ The method call for each Job to be optimized
    """
    #Get the next optimizer
    result = self._getNextOptimizer( jobAttrs )
    if not result[ 'OK' ]:
      return result
    optimizer = result[ 'Value' ]
    if not optimizer:
      return S_OK( { 'done' : True } )
    #If there's no job def then get it
    if not jobDef:
      result = optimizer.getJobDefinition( jobId, jobDef )
      if not result['OK']:
        optimizer.setFailedJob( jobId, result[ 'Message' ] )
        return result
      jobDef = result[ 'Value' ]
    #Does the optimizer require a proxy?
    shifterEnv = False
    if optimizer.am_getModuleParam( 'shifterProxy' ):
      shifterEnv = True
      result = setupShifterProxyInEnv( optimizer.am_getModuleParam( 'shifterProxy' ),
                                       optimizer.am_getShifterProxyLocation() )
      if not result[ 'OK' ]:
        return result
    #Call the initCycle function
    result = self.am_secureCall( optimizer.beginExecution, name = "beginExecution" )
    if not result[ 'OK' ]:
      return result
    #Do the work
    result = optimizer.optimizeJob( jobId, jobDef[ 'classad' ] )
    if not result[ 'OK' ]:
      return result
    nextOptimizer = result[ 'Value' ]
    #If there was a shifter proxy, unset it
    if shifterEnv:
      del( os.environ[ 'X509_USER_PROXY' ] )
    #Check if the JDL has changed
    newJDL = jobDef[ 'classad' ].asJDL()
    if newJDL != jobDef[ 'jdl' ]:
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB [as 别名]
# 或者: from DIRAC.WorkloadManagementSystem.DB.JobDB.JobDB import getAttributesForJobList [as 别名]
class ThreadedMightyOptimizer(AgentModule):
    """
      The specific agents must provide the following methods:
      - initialize() for initial settings
      - beginExecution()
      - execute() - the main method called in the agent cycle
      - endExecution()
      - finalize() - the graceful exit of the method, this one is usually used
                 for the agent restart
  """

    __jobStates = ["Received", "Checking"]
    __defaultValidOptimizers = [
        "WorkloadManagement/JobPath",
        "WorkloadManagement/JobSanity",
        "WorkloadManagement/JobScheduling",
        "WorkloadManagement/TaskQueue",
    ]

    def initialize(self):
        """ Standard constructor
    """
        self.jobDB = JobDB()
        self.jobLoggingDB = JobLoggingDB()
        self._optimizingJobs = JobsInTheWorks()
        self._optimizers = {}
        self._threadedOptimizers = {}
        self.am_setOption("PollingTime", 30)
        return S_OK()

    def execute(self):
        """ Standard Agent module execute method
    """
        # Get jobs from DB
        result = self.jobDB.selectJobs({"Status": self.__jobStates})
        if not result["OK"]:
            gLogger.error("Cannot retrieve jobs in states %s" % self.__jobStates)
            return result
        jobsList = result["Value"]
        for i in range(len(jobsList)):
            jobsList[i] = int(jobsList[i])
        jobsList.sort()
        self.log.info("Got %s jobs for this iteration" % len(jobsList))
        if not jobsList:
            return S_OK()
        # Check jobs that are already being optimized
        newJobsList = self._optimizingJobs.addJobs(jobsList)
        if not newJobsList:
            return S_OK()
        # Get attrs of jobs to be optimized
        result = self.jobDB.getAttributesForJobList(newJobsList)
        if not result["OK"]:
            gLogger.error("Cannot retrieve attributes for %s jobs %s" % len(newJobsList))
            return result
        jobsToProcess = result["Value"]
        for jobId in jobsToProcess:
            self.log.info("== Processing job %s == " % jobId)
            jobAttrs = jobsToProcess[jobId]
            result = self.__dispatchJob(jobId, jobAttrs, False)
            if not result["OK"]:
                gLogger.error("There was a problem optimizing job", "JID %s: %s" % (jobId, result["Message"]))
        return S_OK()

    def __dispatchJob(self, jobId, jobAttrs, jobDef, keepOptimizing=True):
        """ Decide what to do with the Job
    """
        returnValue = S_OK()
        if keepOptimizing:
            result = self.__sendJobToOptimizer(jobId, jobAttrs, jobDef)
            if result["OK"] and result["Value"]:
                return S_OK()
            if not result["OK"]:
                returnValue = result
                gLogger.error(
                    "Could not send job to optimizer\n", "\tJob: %s\n\Message: %s" % (jobId, result["Message"])
                )
        self._optimizingJobs.deleteJob(jobId)
        return returnValue

    def __sendJobToOptimizer(self, jobId, jobAttrs, jobDef):
        """ Send Job to Optimizer queue
    """
        optimizerName = self.__getNextOptimizerName(jobAttrs)
        if not optimizerName:
            return S_OK(False)
        if optimizerName not in self.am_getOption("ValidOptimizers", self.__defaultValidOptimizers):
            return S_OK(False)
        if optimizerName not in self._threadedOptimizers:
            to = ThreadedOptimizer(optimizerName, self.am_getModuleParam("fullName"), self.__dispatchJob)
            result = to.initialize(self.jobDB, self.jobLoggingDB)
            if not result["OK"]:
                return S_OK(False)
            self._threadedOptimizers[optimizerName] = to
        self._threadedOptimizers[optimizerName].optimizeJob(jobId, jobAttrs, jobDef)
        return S_OK(True)

    def __getNextOptimizerName(self, jobAttrs):
        """ Determine next Optimizer
    """
        if jobAttrs["Status"] == "Received":
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB [as 别名]
# 或者: from DIRAC.WorkloadManagementSystem.DB.JobDB.JobDB import getAttributesForJobList [as 别名]
class TaskAgent( AgentModule ):
  """
      The specific agents must provide the following methods:
      - initialize() for initial settings
      - beginExecution()
      - execute() - the main method called in the agent cycle
      - endExecution()
      - finalize() - the graceful exit of the method, this one is usually used
                 for the agent restart
  """

  def initialize( self ):
    self.__taskDB = TaskDB()
    self.__jobDB = JobDB()
    return S_OK()

  def execute( self ):
    """ Main execution method
    """
    condDict = { 'Status': ['Ready', 'Processing', 'Finished'] }
    result = self.__taskDB.getTasks( [ 'TaskID', 'Status' ], condDict )
    if not result['OK']:
      return result

    tasks = result['Value']

    self.log.info( '%d tasks will be refreshed' % len(tasks) )

    for task in tasks:
      taskID = task[0]
      status = task[1]

      if status in ['Ready', 'Processing', 'Finished']:
        self.__refreshTask( taskID )

    return S_OK()


  def __refreshTask( self, taskID ):
    result = self.__refreshTaskStringAttribute( taskID, 'Site' )
    if result['OK']:
      self.log.debug( 'Task %d site is refreshed' % taskID )
    else:
      self.log.error( 'Task %d site refresh failed: %s' % ( taskID, result['Message'] ) )

    result = self.__refreshTaskStringAttribute( taskID, 'JobGroup' )
    if result['OK']:
      self.log.debug( 'Task %d job group is refreshed' % taskID )
    else:
      self.log.error( 'Task %d job group refresh failed: %s' % ( taskID, result['Message'] ) )

    result = self.__refreshTaskStatus( taskID )
    if result['OK']:
      self.log.debug( 'Task %d status is refreshed' % taskID )
    else:
      self.log.error( 'Task %d status refresh failed: %s' % ( taskID, result['Message'] ) )


################################################################################

  def __getTaskProgress( self, taskID ):
    result = self.__taskDB.getTaskJobs( taskID )
    if not result['OK']:
      return result
    jobIDs = result['Value']

    result = self.__jobDB.getAttributesForJobList( jobIDs, ['Status'] )
    if not result['OK']:
      return result
    statuses = result['Value']

    progress = { 'Total': 0, 'Done': 0, 'Failed': 0, 'Running': 0, 'Waiting': 0, 'Deleted': 0 }
    progress['Total'] = len(jobIDs)
    for jobID in jobIDs:
      if jobID in statuses:
        status = statuses[jobID]['Status']
        if status in ['Done']:
          progress['Done'] += 1
        elif status in ['Failed', 'Stalled', 'Killed']:
          progress['Failed'] += 1
        elif status in ['Running', 'Completed']:
          progress['Running'] += 1
        else:
          progress['Waiting'] += 1
      else:
        progress['Deleted'] += 1

    return S_OK( progress )

  def __analyseTaskStatus( self, progress ):
    totalJob = progress.get( 'Total', 0 )
    runningJob = progress.get( 'Running', 0 )
    waitingJob = progress.get( 'Waiting', 0 )
    deletedJob = progress.get( 'Deleted', 0 )

    status = 'Unknown'
    if deletedJob == totalJob:
      status = 'Expired'
    elif runningJob == 0 and waitingJob == 0:
      status = 'Finished'
#.........这里部分代码省略.........