Python JobDB.getJobJDL方法代码示例

# 需要导入模块: from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB [as 别名]
# 或者: from DIRAC.WorkloadManagementSystem.DB.JobDB.JobDB import getJobJDL [as 别名]
class Matcher( object ):
  """ Logic for matching
  """

  def __init__( self, pilotAgentsDB = None, jobDB = None, tqDB = None, jlDB = None, opsHelper = None ):
    """ c'tor
    """
    if pilotAgentsDB:
      self.pilotAgentsDB = pilotAgentsDB
    else:
      self.pilotAgentsDB = PilotAgentsDB()
    if jobDB:
      self.jobDB = jobDB
    else:
      self.jobDB = JobDB()
    if tqDB:
      self.tqDB = tqDB
    else:
      self.tqDB = TaskQueueDB()
    if jlDB:
      self.jlDB = jlDB
    else:
      self.jlDB = JobLoggingDB()

    if opsHelper:
      self.opsHelper = opsHelper
    else:
      self.opsHelper = Operations()

    self.log = gLogger.getSubLogger( "Matcher" )

    self.limiter = Limiter( jobDB = self.jobDB, opsHelper = self.opsHelper )


  def selectJob( self, resourceDescription, credDict ):
    """ Main job selection function to find the highest priority job matching the resource capacity
    """

    startTime = time.time()

    resourceDict = self._getResourceDict( resourceDescription, credDict )

    negativeCond = self.limiter.getNegativeCondForSite( resourceDict['Site'] )
    result = self.tqDB.matchAndGetJob( resourceDict, negativeCond = negativeCond )

    if not result['OK']:
      return result
    result = result['Value']
    if not result['matchFound']:
      self.log.info( "No match found" )
      raise RuntimeError( "No match found" )

    jobID = result['jobId']
    resAtt = self.jobDB.getJobAttributes( jobID, ['OwnerDN', 'OwnerGroup', 'Status'] )
    if not resAtt['OK']:
      raise RuntimeError( 'Could not retrieve job attributes' )
    if not resAtt['Value']:
      raise RuntimeError( "No attributes returned for job" )
    if not resAtt['Value']['Status'] == 'Waiting':
      self.log.error( 'Job matched by the TQ is not in Waiting state', str( jobID ) )
      result = self.tqDB.deleteJob( jobID )
      if not result[ 'OK' ]:
        return result
      raise RuntimeError( "Job %s is not in Waiting state" % str( jobID ) )

    self._reportStatus( resourceDict, jobID )

    result = self.jobDB.getJobJDL( jobID )
    if not result['OK']:
      raise RuntimeError( "Failed to get the job JDL" )

    resultDict = {}
    resultDict['JDL'] = result['Value']
    resultDict['JobID'] = jobID

    matchTime = time.time() - startTime
    self.log.info( "Match time: [%s]" % str( matchTime ) )
    gMonitor.addMark( "matchTime", matchTime )

    # Get some extra stuff into the response returned
    resOpt = self.jobDB.getJobOptParameters( jobID )
    if resOpt['OK']:
      for key, value in resOpt['Value'].items():
        resultDict[key] = value
    resAtt = self.jobDB.getJobAttributes( jobID, ['OwnerDN', 'OwnerGroup'] )
    if not resAtt['OK']:
      raise RuntimeError( 'Could not retrieve job attributes' )
    if not resAtt['Value']:
      raise RuntimeError( 'No attributes returned for job' )

    if self.opsHelper.getValue( "JobScheduling/CheckMatchingDelay", True ):
      self.limiter.updateDelayCounters( resourceDict['Site'], jobID )

    pilotInfoReportedFlag = resourceDict.get( 'PilotInfoReportedFlag', False )
    if not pilotInfoReportedFlag:
      self._updatePilotInfo( resourceDict )
    self._updatePilotJobMapping( resourceDict, jobID )

    resultDict['DN'] = resAtt['Value']['OwnerDN']
    resultDict['Group'] = resAtt['Value']['OwnerGroup']
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB [as 别名]
# 或者: from DIRAC.WorkloadManagementSystem.DB.JobDB.JobDB import getJobJDL [as 别名]

#.........这里部分代码省略.........
    if not result['Value']:
      self.log.verbose('No pending jobs to process')
      return S_OK('No work to do')

    for job in result['Value']:
      result = self.getJobDefinition(job)
      if not result['OK']:
        self.setFailedJob(job, result['Message'], '')
        continue
      jobDef = result['Value']
      result = self.optimizeJob(job, jobDef['classad'])

    return S_OK()

  #############################################################################
  def optimizeJob(self, job, classAdJob):
    """ Call the corresponding Optimizer checkJob method
    """
    self.log.info('Job %s will be processed by %sAgent' % (job, self.am_getModuleParam('optimizerName')))
    result = self.checkJob(job, classAdJob)
    if not result['OK']:
      self.setFailedJob(job, result['Message'], classAdJob)
    return result

  #############################################################################
  def getJobDefinition(self, job, jobDef=False):
    """ Retrieve JDL of the Job and return jobDef dictionary
    """
    if not jobDef:
      jobDef = {}
    # If not jdl in jobinfo load it
    if 'jdl' not in jobDef:
      if self.requiredJobInfo == 'jdlOriginal':
        result = self.jobDB.getJobJDL(job, original=True)
        if not result['OK']:
          self.log.error("No JDL for job", "%s" % job)
          return S_ERROR("No JDL for job")
        jobDef['jdl'] = result['Value']
      if self.requiredJobInfo == 'jdl':
        result = self.jobDB.getJobJDL(job)
        if not result['OK']:
          self.log.error("No JDL for job", "%s" % job)
          return S_ERROR("No JDL for job")
        jobDef['jdl'] = result['Value']
    # Load the classad if needed
    if 'jdl' in jobDef and 'classad' not in jobDef:
      try:
        classad = ClassAd(jobDef['jdl'])
      except BaseException:
        self.log.debug("Cannot load JDL")
        return S_ERROR('Illegal Job JDL')
      if not classad.isOK():
        self.log.debug("Warning: illegal JDL for job %s, will be marked problematic" % (job))
        return S_ERROR('Illegal Job JDL')
      jobDef['classad'] = classad
    return S_OK(jobDef)

  #############################################################################
  def getOptimizerJobInfo(self, job, reportName):
    """This method gets job optimizer information that will
       be used for
    """
    self.log.verbose("self.jobDB.getJobOptParameter(%s,'%s')" % (job, reportName))
    result = self.jobDB.getJobOptParameter(job, reportName)
    if result['OK']:
      value = result['Value']

# 需要导入模块: from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB [as 别名]
# 或者: from DIRAC.WorkloadManagementSystem.DB.JobDB.JobDB import getJobJDL [as 别名]
class OptimizerModule( AgentModule ):
  """
      The specific agents must provide the following methods:
      - initialize() for initial settings
      - beginExecution()
      - execute() - the main method called in the agent cycle
      - endExecution()
      - finalize() - the graceful exit of the method, this one is usually used
                 for the agent restart
  """

  #############################################################################
  def initialize( self, jobDB = False, logDB = False ):
    """ Initialization of the Optimizer Agent.
    """
    if not jobDB:
      self.jobDB = JobDB()
    else:
      self.jobDB = jobDB
    if not logDB:
      self.logDB = JobLoggingDB()
    else:
      self.logDB = logDB

    trailing = "Agent"
    optimizerName = self.am_getModuleParam( 'agentName' )
    if optimizerName[ -len( trailing ):].find( trailing ) == 0:
      optimizerName = optimizerName[ :-len( trailing ) ]
    self.am_setModuleParam( 'optimizerName', optimizerName )

    self.startingMinorStatus = self.am_getModuleParam( 'optimizerName' )
    self.startingMajorStatus = "Checking"
    self.failedStatus = self.am_getOption( "FailedJobStatus" , 'Failed' )
    self.requiredJobInfo = 'jdl'
    self.am_setOption( "PollingTime", 30 )

    return self.initializeOptimizer()

  def initializeOptimizer( self ):
    """ To be overwritten by inheriting class 
    """
    return S_OK()

  #############################################################################
  def execute( self ):
    """ The main agent execution method
    """

    result = self.initializeOptimizer()
    if not result[ 'OK' ]:
      return result
    self._initResult = result[ 'Value' ]

    condition = { 'Status' : self.startingMajorStatus }
    if self.startingMinorStatus:
      condition[ 'MinorStatus' ] = self.startingMinorStatus

    result = self.jobDB.selectJobs( condition )
    if not result['OK']:
      self.log.warn( 'Failed to get a job list from the JobDB' )
      return S_ERROR( 'Failed to get a job list from the JobDB' )

    if not len( result['Value'] ):
      self.log.verbose( 'No pending jobs to process' )
      return S_OK( 'No work to do' )

    for job in result['Value']:
      result = self.getJobDefinition( job )
      if not result['OK']:
        self.setFailedJob( job, result[ 'Message' ], '' )
        continue
      jobDef = result[ 'Value' ]
      result = self.optimizeJob( job, jobDef[ 'classad' ] )

    return S_OK()

  #############################################################################
  def optimizeJob( self, job, classAdJob ):
    """ Call the corresponding Optimizer checkJob method
    """
    self.log.info( 'Job %s will be processed by %sAgent' % ( job, self.am_getModuleParam( 'optimizerName' ) ) )
    result = self.checkJob( job, classAdJob )
    if not result['OK']:
      self.setFailedJob( job, result['Message'], classAdJob )
    return result

  #############################################################################
  def getJobDefinition( self, job, jobDef = False ):
    """ Retrieve JDL of the Job and return jobDef dictionary
    """
    if jobDef == False:
      jobDef = {}
    #If not jdl in jobinfo load it
    if 'jdl' not in jobDef:
      if 'jdlOriginal' == self.requiredJobInfo:
        result = self.jobDB.getJobJDL( job, original = True )
        if not result[ 'OK' ]:
          self.log.error( "No JDL for job", "%s" % job )
          return S_ERROR( "No JDL for job" )
        jobDef[ 'jdl' ] = result[ 'Value' ]
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB [as 别名]
# 或者: from DIRAC.WorkloadManagementSystem.DB.JobDB.JobDB import getJobJDL [as 别名]

#.........这里部分代码省略.........
  #############################################################################
  def __updateJobStatus( self, job, status, minorstatus = None ):
    """ This method updates the job status in the JobDB, this should only be
used to fail jobs due to the optimizer chain.
"""
    self.log.verbose( "self.jobDB.setJobAttribute(%s,'Status','%s',update=True)" % ( job, status ) )

    if self.am_getOption( 'Enable', True ):
      result = self.jobDB.setJobAttribute( job, 'Status', status, update = True )
    else:
      result = S_OK( 'DisabledMode' )

    if result['OK']:
      if minorstatus:
        self.log.verbose( "self.jobDB.setJobAttribute(%s,'MinorStatus','%s',update=True)" % ( job, minorstatus ) )
        result = self.jobDB.setJobAttribute( job, 'MinorStatus', minorstatus, update = True )

    if not minorstatus: #Retain last minor status for stalled jobs
      result = self.jobDB.getJobAttributes( job, ['MinorStatus'] )
      if result['OK']:
        minorstatus = result['Value']['MinorStatus']

    logStatus = status
    result = self.logDB.addLoggingRecord( job, status = logStatus, minor = minorstatus, source = 'StalledJobAgent' )
    if not result['OK']:
      self.log.warn( result )

    return result

  def __getProcessingType( self, jobID ):
    """ Get the Processing Type from the JDL, until it is promoted to a real Attribute
"""
    processingType = 'unknown'
    result = self.jobDB.getJobJDL( jobID, original = True )
    if not result['OK']:
      return processingType
    classAdJob = ClassAd( result['Value'] )
    if classAdJob.lookupAttribute( 'ProcessingType' ):
      processingType = classAdJob.getAttributeString( 'ProcessingType' )
    return processingType


  #############################################################################
  def __sendAccounting( self, jobID ):
    """ Send WMS accounting data for the given job
"""
    try:
      accountingReport = Job()
      endTime = 'Unknown'
      lastHeartBeatTime = 'Unknown'

      result = self.jobDB.getJobAttributes( jobID )
      if not result['OK']:
        return result
      jobDict = result['Value']

      startTime, endTime = self.__checkLoggingInfo( jobID, jobDict )
      lastCPUTime, lastWallTime, lastHeartBeatTime = self.__checkHeartBeat( jobID, jobDict )
      lastHeartBeatTime = fromString( lastHeartBeatTime )
      if lastHeartBeatTime is not None and lastHeartBeatTime > endTime:
        endTime = lastHeartBeatTime

      cpuNormalization = self.jobDB.getJobParameter( jobID, 'CPUNormalizationFactor' )
      if not cpuNormalization['OK'] or not cpuNormalization['Value']:
        cpuNormalization = 0.0
      else:

# 需要导入模块: from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB [as 别名]
# 或者: from DIRAC.WorkloadManagementSystem.DB.JobDB.JobDB import getJobJDL [as 别名]

#.........这里部分代码省略.........
    #############################################################################
    def __updateJobStatus(self, job, status, minorstatus=None):
        """ This method updates the job status in the JobDB, this should only be
used to fail jobs due to the optimizer chain.
"""
        self.log.verbose("self.jobDB.setJobAttribute(%s,'Status','%s',update=True)" % (job, status))

        if self.am_getOption("Enable", True):
            result = self.jobDB.setJobAttribute(job, "Status", status, update=True)
        else:
            result = S_OK("DisabledMode")

        if result["OK"]:
            if minorstatus:
                self.log.verbose("self.jobDB.setJobAttribute(%s,'MinorStatus','%s',update=True)" % (job, minorstatus))
                result = self.jobDB.setJobAttribute(job, "MinorStatus", minorstatus, update=True)

        if not minorstatus:  # Retain last minor status for stalled jobs
            result = self.jobDB.getJobAttributes(job, ["MinorStatus"])
            if result["OK"]:
                minorstatus = result["Value"]["MinorStatus"]

        logStatus = status
        result = self.logDB.addLoggingRecord(job, status=logStatus, minor=minorstatus, source="StalledJobAgent")
        if not result["OK"]:
            self.log.warn(result)

        return result

    def __getProcessingType(self, jobID):
        """ Get the Processing Type from the JDL, until it is promoted to a real Attribute
"""
        processingType = "unknown"
        result = self.jobDB.getJobJDL(jobID, original=True)
        if not result["OK"]:
            return processingType
        classAdJob = ClassAd(result["Value"])
        if classAdJob.lookupAttribute("ProcessingType"):
            processingType = classAdJob.getAttributeString("ProcessingType")
        return processingType

    #############################################################################
    def __sendAccounting(self, jobID):
        """ Send WMS accounting data for the given job
"""
        try:
            accountingReport = Job()
            endTime = "Unknown"
            lastHeartBeatTime = "Unknown"

            result = self.jobDB.getJobAttributes(jobID)
            if not result["OK"]:
                return result
            jobDict = result["Value"]

            startTime, endTime = self.__checkLoggingInfo(jobID, jobDict)
            lastCPUTime, lastWallTime, lastHeartBeatTime = self.__checkHeartBeat(jobID, jobDict)
            lastHeartBeatTime = fromString(lastHeartBeatTime)
            if lastHeartBeatTime is not None and lastHeartBeatTime > endTime:
                endTime = lastHeartBeatTime

            cpuNormalization = self.jobDB.getJobParameter(jobID, "CPUNormalizationFactor")
            if not cpuNormalization["OK"] or not cpuNormalization["Value"]:
                cpuNormalization = 0.0
            else:
                cpuNormalization = float(cpuNormalization["Value"])