Python JobDB.getCounters方法代码示例

# 需要导入模块: from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB [as 别名]
# 或者: from DIRAC.WorkloadManagementSystem.DB.JobDB.JobDB import getCounters [as 别名]
class JobHistoryAgent( AgentModule ):
  """
      The specific agents must provide the following methods:
      - initialize() for initial settings
      - beginExecution()
      - execute() - the main method called in the agent cycle
      - endExecution()
      - finalize() - the graceful exit of the method, this one is usually used
                 for the agent restart
  """

  def initialize( self ):

    self.jobDB = JobDB()

    for status in MONITOR_STATUS:
      for site in MONITOR_SITES:
        gLogger.verbose( "Registering activity %s-%s" % ( status, site ) )
        gLogger.verbose( "Jobs in %s state at %s" % ( status, site ) )
        gMonitor.registerActivity( "%s-%s" % ( status, site ), "Jobs in %s state at %s" % ( status, site ),
                                  "JobHistoryAgent", "Jobs/minute", gMonitor.OP_MEAN )

    self.last_update = 0
    self.resultDB = None
    self.reportPeriod = 60
    return S_OK()

  def execute( self ):
    """ Main execution method
    """
    delta = time.time() - self.last_update
    if delta > self.reportPeriod:
      result = self.jobDB.getCounters( 'Jobs', ['Status', 'Site'], {}, '' )
      if not result['OK']:
        return S_ERROR( 'Failed to get data from the Job Database' )
      self.resultDB = result['Value']
      self.last_update = time.time()

    totalDict = {}
    for status in MONITOR_STATUS:
      totalDict[status] = 0

    for row in self.resultDB:
      site = row[0]['Site']
      status = row[0]['Status']
      count = row[1]
      if site in MONITOR_SITES and status in MONITOR_STATUS:
        gLogger.verbose( "Adding mark %s-%s: " % ( status, site ) + str( count ) )
        gMonitor.addMark( "%s-%s" % ( status, site ), count )
      if status in totalDict:
        totalDict[status] += count

    for status in MONITOR_STATUS:
      gLogger.verbose( "Adding mark %s-All sites: " % status + str( totalDict[status] ) )
      gMonitor.addMark( "%s-All sites" % status, totalDict[status] )

    return S_OK()

# 需要导入模块: from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB [as 别名]
# 或者: from DIRAC.WorkloadManagementSystem.DB.JobDB.JobDB import getCounters [as 别名]
class SPTCorrector(BaseCorrector):

    _GLOBAL_MAX_CORRECTION = "MaxGlobalCorrection"
    _SLICE_TIME_SPAN = "TimeSpan"
    _SLICE_WEIGHT = "Weight"
    _SLICE_MAX_CORRECTION = "MaxCorrection"

    def initialize(self):

        self.__jobDB = JobDB()

        return S_OK()

    def applyCorrection(self, entitiesExpectedShare):

        print "AT >>> entitiesExpectedShare", entitiesExpectedShare

        ownerDNs = entitiesExpectedShare.keys()

        group = self.getGroup()
        result = self.__jobDB.getCounters("Jobs", ["OwnerDN"], {"OwnerGroup": group, "Status": "Waiting"})
        if not result["OK"]:
            print "AT >>> result", result
            return entitiesExpectedShare

        ownerDict = {}
        for row in result["Value"]:
            ownerDict[row[0]["OwnerDN"]] = row[1]
        print "AT >>> ownerDict", ownerDict

        resultShare = {}
        minNumber = 1000000000000
        minOwnerDN = ""
        for ownerDN in ownerDNs:
            resultShare[ownerDN] = 0
            if minNumber > ownerDict[ownerDN]:
                minNumber = ownerDict[ownerDN]
                minOwnerDN = ownerDN
        resultShare[minOwnerDN] = 1

        print "AT >>> resultShare", resultShare

        return resultShare

    def updateHistoryKnowledge(self):

        return S_OK()

# 需要导入模块: from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB [as 别名]
# 或者: from DIRAC.WorkloadManagementSystem.DB.JobDB.JobDB import getCounters [as 别名]
class JobCommand( Command ):
  '''
  Job "master" Command.
  '''


  def __init__( self, args = None, clients = None ):
    
    super( JobCommand, self ).__init__( args, clients )

    if 'JobDB' in self.apis:
      self.jobDB = self.apis[ 'JobDB' ]
    else:
      self.jobDB = JobDB()

    if 'ResourceManagementClient' in self.apis:
      self.rmClient = self.apis[ 'ResourceManagementClient' ]
    else:
      self.rmClient = ResourceManagementClient()


  def _storeCommand( self, result ):
    '''
    Stores the results of doNew method on the database.
    '''
    
    for jobDict in result:
      
      lowerCaseJobDict = {}
      for key, value in jobDict.iteritems():
        lowerCaseJobDict[ key[0].lower() + key[1:] ] = value
      
      resQuery = self.rmClient.addOrModifyJobCache( **lowerCaseJobDict )
      
      if not resQuery[ 'OK' ]:
        return resQuery
    
    return S_OK()

  
  def _prepareCommand( self ):
    '''
    JobCommand requires one arguments:
    - name : <str>
    '''

    if not 'name' in self.args:
      return S_ERROR( '"name" not found in self.args' )
    name = self.args[ 'name' ]
     
    if not 'timespan' in self.args:
      return S_ERROR( '"timespan" not found in self.args' )
    timespan = self.args[ 'timespan' ]
  
    return S_OK( ( name, timespan ) )
  
  
  def doNew( self, masterParams = None ):
    '''
    Gets the parameters to run, either from the master method or from its
    own arguments.
    It contacts the WMSAdministrator with a list of site names, or a single
    site.
    If there are jobs, are recorded and then returned.
    '''
    
    if masterParams is True:
      self.args[ 'name' ] = ''

    params = self._prepareCommand()
    if not params[ 'OK' ]:
      return params

    name, timespan = params[ 'Value' ]
    
    condDict = {}
    if name:
      condDict = { 'Site' : name }

    startTimeWindow = datetime.utcnow() - timedelta( seconds = timespan )
    
    results = self.jobDB.getCounters( 'Jobs', ['Site', 'Status'],
                                      condDict, newer = startTimeWindow,
                                      timeStamp = 'LastUpdateTime' )
    
    if not results[ 'OK' ]:
      return results
    # Results look like this
    # [ ({'Status': 'Checking', 'Site': 'ANY'}, 6L), ...
    
    uniformResult = {}
    
    jobStatuses = ( 'Checking', 'Completed', 'Done', 'Failed', 'Killed', 'Matched',
                    'Received', 'Rescheduled', 'Running', 'Staging', 'Stalled',
                    'Waiting' )
    
    for resultTuple in results[ 'Value' ]:
      
      selectionDict, numberOfJobs = resultTuple
    
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB [as 别名]
# 或者: from DIRAC.WorkloadManagementSystem.DB.JobDB.JobDB import getCounters [as 别名]

#.........这里部分代码省略.........
        return result
      attLimits = result['Value']
      try:
        attLimits = dict([(k, int(attLimits[k])) for k in attLimits])
      except Exception as excp:
        errMsg = "%s/%s has to contain numbers: %s" % (section, attName, str(excp))
        self.log.error(errMsg)
        return S_ERROR(errMsg)
      stuffDict[attName] = attLimits

    self.csDictCache.add(section, 300, stuffDict)
    return S_OK(stuffDict)

  def __getRunningCondition(self, siteName):
    """ Get extra conditions allowing site throttling
    """
    siteSection = "%s/%s" % (self.__runningLimitSection, siteName)
    result = self.__extractCSData(siteSection)
    if not result['OK']:
      return result
    limitsDict = result['Value']
    # limitsDict is something like { 'JobType' : { 'Merge' : 20, 'MCGen' : 1000 } }
    if not limitsDict:
      return S_OK({})
    # Check if the site exceeding the given limits
    negCond = {}
    for attName in limitsDict:
      if attName not in self.jobDB.jobAttributeNames:
        self.log.error("Attribute %s does not exist. Check the job limits" % attName)
        continue
      cK = "Running:%s:%s" % (siteName, attName)
      data = self.condCache.get(cK)
      if not data:
        result = self.jobDB.getCounters(
            'Jobs', [attName], {
                'Site': siteName, 'Status': [
                    'Running', 'Matched', 'Stalled']})
        if not result['OK']:
          return result
        data = result['Value']
        data = dict([(k[0][attName], k[1]) for k in data])
        self.condCache.add(cK, 10, data)
      for attValue in limitsDict[attName]:
        limit = limitsDict[attName][attValue]
        running = data.get(attValue, 0)
        if running >= limit:
          self.log.verbose('Job Limit imposed at %s on %s/%s=%d,'
                           ' %d jobs already deployed' % (siteName, attName, attValue, limit, running))
          if attName not in negCond:
            negCond[attName] = []
          negCond[attName].append(attValue)
    # negCond is something like : {'JobType': ['Merge']}
    return S_OK(negCond)

  def updateDelayCounters(self, siteName, jid):
    # Get the info from the CS
    siteSection = "%s/%s" % (self.__matchingDelaySection, siteName)
    result = self.__extractCSData(siteSection)
    if not result['OK']:
      return result
    delayDict = result['Value']
    # limitsDict is something like { 'JobType' : { 'Merge' : 20, 'MCGen' : 1000 } }
    if not delayDict:
      return S_OK()
    attNames = []
    for attName in delayDict: