Python PilotAgentsDB.setPilotStatus方法代码示例

# 需要导入模块: from DIRAC.WorkloadManagementSystem.DB.PilotAgentsDB import PilotAgentsDB [as 别名]
# 或者: from DIRAC.WorkloadManagementSystem.DB.PilotAgentsDB.PilotAgentsDB import setPilotStatus [as 别名]

#.........这里部分代码省略.........


  def _checkMask( self, resourceDict ):
    """ Check the mask: are we allowed to run normal jobs?

        FIXME: should we move to site OR SE?
    """
    if not 'Site' in resourceDict:
      self.log.error( "Missing Site Name in Resource JDL" )
      raise RuntimeError( "Missing Site Name in Resource JDL" )

    # Get common site mask and check the agent site
    result = self.jobDB.getSiteMask( siteState = 'Active' )
    if not result['OK']:
      self.log.error( "Internal error", "getSiteMask: %s" % result['Message'] )
      raise RuntimeError( "Internal error" )
    maskList = result['Value']

    if resourceDict['Site'] not in maskList:
      return False

    return True

  def _updatePilotInfo( self, resourceDict ):
    """ Update pilot information - do not fail if we don't manage to do it
    """
    pilotReference = resourceDict.get( 'PilotReference', '' )
    if pilotReference:
      gridCE = resourceDict.get( 'GridCE', 'Unknown' )
      site = resourceDict.get( 'Site', 'Unknown' )
      benchmark = resourceDict.get( 'PilotBenchmark', 0.0 )
      self.log.verbose( 'Reporting pilot info for %s: gridCE=%s, site=%s, benchmark=%f' % ( pilotReference, gridCE, site, benchmark ) )

      result = self.pilotAgentsDB.setPilotStatus( pilotReference, status = 'Running', gridSite = site,
                                                  destination = gridCE, benchmark = benchmark )
      if not result['OK']:
        self.log.error( "Problem updating pilot information",
                        "; setPilotStatus. pilotReference: %s; %s" % ( pilotReference, result['Message'] ) )

  def _updatePilotJobMapping( self, resourceDict, jobID ):
    """ Update pilot to job mapping information
    """
    pilotReference = resourceDict.get( 'PilotReference', '' )
    if pilotReference:
      result = self.pilotAgentsDB.setCurrentJobID( pilotReference, jobID )
      if not result['OK']:
        self.log.error( "Problem updating pilot information",
                        ";setCurrentJobID. pilotReference: %s; %s" % ( pilotReference, result['Message'] ) )
      result = self.pilotAgentsDB.setJobForPilot( jobID, pilotReference, updateStatus = False )
      if not result['OK']:
        self.log.error( "Problem updating pilot information",
                        "; setJobForPilot. pilotReference: %s; %s" % ( pilotReference, result['Message'] ) )

  def _checkCredentials( self, resourceDict, credDict ):
    """ Check if we can get a job given the passed credentials
    """
    if Properties.GENERIC_PILOT in credDict[ 'properties' ]:
      # You can only match groups in the same VO
      if credDict[ 'group' ] == "hosts":
        # for the host case the VirtualOrganization parameter
        # is mandatory in resourceDict
        vo = resourceDict.get( 'VirtualOrganization', '' )
      else:
        vo = Registry.getVOForGroup( credDict[ 'group' ] )
      result = Registry.getGroupsForVO( vo )
      if result[ 'OK' ]:

# 需要导入模块: from DIRAC.WorkloadManagementSystem.DB.PilotAgentsDB import PilotAgentsDB [as 别名]
# 或者: from DIRAC.WorkloadManagementSystem.DB.PilotAgentsDB.PilotAgentsDB import setPilotStatus [as 别名]
class PilotStatusAgent(AgentModule):
  """
      The specific agents must provide the following methods:
        - initialize() for initial settings
        - beginExecution()
        - execute() - the main method called in the agent cycle
        - endExecution()
        - finalize() - the graceful exit of the method, this one is usually used
                   for the agent restart
  """

  queryStateList = ['Ready', 'Submitted', 'Running', 'Waiting', 'Scheduled']
  finalStateList = ['Done', 'Aborted', 'Cleared', 'Deleted', 'Failed']

  def __init__(self, *args, **kwargs):
    """ c'tor
    """
    AgentModule.__init__(self, *args, **kwargs)

    self.jobDB = None
    self.pilotDB = None
    self.diracadmin = None

  #############################################################################
  def initialize(self):
    """Sets defaults
    """

    self.am_setOption('PollingTime', 120)
    self.am_setOption('GridEnv', '')
    self.am_setOption('PilotStalledDays', 3)
    self.pilotDB = PilotAgentsDB()
    self.diracadmin = DiracAdmin()
    self.jobDB = JobDB()
    self.clearPilotsDelay = self.am_getOption('ClearPilotsDelay', 30)
    self.clearAbortedDelay = self.am_getOption('ClearAbortedPilotsDelay', 7)
    self.WMSAdministrator = WMSAdministratorClient()

    return S_OK()

  #############################################################################
  def execute(self):
    """The PilotAgent execution method.
    """

    self.pilotStalledDays = self.am_getOption('PilotStalledDays', 3)
    self.gridEnv = self.am_getOption('GridEnv')
    if not self.gridEnv:
      # No specific option found, try a general one
      setup = gConfig.getValue('/DIRAC/Setup', '')
      if setup:
        instance = gConfig.getValue('/DIRAC/Setups/%s/WorkloadManagement' % setup, '')
        if instance:
          self.gridEnv = gConfig.getValue('/Systems/WorkloadManagement/%s/GridEnv' % instance, '')
    result = self.pilotDB._getConnection()
    if result['OK']:
      connection = result['Value']
    else:
      return result

    # Now handle pilots not updated in the last N days (most likely the Broker is no
    # longer available) and declare them Deleted.
    result = self.handleOldPilots(connection)

    connection.close()

    result = self.WMSAdministrator.clearPilots(self.clearPilotsDelay, self.clearAbortedDelay)
    if not result['OK']:
      self.log.warn('Failed to clear old pilots in the PilotAgentsDB')

    return S_OK()

  def clearWaitingPilots(self, condDict):
    """ Clear pilots in the faulty Waiting state
    """

    last_update = Time.dateTime() - MAX_WAITING_STATE_LENGTH * Time.hour
    clearDict = {'Status': 'Waiting',
                 'OwnerDN': condDict['OwnerDN'],
                 'OwnerGroup': condDict['OwnerGroup'],
                 'GridType': condDict['GridType'],
                 'Broker': condDict['Broker']}
    result = self.pilotDB.selectPilots(clearDict, older=last_update)
    if not result['OK']:
      self.log.warn('Failed to get the Pilot Agents for Waiting state')
      return result
    if not result['Value']:
      return S_OK()
    refList = result['Value']

    for pilotRef in refList:
      self.log.info('Setting Waiting pilot to Stalled: %s' % pilotRef)
      result = self.pilotDB.setPilotStatus(pilotRef, 'Stalled', statusReason='Exceeded max waiting time')

    return S_OK()

  def clearParentJob(self, pRef, pDict, connection):
    """ Clear the parameteric parent job from the PilotAgentsDB
    """

#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.WorkloadManagementSystem.DB.PilotAgentsDB import PilotAgentsDB [as 别名]
# 或者: from DIRAC.WorkloadManagementSystem.DB.PilotAgentsDB.PilotAgentsDB import setPilotStatus [as 别名]

#.........这里部分代码省略.........
                    "Getting status for %s pilots for owner %s and group %s" % (len(refList), ownerDN, ownerGroup)
                )

                for start_index in range(0, len(refList), MAX_JOBS_QUERY):
                    refsToQuery = refList[start_index : start_index + MAX_JOBS_QUERY]
                    self.log.verbose(
                        "Querying %d pilots of %s starting at %d" % (len(refsToQuery), len(refList), start_index)
                    )
                    result = self.getPilotStatus(proxy, gridType, refsToQuery)
                    if not result["OK"]:
                        if result["Message"] == "Broker not Available":
                            self.log.error("Broker %s not Available" % broker)
                            break
                        self.log.warn("Failed to get pilot status:")
                        self.log.warn("%s:%s @ %s" % (ownerDN, ownerGroup, gridType))
                        continue

                    statusDict = result["Value"]
                    for pRef in statusDict:
                        pDict = statusDict[pRef]
                        if pDict:
                            if pDict["isParent"]:
                                self.log.verbose("Clear parametric parent %s" % pRef)
                                result = self.clearParentJob(pRef, pDict, connection)
                                if not result["OK"]:
                                    self.log.warn(result["Message"])
                                else:
                                    self.log.info("Parametric parent removed: %s" % pRef)
                            if pDict["FinalStatus"]:
                                self.log.verbose("Marking Status for %s to %s" % (pRef, pDict["Status"]))
                                pilotsToAccount[pRef] = pDict
                            else:
                                self.log.verbose("Setting Status for %s to %s" % (pRef, pDict["Status"]))
                                result = self.pilotDB.setPilotStatus(
                                    pRef,
                                    pDict["Status"],
                                    pDict["DestinationSite"],
                                    updateTime=pDict["StatusDate"],
                                    conn=connection,
                                )

                    if len(pilotsToAccount) > 100:
                        self.accountPilots(pilotsToAccount, connection)
                        pilotsToAccount = {}

        self.accountPilots(pilotsToAccount, connection)
        # Now handle pilots not updated in the last N days (most likely the Broker is no
        # longer available) and declare them Deleted.
        result = self.handleOldPilots(connection)

        connection.close()

        return S_OK()

    def clearWaitingPilots(self, condDict):
        """ Clear pilots in the faulty Waiting state
    """

        last_update = Time.dateTime() - MAX_WAITING_STATE_LENGTH * Time.hour
        clearDict = {
            "Status": "Waiting",
            "OwnerDN": condDict["OwnerDN"],
            "OwnerGroup": condDict["OwnerGroup"],
            "GridType": condDict["GridType"],
            "Broker": condDict["Broker"],
        }

# 需要导入模块: from DIRAC.WorkloadManagementSystem.DB.PilotAgentsDB import PilotAgentsDB [as 别名]
# 或者: from DIRAC.WorkloadManagementSystem.DB.PilotAgentsDB.PilotAgentsDB import setPilotStatus [as 别名]

#.........这里部分代码省略.........

        self.log.verbose( "Getting status for %s pilots for owner %s and group %s" % ( len( refList ),
                                                                                      ownerDN, ownerGroup ) )

        for start_index in range( 0, len( refList ), MAX_JOBS_QUERY ):
          refsToQuery = refList[ start_index : start_index + MAX_JOBS_QUERY ]
          self.log.verbose( 'Querying %d pilots of %s starting at %d' %
                            ( len( refsToQuery ), len( refList ), start_index ) )
          result = self.getPilotStatus( proxy, gridType, refsToQuery )
          if not result['OK']:
            if result['Message'] == 'Broker not Available':
              self.log.error( 'Broker %s not Available' % broker )
              break
            self.log.warn( 'Failed to get pilot status:' )
            self.log.warn( '%s:%s @ %s' % ( ownerDN, ownerGroup, gridType ) )
            continue

          statusDict = result[ 'Value' ]
          for pRef in statusDict:
            pDict = statusDict[ pRef ]
            if pDict:
              if pDict['isParent']:
                self.log.verbose( 'Clear parametric parent %s' % pRef )
                result = self.clearParentJob( pRef, pDict, connection )
                if not result['OK']:
                  self.log.warn( result['Message'] )
                else:
                  self.log.info( 'Parametric parent removed: %s' % pRef )
              if pDict[ 'FinalStatus' ]:
                self.log.verbose( 'Marking Status for %s to %s' % ( pRef, pDict['Status'] ) )
                pilotsToAccount[ pRef ] = pDict
              else:
                self.log.verbose( 'Setting Status for %s to %s' % ( pRef, pDict['Status'] ) )
                result = self.pilotDB.setPilotStatus( pRef,
                                                      pDict['Status'],
                                                      pDict['DestinationSite'],
                                                      updateTime = pDict['StatusDate'],
                                                      conn = connection )

          if len( pilotsToAccount ) > 100:
            self.accountPilots( pilotsToAccount, connection )
            pilotsToAccount = {}

    self.accountPilots( pilotsToAccount, connection )
    # Now handle pilots not updated in the last N days (most likely the Broker is no 
    # longer available) and declare them Deleted.
    result = self.handleOldPilots( connection )

    connection.close()

    return S_OK()

  def clearWaitingPilots( self, condDict ):
    """ Clear pilots in the faulty Waiting state
    """

    last_update = Time.dateTime() - MAX_WAITING_STATE_LENGTH * Time.hour
    clearDict = {'Status':'Waiting',
                 'OwnerDN':condDict['OwnerDN'],
                 'OwnerGroup':condDict['OwnerGroup'],
                 'GridType':condDict['GridType'],
                 'Broker':condDict['Broker']}
    result = self.pilotDB.selectPilots( clearDict, older = last_update )
    if not result['OK']:
      self.log.warn( 'Failed to get the Pilot Agents for Waiting state' )
      return result