Python ResourceStatusClient.selectStatusElement方法代码示例

# 需要导入模块: from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient import ResourceStatusClient [as 别名]
# 或者: from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient.ResourceStatusClient import selectStatusElement [as 别名]
  def getSiteMaskLogging( self, site = None, printOutput = False ):
    """Retrieves site mask logging information.

       Example usage:

       >>> print diracAdmin.getSiteMaskLogging('LCG.AUVER.fr')
       {'OK': True, 'Value': }

       :returns: S_OK,S_ERROR
    """
    result = self.__checkSiteIsValid( site )
    if not result['OK']:
      return result
    
    rssClient = ResourceStatusClient()
    result = rssClient.selectStatusElement( 'Site', 'History', name = site, 
                                            statusType = 'ComputingAccess' )
    
    if not result['OK']:
      return result

    siteDict = {}
    for logTuple in result['Value']:
      status,reason,siteName,dateEffective,dateTokenExpiration,eType,sType,eID,lastCheckTime,author = logTuple
      result = getSiteFullNames( siteName )
      if not result['OK']:
        continue
      for sName in result['Value']:
        if site is None or (site and site == sName):
          siteDict.setdefault( sName, [] )
          siteDict[sName].append( (status,reason,dateEffective,author,dateTokenExpiration) )

    if printOutput:
      if site:
        print '\nSite Mask Logging Info for %s\n' % site
      else:
        print '\nAll Site Mask Logging Info\n'

      for site, tupleList in siteDict.items():
        if not site:
          print '\n===> %s\n' % site
        for tup in tupleList:
          print str( tup[0] ).ljust( 8 ) + str( tup[1] ).ljust( 20 ) + \
               '( ' + str( tup[2] ).ljust( len( str( tup[2] ) ) ) + ' )  "' + str( tup[3] ) + '"'
        print ' '
        
    return S_OK( siteDict )

# 需要导入模块: from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient import ResourceStatusClient [as 别名]
# 或者: from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient.ResourceStatusClient import selectStatusElement [as 别名]
class PropagationCommand(Command):

  def __init__(self, args=None, clients=None):

    self.rssClient = ResourceStatusClient()
    super(PropagationCommand, self).__init__(args, clients)

  def doNew(self, masterParams=None):
    return S_OK()

  def doCache(self):

    if not self.args['site']:
      return S_ERROR('site was not found in args')

    site = self.args['site']

    elements = CSHelpers.getSiteElements(site)

    statusList = []

    if elements['OK']:
      for element in elements['Value']:
        status = self.rssClient.selectStatusElement("Resource", "Status", element, meta={'columns': ['Status']})
        if not status['OK']:
          return status

        if status['Value']:
          statusList.append(status['Value'][0][0])
        else:  # forcing in the case the resource has no status (yet)
          statusList.append('Active')

      if 'Active' in statusList:
        return S_OK({'Status': 'Active', 'Reason': 'An element that belongs to the site is Active'})

      if 'Degraded' in statusList:
        return S_OK({'Status': 'Degraded', 'Reason': 'An element that belongs to the site is Degraded'})

    return S_OK({'Status': 'Banned', 'Reason': 'There is no Active element in the site'})

  def doMaster(self):
    return S_OK()

# 需要导入模块: from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient import ResourceStatusClient [as 别名]
# 或者: from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient.ResourceStatusClient import selectStatusElement [as 别名]
class TokenAgent( AgentModule ):
  '''
    TokenAgent is in charge of checking tokens assigned on resources.
    Notifications are sent to those users owning expiring tokens.
  '''

  # Hours to notify a user
  __notifyHours = 12

  # Rss token
  __rssToken = 'rs_svc'

  # Admin mail
  __adminMail = None

  def __init__( self, *args, **kwargs ):
    ''' c'tor
    '''

    AgentModule.__init__( self, *args, **kwargs )

    self.notifyHours = self.__notifyHours
    self.adminMail = self.__adminMail

    self.rsClient = None
    self.rmClient = None
    self.noClient = None

    self.tokenDict = None
    self.diracAdmin = None

  def initialize( self ):
    ''' TokenAgent initialization
        Uses the ProductionManager shifterProxy to modify the ResourceStatus DB
    '''
    self.am_setOption( 'shifterProxy', 'ProductionManager' )

    self.notifyHours = self.am_getOption( 'notifyHours', self.notifyHours )

    self.rsClient = ResourceStatusClient()
    self.rmClient = ResourceManagementClient()
    self.noClient = NotificationClient()

    self.diracAdmin = DiracAdmin()

    return S_OK()

  def execute( self ):
    '''
      Looks for user tokens. If they are expired, or expiring, it notifies users.
    '''

    # Initialized here, as it is needed empty at the beginning of the execution
    self.tokenDict = {}

    # FIXME: probably this can be obtained from RssConfiguration instead
    elements = ( 'Site', 'Resource', 'Node' )

    for element in elements:

      self.log.info( 'Processing %s' % element )

      interestingTokens = self._getInterestingTokens( element )
      if not interestingTokens[ 'OK' ]:
        self.log.error( interestingTokens[ 'Message' ] )
        continue
      interestingTokens = interestingTokens[ 'Value' ]

      processTokens = self._processTokens( element, interestingTokens )
      if not processTokens[ 'OK' ]:
        self.log.error( processTokens[ 'Message' ] )
        continue

    notificationResult = self._notifyOfTokens()
    if not notificationResult[ 'OK' ]:
      self.log.error( notificationResult[ 'Message' ] )

    return S_OK()

  ## Protected methods #########################################################

  def _getInterestingTokens( self, element ):
    '''
      Given an element, picks all the entries with TokenExpiration < now + X<hours>
      If the TokenOwner is not the rssToken ( rs_svc ), it is selected.
    '''

    tokenExpLimit = datetime.utcnow() + timedelta( hours = self.notifyHours )

    tokenElements = self.rsClient.selectStatusElement( element, 'Status',
                                                       meta = { 'older' : ( 'TokenExpiration', tokenExpLimit ) } )

    if not tokenElements[ 'OK' ]:
      return tokenElements

    tokenColumns = tokenElements[ 'Columns' ]
    tokenElements = tokenElements[ 'Value' ]

    interestingTokens = []

#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient import ResourceStatusClient [as 别名]
# 或者: from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient.ResourceStatusClient import selectStatusElement [as 别名]

#.........这里部分代码省略.........
    # without having to spawn too many threads. We assume 10 seconds per element
    # to be processed ( actually, it takes something like 1 sec per element ):
    # numberOfThreads = elements * 10(s/element) / pollingTime
    numberOfThreads = int( math.ceil( queueSize * 10. / pollingTime ) )
            
    self.log.info( 'Needed %d threads to process %d elements' % ( numberOfThreads, queueSize ) )
    
    for _x in xrange( numberOfThreads ):
      jobUp = self.threadPool.generateJobAndQueueIt( self._execute )
      if not jobUp[ 'OK' ]:
        self.log.error( jobUp[ 'Message' ] )
        
    self.log.info( 'blocking until all elements have been processed' )
    # block until all tasks are done
    self.elementsToBeChecked.join()
    self.log.info( 'done')  
    
    return S_OK()


  def getElementsToBeChecked( self ):
    """ getElementsToBeChecked
    
    This method gets all the rows in the <self.elementType>Status table, and then
    discards entries with TokenOwner != rs_svc. On top of that, there are check
    frequencies that are applied: depending on the current status of the element,
    they will be checked more or less often.
    
    """
    
    toBeChecked = Queue.Queue()
    
    # We get all the elements, then we filter.
    elements = self.rsClient.selectStatusElement( self.elementType, 'Status' )
    if not elements[ 'OK' ]:
      return elements
      
    utcnow = datetime.datetime.utcnow().replace( microsecond = 0 )  
       
    # filter elements by Type
    for element in elements[ 'Value' ]:
      
      # Maybe an overkill, but this way I have NEVER again to worry about order
      # of elements returned by mySQL on tuples
      elemDict = dict( zip( elements[ 'Columns' ], element ) )
      
      # This if-clause skips all the elements that are should not be checked yet
      timeToNextCheck = self.__checkingFreqs[ elemDict[ 'Status' ] ]
      if utcnow <= elemDict[ 'LastCheckTime' ] + datetime.timedelta( minutes = timeToNextCheck ):
        continue
      
      # We skip the elements with token different than "rs_svc"
      if elemDict[ 'TokenOwner' ] != 'rs_svc':
        self.log.verbose( 'Skipping %s ( %s ) with token %s' % ( elemDict[ 'Name' ],
                                                                 elemDict[ 'StatusType' ],
                                                                 elemDict[ 'TokenOwner' ]
                                                               ))
        continue
              
      # We are not checking if the item is already on the queue or not. It may
      # be there, but in any case, it is not a big problem.
        
      lowerElementDict = { 'element' : self.elementType }
      for key, value in elemDict.items():
        lowerElementDict[ key[0].lower() + key[1:] ] = value
        

# 需要导入模块: from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient import ResourceStatusClient [as 别名]
# 或者: from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient.ResourceStatusClient import selectStatusElement [as 别名]
class TokenAgent(AgentModule):
    """
    TokenAgent is in charge of checking tokens assigned on resources.
    Notifications are sent to those users owning expiring tokens.
  """

    # Hours to notify a user
    __notifyHours = 12

    # Rss token
    __rssToken = "rs_svc"

    # Admin mail
    __adminMail = None

    def __init__(self, *args, **kwargs):
        """ c'tor
    """

        AgentModule.__init__(self, *args, **kwargs)

        self.notifyHours = self.__notifyHours
        self.adminMail = self.__adminMail

        self.rsClient = None
        self.tokenDict = None
        self.diracAdmin = None

    def initialize(self):
        """ TokenAgent initialization
    """

        self.notifyHours = self.am_getOption("notifyHours", self.notifyHours)
        self.adminMail = self.am_getOption("adminMail", self.adminMail)

        self.rsClient = ResourceStatusClient()
        self.diracAdmin = DiracAdmin()

        return S_OK()

    def execute(self):
        """
      Looks for user tokens. If they are expired, or expiring, it notifies users.
    """

        # Initialized here, as it is needed empty at the beginning of the execution
        self.tokenDict = {}

        # FIXME: probably this can be obtained from RssConfiguration instead
        elements = ("Site", "Resource", "Node")

        for element in elements:

            self.log.info("Processing %s" % element)

            interestingTokens = self._getInterestingTokens(element)
            if not interestingTokens["OK"]:
                self.log.error(interestingTokens["Message"])
                continue
            interestingTokens = interestingTokens["Value"]

            processTokens = self._processTokens(element, interestingTokens)
            if not processTokens["OK"]:
                self.log.error(processTokens["Message"])
                continue

        notificationResult = self._notifyOfTokens()
        if not notificationResult["OK"]:
            self.log.error(notificationResult["Message"])

        return S_OK()

    ## Protected methods #########################################################

    def _getInterestingTokens(self, element):
        """
      Given an element, picks all the entries with TokenExpiration < now + X<hours>
      If the TokenOwner is not the rssToken ( rs_svc ), it is selected.
    """

        tokenExpLimit = datetime.utcnow() + timedelta(hours=self.notifyHours)

        tokenElements = self.rsClient.selectStatusElement(
            element, "Status", meta={"older": ("TokenExpiration", tokenExpLimit)}
        )

        if not tokenElements["OK"]:
            return tokenElements

        tokenColumns = tokenElements["Columns"]
        tokenElements = tokenElements["Value"]

        interestingTokens = []

        for tokenElement in tokenElements:

            tokenElement = dict(zip(tokenColumns, tokenElement))

            if tokenElement["TokenOwner"] != self.__rssToken:
                interestingTokens.append(tokenElement)
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient import ResourceStatusClient [as 别名]
# 或者: from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient.ResourceStatusClient import selectStatusElement [as 别名]
def setToken(user):
  '''
    Function that gets the user token, sets the validity for it. Gets the elements
    in the database for a given name and statusType(s). Then updates the status
    of all them adding a reason and the token.
  '''

  rssClient = ResourceStatusClient()

  # This is a little bit of a nonsense, and certainly needs to be improved.
  # To modify a list of elements, we have to do it one by one. However, the
  # modify method does not discover the StatusTypes ( which in this script is
  # an optional parameter ). So, we get them from the DB and iterate over them.
  elements = rssClient.selectStatusElement(switchDict['element'], 'Status',
                                           name=switchDict['name'],
                                           statusType=switchDict['statusType'],
                                           meta={'columns': ['StatusType', 'TokenOwner']})

  if not elements['OK']:
    return elements
  elements = elements['Value']

  # If there list is empty they do not exist on the DB !
  if not elements:
    subLogger.warn('Nothing found for %s, %s, %s' % (switchDict['element'],
                                                     switchDict['name'],
                                                     switchDict['statusType']))
    return S_OK()

  # If we want to release the token
  if switchDict['releaseToken']:
    tokenExpiration = datetime.max
    newTokenOwner = 'rs_svc'
  else:
    tokenExpiration = datetime.utcnow().replace(microsecond=0) + timedelta(days=int(switchDict['days']))
    newTokenOwner = user

  subLogger.always('New token: %s --- until %s' % (newTokenOwner, tokenExpiration))

  for statusType, tokenOwner in elements:

    # If a user different than the one issuing the command and RSS
    if tokenOwner != user and tokenOwner != 'rs_svc':
      subLogger.info('%s(%s) belongs to the user: %s' % (switchDict['name'], statusType, tokenOwner))

    # does the job
    result = rssClient.modifyStatusElement(switchDict['element'], 'Status',
                                           name=switchDict['name'],
                                           statusType=statusType,
                                           reason=switchDict['reason'],
                                           tokenOwner=newTokenOwner,
                                           tokenExpiration=tokenExpiration)
    if not result['OK']:
      return result

    if tokenOwner == newTokenOwner:
      msg = '(extended)'
    elif newTokenOwner == 'rs_svc':
      msg = '(released)'
    else:
      msg = '(aquired from %s)' % tokenOwner

    subLogger.info('%s:%s %s' % (switchDict['name'], statusType, msg))
  return S_OK()

# 需要导入模块: from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient import ResourceStatusClient [as 别名]
# 或者: from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient.ResourceStatusClient import selectStatusElement [as 别名]
class ElementInspectorAgent( AgentModule ):
  '''
    The ElementInspector agent is a generic agent used to check the elements
    of one of the elementTypes ( e.g. Site, Resource, Node ).

    This Agent takes care of the Elements. In order to do so, it gathers
    the eligible ones and then evaluates their statuses with the PEP.
  '''

  # Max number of worker threads by default
  __maxNumberOfThreads = 5
  # ElementType, to be defined among Site, Resource or Node
  __elementType = None
  # Inspection freqs, defaults, the lower, the higher priority to be checked.
  # Error state usually means there is a glitch somewhere, so it has the highest
  # priority.
  __checkingFreqs = { 'Default' : 
                       { 
                         'Active' : 60, 'Degraded' : 30,  'Probing' : 30, 
                         'Banned' : 30, 'Unknown'  : 15,  'Error'   : 15 
                         } 
                     }
  # queue size limit to stop feeding
  __limitQueueFeeder = 15
  
  def __init__( self, *args, **kwargs ):
    ''' c'tor
    '''
    
    AgentModule.__init__( self, *args, **kwargs )

    # members initialization

    self.maxNumberOfThreads = self.__maxNumberOfThreads
    self.elementType        = self.__elementType
    self.checkingFreqs      = self.__checkingFreqs
    self.limitQueueFeeder   = self.__limitQueueFeeder
    
    self.elementsToBeChecked = None
    self.threadPool          = None
    self.rsClient            = None
    self.clients             = {}

  def initialize( self ):
    ''' Standard initialize.
        Uses the ProductionManager shifterProxy to modify the ResourceStatus DB
    '''

    self.maxNumberOfThreads = self.am_getOption( 'maxNumberOfThreads', self.maxNumberOfThreads )   
    self.elementType        = self.am_getOption( 'elementType',        self.elementType )
    self.checkingFreqs      = self.am_getOption( 'checkingFreqs',      self.checkingFreqs )
    self.limitQueueFeeder   = self.am_getOption( 'limitQueueFeeder',   self.limitQueueFeeder )      
    
    self.elementsToBeChecked = Queue.Queue()
    self.threadPool          = ThreadPool( self.maxNumberOfThreads,
                                           self.maxNumberOfThreads )

    self.rsClient = ResourceStatusClient()

    self.clients[ 'ResourceStatusClient' ]     = self.rsClient
    self.clients[ 'ResourceManagementClient' ] = ResourceManagementClient() 

    return S_OK()
  
  def execute( self ):
    
    # If there are elements in the queue to be processed, we wait ( we know how
    # many elements in total we can have, so if there are more than 15% of them
    # on the queue, we do not add anything ), but the threads are running and
    # processing items from the queue on background.    
    
    qsize = self.elementsToBeChecked.qsize() 
    if qsize > self.limitQueueFeeder:
      self.log.warn( 'Queue not empty ( %s > %s ), skipping feeding loop' % ( qsize, self.limitQueueFeeder ) )
      return S_OK()
    
    # We get all the elements, then we filter.
    elements = self.rsClient.selectStatusElement( self.elementType, 'Status' )
    if not elements[ 'OK' ]:
      self.log.error( elements[ 'Message' ] )
      return elements
      
    utcnow = datetime.datetime.utcnow().replace( microsecond = 0 )  
       
    # filter elements by Type
    for element in elements[ 'Value' ]:
      
      # Maybe an overkill, but this way I have NEVER again to worry about order
      # of elements returned by mySQL on tuples
      elemDict = dict( zip( elements[ 'Columns' ], element ) )
      
      # We skip the elements with token different than "rs_svc"
      if elemDict[ 'TokenOwner' ] != 'rs_svc':
        self.log.info( 'Skipping %s ( %s ) with token %s' % ( elemDict[ 'Name' ],
                                                              elemDict[ 'StatusType' ],
                                                              elemDict[ 'TokenOwner' ]
                                                             ))
        continue
      
      if not elemDict[ 'ElementType' ] in self.checkingFreqs:
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient import ResourceStatusClient [as 别名]
# 或者: from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient.ResourceStatusClient import selectStatusElement [as 别名]
class SummarizeLogsAgent(AgentModule):
  """ SummarizeLogsAgent as extension of AgentModule.
  """

  def __init__(self, *args, **kwargs):
    """ Constructor.

    """

    AgentModule.__init__(self, *args, **kwargs)

    self.rsClient = None

  def initialize(self):
    """ Standard initialize.

    :return: S_OK

    """

    self.rsClient = ResourceStatusClient()
    return S_OK()

  def execute(self):
    """ execute ( main method )

    The execute method runs over the three families of tables ( Site, Resource and
    Node ) performing identical operations. First, selects all logs for a given
    family ( and keeps track of which one is the last row ID ). It summarizes the
    logs and finally, deletes the logs from the database.

    :return: S_OK

    """

    # loop over the tables
    for element in ('Site', 'Resource', 'Node'):

      self.log.info('Summarizing %s' % element)

      # get all logs to be summarized
      selectLogElements = self._summarizeLogs(element)
      if not selectLogElements['OK']:
        self.log.error(selectLogElements['Message'])
        continue

      lastID, logElements = selectLogElements['Value']

      # logElements is a dictionary of key-value pairs as follows:
      # ( name, statusType ) : list( logs )
      for key, logs in logElements.iteritems():

        sumResult = self._registerLogs(element, key, logs)
        if not sumResult['OK']:
          self.log.error(sumResult['Message'])
          continue

      if lastID is not None:
        self.log.info('Deleting %sLog till ID %s' % (element, lastID))
        deleteResult = self.rsClient.deleteStatusElement(element, 'Log',
                                                         meta={'older': ('ID', lastID)})
        if not deleteResult['OK']:
          self.log.error(deleteResult['Message'])
          continue

    return S_OK()

  def _summarizeLogs(self, element):
    """ given an element, selects all logs in table <element>Log.

    :Parameters:
      **element** - `string`
        name of the table family ( either Site, Resource or Node )

    :return: S_OK( lastID, listOfLogs ) / S_ERROR

    """

    selectResults = self.rsClient.selectStatusElement(element, 'Log')

    if not selectResults['OK']:
      return selectResults

    selectedItems = {}
    latestID = None

    if not selectResults['Value']:
      return S_OK((latestID, selectedItems))

    selectColumns = selectResults['Columns']
    selectResults = selectResults['Value']

    if selectResults:
      latestID = dict(zip(selectColumns, selectResults[-1]))['ID']

    for selectResult in selectResults:

      elementDict = dict(zip(selectColumns, selectResult))

      key = (elementDict['Name'], elementDict['StatusType'])
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient import ResourceStatusClient [as 别名]
# 或者: from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient.ResourceStatusClient import selectStatusElement [as 别名]
class SummarizeLogsAgent( AgentModule ):

  # Date format in database
  __dateFormat = '%Y-%m-%d %H:%M:%S'

  def __init__( self, *args, **kwargs ):
    ''' c'tor
    '''

    AgentModule.__init__( self, *args, **kwargs )

    self.rsClient = None

  def initialize( self ):
    ''' Standard initialize.
        Uses the ProductionManager shifterProxy to modify the ResourceStatus DB
    '''

    self.rsClient = ResourceStatusClient()

    return S_OK()

  def execute( self ):

    # FIXME: probably this can be obtained from RssConfiguration instead
    elements = ( 'Site', 'Resource', 'Node' )

    # We do not want neither minutes, nor seconds nor microseconds
    thisHour = datetime.utcnow().replace( microsecond = 0 )
    thisHour = thisHour.replace( second = 0 ).replace( minute = 0 )

    for element in elements:

      self.log.info( 'Summarizing %s' % element )

      selectLogElements = self._selectLogElements( element, thisHour )
      if not selectLogElements[ 'OK' ]:
        self.log.error( selectLogElements[ 'Message' ] )
        continue
      selectLogElements = selectLogElements[ 'Value' ]

      for selectedKey, selectedItem in selectLogElements.items():

        sRes = self._logSelectedLogElement( element, selectedKey, selectedItem, thisHour )
        if not sRes[ 'OK' ]:
          self.log.error( sRes[ 'Message' ] )
          break

    return S_OK()

  def _selectLogElements( self, element, thisHour ):
    '''
      For a given element, selects all the entries on the <element>Log table
      with LastCheckTime > <lastHour>. It groups them by tuples of
      ( <name>, <statusType> ) and keeps only the statuses that represent
      a change in the status.
    '''

    lastHour = thisHour - timedelta( hours = 1 )

    selectResults = self.rsClient.selectStatusElement( element, 'Log',
                                                       meta = { 'newer' : ( 'LastCheckTime', lastHour ) } )
    if not selectResults[ 'OK' ]:
      return selectResults

    selectedItems = {}
    selectColumns = selectResults[ 'Columns' ]
    selectResults = selectResults[ 'Value' ]

    for selectResult in selectResults:

      elementDict = dict( zip( selectColumns, selectResult ) )

      if elementDict[ 'LastCheckTime' ] > thisHour:
        continue

      key = ( elementDict[ 'Name' ], elementDict[ 'StatusType' ] )

      if not key in selectedItems:
        selectedItems[ key ] = [ elementDict ]
      else:
        lastStatus = selectedItems[ key ][ -1 ][ 'Status' ]
        if lastStatus != elementDict[ 'Status' ]:
          selectedItems[ key ].append( elementDict )

    return S_OK( selectedItems )

  def _logSelectedLogElement( self, element, selectedKey, selectedItem, thisHour ):
    '''
      Given an element, a selectedKey - which is a tuple ( <name>, <statusType> )
      and a list of dictionaries, this method inserts them. Before inserting
      them, checks whether the first one is or is not on the <element>History
      table. If it is, it is not inserted.
    '''

    name, statusType = selectedKey

    selectedRes = self.rsClient.selectStatusElement( element, 'History', name,
                                                     statusType,
                                                     meta = { 'columns' : [ 'Status', 'LastCheckTime' ] } )
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient import ResourceStatusClient [as 别名]
# 或者: from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient.ResourceStatusClient import selectStatusElement [as 别名]
class SiteStatus(object):
  """
  RSS helper to interact with the 'Site' family on the DB. It provides the most
  demanded functions and a cache to avoid hitting the server too often.

  It provides four methods to interact with the site statuses:
  * getSiteStatuses
  * isUsableSite
  * getUsableSites
  * getSites
  """

  __metaclass__ = DIRACSingleton

  def __init__(self):
    """
    Constructor, initializes the rssClient.
    """

    self.log = gLogger.getSubLogger(self.__class__.__name__)
    self.rssConfig = RssConfiguration()
    self.__opHelper = Operations()
    self.rssFlag = ResourceStatus().rssFlag
    self.rsClient = ResourceStatusClient()

    # We can set CacheLifetime and CacheHistory from CS, so that we can tune them.
    cacheLifeTime = int(self.rssConfig.getConfigCache())

    # RSSCache only affects the calls directed to RSS, if using the CS it is not used.
    self.rssCache = RSSCache(cacheLifeTime, self.__updateRssCache)

  def __updateRssCache(self):
    """ Method used to update the rssCache.

        It will try 5 times to contact the RSS before giving up
    """

    meta = {'columns': ['Name', 'Status']}

    for ti in xrange(5):
      rawCache = self.rsClient.selectStatusElement('Site', 'Status', meta=meta)
      if rawCache['OK']:
        break
      self.log.warn("Can't get resource's status", rawCache['Message'] + "; trial %d" % ti)
      sleep(math.pow(ti, 2))
      self.rsClient = ResourceStatusClient()

    if not rawCache['OK']:
      return rawCache
    return S_OK(getCacheDictFromRawData(rawCache['Value']))

  def getSiteStatuses(self, siteNames=None):
    """
    Method that queries the database for status of the sites in a given list.
    A single string site name may also be provides as "siteNames"
    If the input is None, it is interpreted as * ( all ).

    If match is positive, the output looks like:
    {
     'test1.test1.org': 'Active',
     'test2.test2.org': 'Banned',
    }

    examples
      >>> siteStatus.getSiteStatuses( ['test1.test1.uk', 'test2.test2.net', 'test3.test3.org'] )
          S_OK( { 'test1.test1.org': 'Active', 'test2.test2.net': 'Banned', 'test3.test3.org': 'Active' }  )
      >>> siteStatus.getSiteStatuses( 'NotExists')
          S_ERROR( ... ))
      >>> siteStatus.getSiteStatuses( None )
          S_OK( { 'test1.test1.org': 'Active',
                  'test2.test2.net': 'Banned', },
                  ...
                }
              )

    :Parameters:
      **siteNames** - `list` or `str`
        name(s) of the sites to be matched

    :return: S_OK() || S_ERROR()
    """

    if self.rssFlag:
      return self.__getRSSSiteStatus(siteNames)
    else:
      siteStatusDict = {}
      wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator')
      if siteNames:
        if isinstance(siteNames, basestring):
          siteNames = [siteNames]
        for siteName in siteNames:
          result = wmsAdmin.getSiteMaskStatus(siteName)
          if not result['OK']:
            return result
          else:
            siteStatusDict[siteName] = result['Value']
      else:
        result = wmsAdmin.getSiteMaskStatus()
        if not result['OK']:
          return result
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient import ResourceStatusClient [as 别名]
# 或者: from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient.ResourceStatusClient import selectStatusElement [as 别名]

#.........这里部分代码省略.........
    the policies evaluated ( `singlePolicyResults` ) and the computed final result
    ( `policyCombinedResult` ).
    
    To know more about decisionParams, please read PDP.setup where the decisionParams
    are sanitized.
    
    examples:
       >>> pep.enforce( { 'element' : 'Site', 'name' : 'MySite' } )
       >>> pep.enforce( { 'element' : 'Resource', 'name' : 'myce.domain.ch' } )
    
    :Parameters:
      **decisionParams** - `dict`
        dictionary with the parameters that will be used to match policies.
    
    """ 
    
    # Setup PDP with new parameters dictionary
    self.pdp.setup( decisionParams )

    # Run policies, get decision, get actions to apply
    resDecisions = self.pdp.takeDecision()
    if not resDecisions[ 'OK' ]:
      gLogger.error( 'PEP: Something went wrong, not enforcing policies for %s' % decisionParams )
      return resDecisions
    resDecisions = resDecisions[ 'Value' ]
    
    # We take from PDP the decision parameters used to find the policies
    decisionParams       = resDecisions[ 'decissionParams' ]
    policyCombinedResult = resDecisions[ 'policyCombinedResult' ]
    singlePolicyResults  = resDecisions[ 'singlePolicyResults' ]

    # We have run the actions and at this point, we are about to execute the actions.
    # One more final check before proceeding
    isNotUpdated = self.__isNotUpdated( decisionParams )
    if not isNotUpdated[ 'OK' ]:
      return isNotUpdated
                
    for policyActionName, policyActionType in policyCombinedResult[ 'PolicyAction' ]:
      
      try:
        actionMod = Utils.voimport( 'DIRAC.ResourceStatusSystem.PolicySystem.Actions.%s' % policyActionType )
      except ImportError:
        gLogger.error( 'Error importing %s action' % policyActionType )
        continue
      
      try:
        action = getattr( actionMod, policyActionType )
      except AttributeError:
        gLogger.error( 'Error importing %s action class' % policyActionType )
        continue  
              
      actionObj = action( policyActionName, decisionParams, policyCombinedResult,
                          singlePolicyResults, self.clients )
      
      gLogger.debug( ( policyActionName, policyActionType ) )
      
      actionResult = actionObj.run()
      if not actionResult[ 'OK' ]:
        gLogger.error( actionResult[ 'Message' ] ) 
        
    return S_OK( resDecisions )


  def __isNotUpdated( self, decisionParams ):
    """ Checks for the existence of the element as it was passed to the PEP. It may
    happen that while being the element processed by the PEP an user through the 
    web interface or the CLI has updated the status for this particular element. As
    a result, the PEP would overwrite whatever the user had set. This check is not
    perfect, as still an user action can happen while executing the actions, but
    the probability is close to 0. However, if there is an action that takes seconds
    to be executed, this must be re-evaluated. !
    
    :Parameters:
      **decisionParams** - `dict`
        dictionary with the parameters that will be used to match policies
        
    :return: S_OK / S_ERROR
    
    """
    
    # Copy original dictionary and get rid of one key we cannot pass as kwarg
    selectParams = decisionParams.copy()
    del selectParams[ 'element' ]
    del selectParams[ 'active' ]
    
    # We expect to have an exact match. If not, then something has changed and
    # we cannot proceed with the actions.    
    unchangedRow = self.rsClient.selectStatusElement( decisionParams[ 'element' ], 
                                                      'Status', **selectParams )
    if not unchangedRow[ 'OK' ]:
      return unchangedRow
    
    if not unchangedRow[ 'Value' ]:
      msg = '%(name)s  ( %(status)s / %(statusType)s ) has been updated after PEP started running'
      return S_ERROR( msg % selectParams )
    
    return S_OK()

#...............................................................................
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF

# 需要导入模块: from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient import ResourceStatusClient [as 别名]
# 或者: from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient.ResourceStatusClient import selectStatusElement [as 别名]
class SummarizeLogsAgent( AgentModule ):
  """ SummarizeLogsAgent as extension of AgentModule.
  
  """

  def __init__( self, *args, **kwargs ):
    """ Constructor.
    
    """

    AgentModule.__init__( self, *args, **kwargs )
    
    self.rsClient = None


  def initialize( self ):
    """ Standard initialize.
    
    :return: S_OK
    
    """

    self.rsClient = ResourceStatusClient()
    return S_OK()


  def execute( self ):
    """ execute ( main method )
    
    The execute method runs over the three families of tables ( Site, Resource and
    Node ) performing identical operations. First, selects all logs for a given
    family ( and keeps track of which one is the last row ID ). It summarizes the
    logs and finally, deletes the logs from the database.
    
    :return: S_OK
    
    """

    # loop over the tables
    for element in ( 'Site', 'Resource', 'Node' ):

      self.log.info( 'Summarizing %s' % element )

      # get all logs to be summarized
      selectLogElements = self._summarizeLogs( element )
      if not selectLogElements[ 'OK' ]:
        self.log.error( selectLogElements[ 'Message' ] )
        continue
      
      lastID, logElements = selectLogElements[ 'Value' ]
      
      # logElements is a dictionary of key-value pairs as follows:
      # ( name, statusType ) : list( logs )
      for key, logs in logElements.iteritems():

        sumResult = self._registerLogs( element, key, logs )
        if not sumResult[ 'OK' ]:
          self.log.error( sumResult[ 'Message' ] )
          continue

      if lastID is not None:
        self.log.info( 'Deleting %sLog till ID %s' % ( element, lastID ) )
        deleteResult = self.rsClient.deleteStatusElement( element, 'Log', 
                                                        meta = { 'older' : ( 'ID', lastID ) } )
        if not deleteResult[ 'OK' ]:
          self.log.error( deleteResult[ 'Message' ] )
          continue

    return S_OK()


  #.............................................................................


  def _summarizeLogs( self, element ):
    """ given an element, selects all logs in table <element>Log.
    
    :Parameters:
      **element** - `string`
        name of the table family ( either Site, Resource and Node )
    
    :return: S_OK( lastID, listOfLogs ) / S_ERROR
    
    """
    
    selectResults = self.rsClient.selectStatusElement( element, 'Log' )
    
    if not selectResults[ 'OK' ]:
      return selectResults
  
    selectedItems = {}
    selectColumns = selectResults[ 'Columns' ]
    selectResults = selectResults[ 'Value' ]
    
    latestID = None
    if selectResults:
      latestID = dict( zip( selectColumns, selectResults[ -1 ] ) )[ 'ID' ]
    
    for selectResult in selectResults:
      
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient import ResourceStatusClient [as 别名]
# 或者: from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient.ResourceStatusClient import selectStatusElement [as 别名]
def setToken(user):
    """
    Function that gets the user token, sets the validity for it. Gets the elements
    in the database for a given name and statusType(s). Then updates the status
    of all them adding a reason and the token.
  """

    rssClient = ResourceStatusClient()

    # This is a little bit of a nonsense, and certainly needs to be improved.
    # To modify a list of elements, we have to do it one by one. However, the
    # modify method does not discover the StatusTypes ( which in this script is
    # an optional parameter ). So, we get them from the DB and iterate over them.
    elements = rssClient.selectStatusElement(
        switchDict["element"],
        "Status",
        name=switchDict["name"],
        statusType=switchDict["statusType"],
        meta={"columns": ["StatusType", "TokenOwner"]},
    )

    if not elements["OK"]:
        return elements
    elements = elements["Value"]

    # If there list is empty they do not exist on the DB !
    if not elements:
        subLogger.warn(
            "Nothing found for %s, %s, %s" % (switchDict["element"], switchDict["name"], switchDict["statusType"])
        )
        return S_OK()

    # If we want to release the token
    if switchDict["releaseToken"] != False:
        tokenExpiration = datetime.max
        newTokenOwner = "rs_svc"
    else:
        tokenExpiration = datetime.utcnow().replace(microsecond=0) + timedelta(days=1)
        newTokenOwner = user

    subLogger.info("New token : %s until %s" % (newTokenOwner, tokenExpiration))

    for statusType, tokenOwner in elements:

        # If a user different than the one issuing the command and RSS
        if tokenOwner != user and tokenOwner != "rs_svc":
            subLogger.info("%s(%s) belongs to the user: %s" % (switchDict["name"], statusType, tokenOwner))

        # does the job
        result = rssClient.modifyStatusElement(
            switchDict["element"],
            "Status",
            name=switchDict["name"],
            statusType=statusType,
            reason=switchDict["reason"],
            tokenOwner=newTokenOwner,
            tokenExpiration=tokenExpiration,
        )
        if not result["OK"]:
            return result

        if tokenOwner == newTokenOwner:
            msg = "(extended)"
        elif newTokenOwner == "rs_svc":
            msg = "(released)"
        else:
            msg = "(aquired from %s)" % tokenOwner

        subLogger.info("%s:%s %s" % (switchDict["name"], statusType, msg))
    return S_OK()

# 需要导入模块: from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient import ResourceStatusClient [as 别名]
# 或者: from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient.ResourceStatusClient import selectStatusElement [as 别名]
class Statistics( object ):
  """
  Statistics class that provides helpers to extract information from the database
  more easily.
  """

  def __init__( self ):
    """
    Constructor
    """

    self.rsClient = ResourceStatusClient()
    #self.rmClient = ResourceManagementClient()

  def getElementHistory( self, element, elementName, statusType,
                         oldAs = None, newAs = None ):
    """
    Returns the succession of statuses and the dates since they are effective. The
    values are comprised in the time interval [ oldAs, newAs ]. If not specified,
    all values up to the present are returned.

    It returns a list of tuples, of which the first element is the Status and the
    second one the time-stamp since it is effective. Note that the time-stamps will
    not necessarily match the time window.

    :Parameters:
      **element** - `str`
        element family ( either Site, Resource or Node )
      **elementName** - `str`
        element name
      **statusType** - `str`
        status type of the element <elementName> (e.g. 'all', 'ReadAccess',... )
      **oldAs** - [ None, `datetime` ]
        datetime with the start point for the time window. If not specified, it
        is used the oldest time in the history.
      **newAs** - [ None, `datetime` ]
        datetime with the end point for the time window. If not specified, it
        is used datetime.utcnow.

    :return: S_OK( [ (StatusA, datetimeA),(StatusB,datetimeB) ] ) | S_ERROR
    """

    # Checks we are not passing a silly element ( we only accept Site, Resource and Node )
    if not element in getValidElements():
      return S_ERROR( '"%s" is not a valid element' % element )

    # FIXME: read below
    # Gets all elements in history. If the history is long, this query is going to
    # be rather heavy...
    result = self.rsClient.selectStatusElement( element, 'History', name = elementName,
                                                statusType = statusType,
                                                meta = { 'columns' : [ 'Status', 'DateEffective' ] } )
    if not result[ 'OK' ]:
      return result
    result = result[ 'Value' ]

    if not result:
      return S_OK( [] )

    # To avoid making exceptions in the for-loop, we feed history with the first
    # item in the results
    history = [ result[ 0 ] ]

    # Sets defaults.
    # OldAs is as old as datetime.min if not defined.

    #oldAs = ( 1 and oldAs ) or history[ 0 ][ 1 ]
    oldAs = ( 1 and oldAs ) or datetime.datetime.min

    # NewAs is as new as as set or datetime.now
    newAs = ( 1 and newAs ) or datetime.datetime.utcnow()

    # Sanity check: no funny time windows
    if oldAs > newAs:
      return S_ERROR( "oldAs (%s) > newAs (%s)" % ( oldAs, newAs ) )

    # This avoids that the window finishes before having the first point in the
    # history.
    if history[ 0 ][ 1 ] > newAs:
      return S_OK( [] )

    # Iterate starting from the second element in the list. The elements in the
    # list are SORTED. Otherwise, the break statement would be a mess. And same
    # applies for the elif
    for historyElement in result[1:]:

      # If the point is newer than the superior limit of the window, we are done.
      if historyElement[ 1 ] > newAs:
        break
      # If the point is older than the window lower limit, we buffer it. We just
      # want the closest point to the lower limit.
      elif historyElement[ 1 ] <= oldAs:
        history = [ historyElement ]
      # Otherwise, we add it to the history
      else:
        history.append( historyElement )

    return S_OK( history )

  def getElementStatusAt( self, element, elementName, statusType, statusTime ):
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient import ResourceStatusClient [as 别名]
# 或者: from DIRAC.ResourceStatusSystem.Client.ResourceStatusClient.ResourceStatusClient import selectStatusElement [as 别名]
class Synchronizer(object):
    '''
    Every time there is a successful write on the CS, Synchronizer().sync() is
    executed. It updates the database with the values on the CS.

    '''

    def __init__(self, rStatus=None, rManagement=None, defaultStatus="Unknown"):

        # Warm up local CS
        CSHelpers.warmUp()

        if rStatus is None:
            self.rStatus = ResourceStatusClient()
        if rManagement is None:
            self.rManagement = ResourceManagementClient()
        self.defaultStatus = defaultStatus

        self.rssConfig = RssConfiguration()
        self.tokenOwner = "rs_svc"
        result = getProxyInfo()
        if result['OK']:
            self.tokenOwner = result['Value']['username']

    def sync(self, _eventName, _params):
        '''
        Main synchronizer method. It synchronizes the three types of elements: Sites,
        Resources and Nodes. Each _syncX method returns a dictionary with the additions
        and deletions.

        examples:
          >>> s.sync( None, None )
              S_OK()

        :Parameters:
          **_eventName** - any
            this parameter is ignored, but needed by caller function.
          **_params** - any
            this parameter is ignored, but needed by caller function.

        :return: S_OK
        '''

        syncSites = self._syncSites()
        if not syncSites['OK']:
            gLogger.error(syncSites['Message'])

        syncResources = self._syncResources()
        if not syncResources['OK']:
            gLogger.error(syncResources['Message'])

        syncNodes = self._syncNodes()
        if not syncNodes['OK']:
            gLogger.error(syncNodes['Message'])

        return S_OK()

    ## Protected methods #########################################################

    def _syncSites(self):
        '''
          Sync sites: compares CS with DB and does the necessary modifications.
        '''

        gLogger.info('-- Synchronizing sites --')

        # sites in CS
        res = CSHelpers.getSites()
        if not res['OK']:
            return res
        sitesCS = res['Value']

        gLogger.verbose('%s sites found in CS' % len(sitesCS))

        # sites in RSS
        result = self.rStatus.selectStatusElement('Site', 'Status',
                                                  meta={'columns': ['Name']})
        if not result['OK']:
            return result
        sitesDB = [siteDB[0] for siteDB in result['Value']]

        # Sites that are in DB but not (anymore) in CS
        toBeDeleted = list(set(sitesDB).difference(set(sitesCS)))
        gLogger.verbose('%s sites to be deleted' % len(toBeDeleted))

        # Delete sites
        for siteName in toBeDeleted:
            deleteQuery = self.rStatus._extermineStatusElement(
                'Site', siteName)
            gLogger.verbose('Deleting site %s' % siteName)
            if not deleteQuery['OK']:
                return deleteQuery

        # Sites that are in CS but not (anymore) in DB
        toBeAdded = list(set(sitesCS).difference(set(sitesDB)))
        gLogger.verbose('%s site entries to be added' % len(toBeAdded))

        for site in toBeAdded:
            query = self.rStatus.addIfNotThereStatusElement('Site', 'Status',
                                                            name=site,
#.........这里部分代码省略.........