Python JobDB.getSummarySnapshot方法代码示例

# 需要导入模块: from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB [as 别名]
# 或者: from DIRAC.WorkloadManagementSystem.DB.JobDB.JobDB import getSummarySnapshot [as 别名]
class StatesAccountingAgent( AgentModule ):
  """
      The specific agents must provide the following methods:
      - initialize() for initial settings
      - beginExecution()
      - execute() - the main method called in the agent cycle
      - endExecution()
      - finalize() - the graceful exit of the method, this one is usually used
                 for the agent restart
  """

  __summaryKeyFieldsMapping = [ 'Status',
                                'Site',
                                'User',
                                'UserGroup',
                                'JobGroup',
                                'JobType',
                              ]
  __summaryDefinedFields = [ ( 'ApplicationStatus', 'unset' ), ( 'MinorStatus', 'unset' ) ]
  __summaryValueFieldsMapping = [ 'Jobs',
                                  'Reschedules',
                                ]
  __renameFieldsMapping = { 'JobType' : 'JobSplitType' }


  def initialize( self ):
    """ Standard constructor
    """
    self.dsClients = {}
    self.jobDB = JobDB()

    self.reportPeriod = 850
    self.am_setOption( "PollingTime", self.reportPeriod )
    self.__jobDBFields = []
    for field in self.__summaryKeyFieldsMapping:
      if field == 'User':
        field = 'Owner'
      elif field == 'UserGroup':
        field = 'OwnerGroup'
      self.__jobDBFields.append( field )
    return S_OK()

  def execute( self ):
    """ Main execution method
    """
    result = gConfig.getSections( "/DIRAC/Setups" )
    if not result[ 'OK' ]:
      return result
    validSetups = result[ 'Value' ]
    gLogger.info( "Valid setups for this cycle are %s" % ", ".join( validSetups ) )
    #Get the WMS Snapshot!
    result = self.jobDB.getSummarySnapshot( self.__jobDBFields )
    now = Time.dateTime()
    if not result[ 'OK' ]:
      gLogger.error( "Can't the the jobdb summary", result[ 'Message' ] )
    else:
      values = result[ 'Value' ][1]
      for record in values:
        recordSetup = record[0]
        if recordSetup not in validSetups:
          gLogger.error( "Setup %s is not valid" % recordSetup )
          continue
        if recordSetup not in self.dsClients:
          gLogger.info( "Creating DataStore client for %s" % recordSetup )
          self.dsClients[ recordSetup ] = DataStoreClient( setup = recordSetup, retryGraceTime = 900 )
        record = record[1:]
        rD = {}
        for fV in self.__summaryDefinedFields:
          rD[ fV[0] ] = fV[1]
        for iP in range( len( self.__summaryKeyFieldsMapping ) ):
          fieldName = self.__summaryKeyFieldsMapping[iP]
          rD[ self.__renameFieldsMapping.get( fieldName, fieldName ) ] = record[iP]
        record = record[ len( self.__summaryKeyFieldsMapping ): ]
        for iP in range( len( self.__summaryValueFieldsMapping ) ):
          rD[ self.__summaryValueFieldsMapping[iP] ] = int( record[iP] )
        acWMS = WMSHistory()
        acWMS.setStartTime( now )
        acWMS.setEndTime( now )
        acWMS.setValuesFromDict( rD )
        retVal = acWMS.checkValues()
        if not retVal[ 'OK' ]:
          gLogger.error( "Invalid accounting record ", "%s -> %s" % ( retVal[ 'Message' ], rD ) )
        else:
          self.dsClients[ recordSetup ].addRegister( acWMS )
      for setup in self.dsClients:
        gLogger.info( "Sending records for setup %s" % setup )
        result = self.dsClients[ setup ].commit()
        if not result[ 'OK' ]:
          gLogger.error( "Couldn't commit wms history for setup %s" % setup, result[ 'Message' ] )
        else:
          gLogger.info( "Sent %s records for setup %s" % ( result[ 'Value' ], setup ) )
    return S_OK()

# 需要导入模块: from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB [as 别名]
# 或者: from DIRAC.WorkloadManagementSystem.DB.JobDB.JobDB import getSummarySnapshot [as 别名]
class StatesMonitoringAgent( AgentModule ):
  """
      The specific agents must provide the following methods:
      - initialize() for initial settings
      - beginExecution()
      - execute() - the main method called in the agent cycle
      - endExecution()
      - finalize() - the graceful exit of the method, this one is usually used
                 for the agent restart
  """

  __summaryKeyFieldsMapping = [ 'Status',
                                'Site',
                                'User',
                                'UserGroup',
                                'JobGroup',
                                'JobType',
                              ]
  __summaryDefinedFields = [ ( 'ApplicationStatus', 'unset' ), ( 'MinorStatus', 'unset' ) ]
  __summaryValueFieldsMapping = [ 'value',
                                  'Reschedules',
                                ]
  __renameFieldsMapping = { 'JobType' : 'JobSplitType' }


  def initialize( self ):
    """ Standard constructor
    """
    
    self.jobDB = JobDB()

    self.am_setOption( "PollingTime", 120 )
    self.__jobDBFields = []
    for field in self.__summaryKeyFieldsMapping:
      if field == 'User':
        field = 'Owner'
      elif field == 'UserGroup':
        field = 'OwnerGroup'
      self.__jobDBFields.append( field )
    
    result = gConfig.getOption( "/Systems/RabbitMQ/User" )
    if not result['OK']:
      raise RuntimeError( 'Failed to get the configuration parameters: User' )
    user = result['Value']
    
    result = gConfig.getOption( "/Systems/RabbitMQ/Password" )
    if not result['OK']:
      raise RuntimeError( 'Failed to get the configuration parameters: Password' )
    password = result['Value']
    
    result = gConfig.getOption( "/Systems/RabbitMQ/Host" )
    if not result['OK']:
      raise RuntimeError( 'Failed to get the configuration parameters: Host' )
    self.host = result['Value']
    
    self.credentials = pika.PlainCredentials( user, password )

    self.connection = pika.BlockingConnection( pika.ConnectionParameters( host = self.host, credentials = self.credentials ) )
    self.channel = self.connection.channel()

    self.channel.exchange_declare( exchange = 'mdatabase',
                         exchange_type = 'fanout' )

    return S_OK()

  def sendRecords( self, data ):
    try:
      self.channel.basic_publish( exchange = 'mdatabase',
                        routing_key = '',
                        body = data,
                        properties = pika.BasicProperties( 
                        delivery_mode = 2,  # make message persistent
                      ) )
    except Exception as e:
      gLogger.error( "Connection closed:" + str( e ) )
      self.connection = pika.BlockingConnection( pika.ConnectionParameters( host = self.host, credentials = self.credentials ) )
      self.channel = self.connection.channel()
      self.channel.exchange_declare( exchange = 'mdatabase',
                         exchange_type = 'fanout' )
      gLogger.info( "Resend records" )
      self.sendRecords( data )
      
  def finalize( self ):
    self.connection.close()
    
  def execute( self ):
    """ Main execution method
    """
    result = gConfig.getSections( "/DIRAC/Setups" )
    if not result[ 'OK' ]:
      return result
    validSetups = result[ 'Value' ]
    gLogger.info( "Valid setups for this cycle are %s" % ", ".join( validSetups ) )
    # Get the WMS Snapshot!
    result = self.jobDB.getSummarySnapshot( self.__jobDBFields )
    now = Time.dateTime()
    if not result[ 'OK' ]:
      gLogger.error( "Can't the the jobdb summary", result[ 'Message' ] )
    else:
      values = result[ 'Value' ][1]
#.........这里部分代码省略.........