Python GOCDBClient.GOCDBClient类代码示例

  def __init__( self, rsClient = None, rmClient = None ):

    self.GOCDBClient = GOCDBClient()
    self.rsClient = ResourceStatusClient()     if rsClient == None else rsClient
    self.rmClient = ResourceManagementClient() if rmClient == None else rmClient

    self.synclist = [ 'Sites', 'Resources', 'StorageElements', 'Services', 'RegistryUsers' ]

  def initialize( self ):

    # client to connect to GOCDB
    self.GOCDBClient = GOCDBClient()

    # API needed to update configuration stored by CS
    self.csAPI = CSAPI()
    return self.csAPI.initialize()

  def __init__( self, rsDBin = None ):

    self.rsDB = rsDBin

    if self.rsDB == None:
      from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
      self.rsDB = ResourceStatusDB()

    self.GOCDBClient = GOCDBClient()

class DTEveryResources_Command( Command ):

  def doCommand( self, resources = None ):
    """ 
    Returns downtimes information for all the resources in input.
        
    :params:
      :attr:`sites`: list of resource names (when not given, take every resource)
    
    :returns:
      {'ResourceName': {'SEVERITY': 'OUTAGE'|'AT_RISK', 
                    'StartDate': 'aDate', ...} ... }
    """

    if self.client is None:
      from DIRAC.Core.LCG.GOCDBClient import GOCDBClient
      self.client = GOCDBClient()

    if resources is None:
#      from DIRAC.Core.DISET.RPCClient import RPCClient
      RPC = RPCClient( "ResourceStatus/ResourceStatus" )
      resources = RPC.getResourcesList()
      if not resources['OK']:
        raise RSSException, where( self, self.doCommand ) + " " + resources['Message']
      else:
        resources = resources['Value']

    try:
      res = self.client.getStatus( 'Resource', resources, None, 120 )
    except:
      gLogger.exception( "Exception when calling GOCDBClient." )
      return {}

    if not res['OK']:
      raise RSSException, where( self, self.doCommand ) + " " + res['Message']
    else:
      res = res['Value']

    if res == None:
      return {}

    resToReturn = {}

    for dt_ID in res:
      dt = {}
      dt['ID'] = dt_ID
      dt['StartDate'] = res[dt_ID]['FORMATED_START_DATE']
      dt['EndDate'] = res[dt_ID]['FORMATED_END_DATE']
      dt['Severity'] = res[dt_ID]['SEVERITY']
      dt['Description'] = res[dt_ID]['DESCRIPTION'].replace( '\'', '' )
      dt['Link'] = res[dt_ID]['GOCDB_PORTAL_URL']
      resToReturn[dt_ID] = dt

    return resToReturn

  doCommand.__doc__ = Command.doCommand.__doc__ + doCommand.__doc__

  def initialize(self):
    """ Run at the agent initialization (normally every 500 cycles)
    """
    # client to connect to GOCDB
    self.GOCDBClient = GOCDBClient()
    self.dryRun = self.am_getOption('DryRun', self.dryRun)

    # API needed to update configuration stored by CS
    self.csAPI = CSAPI()
    return self.csAPI.initialize()

    def initialize(self, *args, **kwargs):
        """
        Initialize.

        Initialise method pulls in some extra configuration options
        These include:
        VOKeys            - List of VO identifiers
        """
        self.vokeys = self.am_getOption('VOKeys', ['GridPP'])
        self.removal_threshold = self.am_getOption('RemovalThreshold', 5)
        self.gocdb_client = GOCDBClient()
        return S_OK()

    def doCommand(self, resources=None):
        """ 
    Returns downtimes information for all the resources in input.
        
    :params:
      :attr:`sites`: list of resource names (when not given, take every resource)
    
    :returns:
      {'ResourceName': {'SEVERITY': 'OUTAGE'|'AT_RISK', 
                    'StartDate': 'aDate', ...} ... }
    """

        if self.client is None:
            from DIRAC.Core.LCG.GOCDBClient import GOCDBClient

            self.client = GOCDBClient()

        if resources is None:
            #      from DIRAC.Core.DISET.RPCClient import RPCClient
            RPC = RPCClient("ResourceStatus/ResourceStatus")
            resources = RPC.getResourcesList()
            if not resources["OK"]:
                raise RSSException, where(self, self.doCommand) + " " + resources["Message"]
            else:
                resources = resources["Value"]

        try:
            res = self.client.getStatus("Resource", resources, None, 120)
        except:
            gLogger.exception("Exception when calling GOCDBClient.")
            return {}

        if not res["OK"]:
            raise RSSException, where(self, self.doCommand) + " " + res["Message"]
        else:
            res = res["Value"]

        if res == None:
            return {}

        resToReturn = {}

        for dt_ID in res:
            dt = {}
            dt["ID"] = dt_ID
            dt["StartDate"] = res[dt_ID]["FORMATED_START_DATE"]
            dt["EndDate"] = res[dt_ID]["FORMATED_END_DATE"]
            dt["Severity"] = res[dt_ID]["SEVERITY"]
            dt["Description"] = res[dt_ID]["DESCRIPTION"].replace("'", "")
            dt["Link"] = res[dt_ID]["GOCDB_PORTAL_URL"]
            resToReturn[dt_ID] = dt

        return resToReturn

  def __init__( self, args = None, clients = None ):

    super( DowntimeCommand, self ).__init__( args, clients )

    if 'GOCDBClient' in self.apis:
      self.gClient = self.apis[ 'GOCDBClient' ]
    else:
      self.gClient = GOCDBClient()

    if 'ResourceManagementClient' in self.apis:
      self.rmClient = self.apis[ 'ResourceManagementClient' ]
    else:
      self.rmClient = ResourceManagementClient()

  def __init__(self, args=None, clients=None):

    super(GOCDBSyncCommand, self).__init__(args, clients)

    if 'GOCDBClient' in self.apis:
      self.gClient = self.apis['GOCDBClient']
    else:
      self.gClient = GOCDBClient()

    if 'ResourceManagementClient' in self.apis:
      self.rmClient = self.apis['ResourceManagementClient']
    else:
      self.rmClient = ResourceManagementClient()

    self.seenHostnames = set()

class Synchronizer:

#############################################################################

  def __init__( self, rsDBin = None, rmDBin = None ):

    self.rsDB = rsDBin
    self.rmDB = rmDBin

    if self.rsDB == None and self.rmDB == None:
      from DIRAC.ResourceStatusSystem.DB.ResourceStatusDB import ResourceStatusDB
      from DIRAC.ResourceStatusSystem.DB.ResourceManagementDB import ResourceManagementDB
      self.rsDB = ResourceStatusDB()
      self.rmDB = ResourceManagementDB()

    self.GOCDBClient = GOCDBClient()

#############################################################################

#  def sync(self, thingsToSync = None, fake_param = None):
  def sync( self, _a, _b ):
    """
    :params:
      :attr:`thingsToSync`: list of things to sync
    """

    thingsToSync = ['Utils', 'Sites', 'VOBOX', 'Resources', 'StorageElements', 'RegistryUsers']

    gLogger.info( "!!! Sync DB content with CS content for %s !!!" % ( ' '.join( x for x in thingsToSync ) ) )

    for thing in thingsToSync:
      getattr( self, '_sync' + thing )()

    return S_OK()

#############################################################################

  def _syncUtils( self ):
    """
    Sync DB content with what is in :mod:`DIRAC.ResourceStatusSystem.Utilities.Utils`
    """

    statusIn = self.rsDB.getStatusList()
    #delete status not more in Utils
    for stIn in statusIn:
      if stIn not in ValidStatus:
        self.rsDB.removeStatus( stIn )
    #Add new status
    for s in ValidStatus:
      if s not in statusIn:
        self.rsDB.addStatus( s )

    for g in ( 'Site', 'Service', 'Resource' ):
      typeIn = self.rsDB.getTypesList( g )
      if g == 'Site':
        typesList = ValidSiteType
      elif g == 'Service':
        typesList = ValidServiceType
      if g == 'Resource':
        typesList = ValidResourceType
      #delete types not more in Utils
      for tIn in typeIn:
        if tIn not in typesList:
          self.rsDB.removeType( g, tIn )
      #Add new types
      for t in typesList:
        if t not in typeIn:
          self.rsDB.addType( g, t )

#############################################################################

  def _syncSites( self ):
    """
    Sync DB content with sites that are in the CS
    """

    # sites in the DB now
    sitesIn = self.rsDB.getMonitoredsList( 'Site', paramsList = ['SiteName'] )
    sitesIn = [s[0] for s in sitesIn]

    # sites in CS now
    sitesList = getSites()['Value']

    try:
      sitesList.remove( 'LCG.Dummy.ch' )
    except ValueError:
      pass

    # remove sites from the DB not more in the CS
    for site in sitesIn:
      if site not in sitesList:
        self.rsDB.removeSite( site )

    # add to DB what is in CS now and wasn't before
    for site in sitesList:
      if site not in sitesIn:
        # DIRAC Tier
        tier = getSiteTier( site )['Value'][0]
        if tier == 0 or tier == '0':
          t = 'T0'
#.........这里部分代码省略.........

class DTEverySites_Command(Command):
    def doCommand(self, sites=None):
        """ 
    Returns downtimes information for all the sites in input.
        
    :params:
      :attr:`sites`: list of site names (when not given, take every site)
    
    :returns:
      {'SiteName': {'SEVERITY': 'OUTAGE'|'AT_RISK', 
                    'StartDate': 'aDate', ...} ... }
    """

        if self.client is None:
            from DIRAC.Core.LCG.GOCDBClient import GOCDBClient

            self.client = GOCDBClient()

        if sites is None:
            #      from DIRAC.Core.DISET.RPCClient import RPCClient
            RPC = RPCClient("ResourceStatus/ResourceStatus")
            GOC_sites = RPC.getGridSitesList()
            if not GOC_sites["OK"]:
                raise RSSException, where(self, self.doCommand) + " " + sites["Message"]
            else:
                GOC_sites = GOC_sites["Value"]
        else:
            GOC_sites = [getGOCSiteName(x)["Value"] for x in sites]

        try:
            res = self.client.getStatus("Site", GOC_sites, None, 120)
        except:
            gLogger.exception("Exception when calling GOCDBClient.")
            return {}

        if not res["OK"]:
            raise RSSException, where(self, self.doCommand) + " " + res["Message"]
        else:
            res = res["Value"]

        if res == None:
            return {}

        resToReturn = {}

        for dt_ID in res:
            try:
                dt = {}
                dt["ID"] = dt_ID
                dt["StartDate"] = res[dt_ID]["FORMATED_START_DATE"]
                dt["EndDate"] = res[dt_ID]["FORMATED_END_DATE"]
                dt["Severity"] = res[dt_ID]["SEVERITY"]
                dt["Description"] = res[dt_ID]["DESCRIPTION"].replace("'", "")
                dt["Link"] = res[dt_ID]["GOCDB_PORTAL_URL"]
                DIRACnames = getDIRACSiteName(res[dt_ID]["SITENAME"])
                if not DIRACnames["OK"]:
                    raise RSSException, DIRACnames["Message"]
                DIRACnames = DIRACnames["Value"]
                for DIRACname in DIRACnames:
                    resToReturn[dt_ID.split()[0] + " " + DIRACname] = dt
            except KeyError:
                continue

        return resToReturn

    doCommand.__doc__ = Command.doCommand.__doc__ + doCommand.__doc__

class GOCDBSyncCommand(Command):

  def __init__(self, args=None, clients=None):

    super(GOCDBSyncCommand, self).__init__(args, clients)

    if 'GOCDBClient' in self.apis:
      self.gClient = self.apis['GOCDBClient']
    else:
      self.gClient = GOCDBClient()

    if 'ResourceManagementClient' in self.apis:
      self.rmClient = self.apis['ResourceManagementClient']
    else:
      self.rmClient = ResourceManagementClient()

    self.seenHostnames = set()

  def doNew(self, masterParams=None):
    """
    Gets the downtime IDs and dates of a given hostname from the local database and compares the results
    with the remote database of GOCDB. If the downtime dates have been changed it updates the local database.

    :param: `masterParams` - string
    :return: S_OK / S_ERROR
    """

    if masterParams:
      hostname = masterParams
    else:
      return S_ERROR(errno.EINVAL, 'masterParams is not provided')

    result = self.rmClient.selectDowntimeCache(name=hostname)
    if not result['OK']:
      return result

    for downtimes in result['Value']:

      localDBdict = {'DowntimeID': downtimes[3],
                     'FORMATED_START_DATE': downtimes[6].strftime('%Y-%m-%d %H:%M'),
                     'FORMATED_END_DATE': downtimes[7].strftime('%Y-%m-%d %H:%M')}

      response = self.gClient.getHostnameDowntime(hostname, ongoing=True)

      if not response['OK']:
        return response

      doc = minidom.parseString(response['Value'])
      downtimeElements = doc.getElementsByTagName("DOWNTIME")

      for dtElement in downtimeElements:
        GOCDBdict = _parseSingleElement(dtElement, ['PRIMARY_KEY', 'ENDPOINT',
                                                    'FORMATED_START_DATE', 'FORMATED_END_DATE'])

        localDowntimeID = localDBdict['DowntimeID']
        GOCDBDowntimeID = GOCDBdict['PRIMARY_KEY'] + ' ' + GOCDBdict['ENDPOINT']

        if localDowntimeID == GOCDBDowntimeID:

          if localDBdict['FORMATED_START_DATE'] != GOCDBdict['FORMATED_START_DATE']:
            result = self.rmClient.addOrModifyDowntimeCache(downtimeID=localDBdict['DowntimeID'],
                                                            startDate=GOCDBdict['FORMATED_START_DATE'])
            gLogger.verbose("The start date of %s has been changed!" % downtimes[3])

            if not result['OK']:
              return result

          if localDBdict['FORMATED_END_DATE'] != GOCDBdict['FORMATED_END_DATE']:
            result = self.rmClient.addOrModifyDowntimeCache(downtimeID=localDBdict['DowntimeID'],
                                                            endDate=GOCDBdict['FORMATED_END_DATE'])
            gLogger.verbose("The end date of %s has been changed!" % downtimes[3])

            if not result['OK']:
              return result

    return S_OK()

  def doCache(self):
    return S_OK()

  def doMaster(self):
    """
    This method calls the doNew method for each hostname that exists
    in the DowntimeCache table of the local database.

    :return: S_OK / S_ERROR
    """

    # Query DB for all downtimes
    result = self.rmClient.selectDowntimeCache()
    if not result['OK']:
      return result

    for data in result['Value']:

      # If already processed don't do it again
      if data[0] in self.seenHostnames:
        continue

      # data[0] contains the hostname
#.........这里部分代码省略.........

class AutoVac2CSAgent(AgentModule):
    """
    AutoBdii2CSAgent.

    Automatically updates the CS automatically for CEs and SEs.
    """

    max_cputime_map = {'VAC': 400000, 'CLOUD': 24000000}
    cc_regex = re.compile(r'\.([a-zA-Z]{2})$')
    cc_mappings = {'.gov': 'us',
                   '.edu': 'us',
                   'efda.org': 'uk',
                   'atlas-swt2.org': 'us'}

    def initialize(self, *args, **kwargs):
        """
        Initialize.

        Initialise method pulls in some extra configuration options
        These include:
        VOKeys            - List of VO identifiers
        """
        self.vokeys = self.am_getOption('VOKeys', ['GridPP'])
        self.removal_threshold = self.am_getOption('RemovalThreshold', 5)
        self.gocdb_client = GOCDBClient()
        return S_OK()

    def execute(self):
        """General agent execution method."""
        cfg_system = ConfigurationSystem()
        cfg_system.initialize()

        # Get VAC sites.
        # ##############
        result = self.gocdb_client.getServiceEndpointInfo('service_type', "uk.ac.gridpp.vac")
        if not result['OK']:
            self.log.error("Problem getting GOCDB VAC information")
            return result

        try:
            self.process_gocdb_results(result['Value'], 'VAC', cfg_system)
        except:
            self.log.exception("Problem processing GOCDB VAC information")
            return S_ERROR("Problem processing GOCDB VAC information")

        # Get CLOUD (vcycle) sites.
        # #########################
        result = self.gocdb_client.getServiceEndpointInfo('service_type', "uk.ac.gridpp.vcycle")
        if not result['OK']:
            self.log.error("Problem getting GOCDB CLOUD (vcycle) information")
            return result

        try:
            self.process_gocdb_results(result['Value'], 'CLOUD', cfg_system)
        except:
            self.log.exception("Problem processing GOCDB CLOUD (vcycle) information")
            return S_ERROR("Problem processing GOCDB CLOUD (vcycle) information")

        cfg_system.commit()

        # Remove old hosts/sites
        # ######################
        try:
            self.remove_old(self.removal_threshold)
        except:
            self.log.exception("Problem removing old hosts/sites.")
            return S_ERROR("Problem processing GOCDB CLOUD (vcycle) information")

        return S_OK()

    def process_gocdb_results(self, services, site_path_prefix, cfg_system, country_default='xx'):
        """
        Process GOCDB results.

        Args:
            services (list): List of services returned from GOCDB query.
            site_path_prefix (str): The CS path prefix (VAC or CLOUD) for the type of
                                    service that we are processing.
            cfg_system (ConfigurationSystem): A ConfigurationSystem instance used to update
                                              the CS.
        """
        for service in services:
            # Resources
            sitename = service.get('SITENAME')
            hostname = service.get('HOSTNAME')
            country_code = AutoVac2CSAgent.extract_cc(hostname) or country_default
            if sitename is None or hostname is None:
                self.log.warn("Missing sitename or hostname for service:\n%s" % pformat(service))
                continue

            site_path = cfgPath(SITES_BASE, site_path_prefix, "%s.%s.%s"
                                % (site_path_prefix, sitename, country_code))
            ce_path = cfgPath(site_path, 'CEs', hostname)
            queue_path = cfgPath(ce_path, 'Queues', 'default')
            cfg_system.add(site_path, 'Name', sitename)
            cfg_system.append_unique(site_path, 'CE', hostname)
            cfg_system.add(ce_path, 'CEType', site_path_prefix.capitalize())
            cfg_system.add(ce_path, 'Architecture', 'x86_64')
            cfg_system.add(ce_path, 'OS', 'EL6')
            cfg_system.add(ce_path, 'LastSeen', date.today().strftime('%d/%m/%Y'))
#.........这里部分代码省略.........

class DowntimeCommand( Command ):
  '''
    Downtime "master" Command.
  '''

  def __init__( self, args = None, clients = None ):

    super( DowntimeCommand, self ).__init__( args, clients )

    if 'GOCDBClient' in self.apis:
      self.gClient = self.apis[ 'GOCDBClient' ]
    else:
      self.gClient = GOCDBClient()

    if 'ResourceManagementClient' in self.apis:
      self.rmClient = self.apis[ 'ResourceManagementClient' ]
    else:
      self.rmClient = ResourceManagementClient()

  def _storeCommand( self, result ):
    '''
      Stores the results of doNew method on the database.
    '''

    for dt in result:
      resQuery = self.rmClient.addOrModifyDowntimeCache( 
                               downtimeID = dt[ 'DowntimeID' ],
                               element = dt[ 'Element' ],
                               name = dt[ 'Name' ],
                               startDate = dt[ 'StartDate' ],
                               endDate = dt[ 'EndDate' ],
                               severity = dt[ 'Severity' ],
                               description = dt[ 'Description' ],
                               link = dt[ 'Link' ],
                               gocdbServiceType = dt[ 'GOCDBServiceType' ] )
      if not resQuery[ 'OK' ]:
        return resQuery
    return S_OK()

  def _prepareCommand( self ):
    '''
      DowntimeCommand requires four arguments:
      - name : <str>
      - element : Site / Resource
      - elementType: <str>

      If the elements are Site(s), we need to get their GOCDB names. They may
      not have, so we ignore them if they do not have.
    '''

    if 'name' not in self.args:
      return S_ERROR( '"name" not found in self.args' )
    elementName = self.args[ 'name' ]

    if 'element' not in self.args:
      return S_ERROR( '"element" not found in self.args' )
    element = self.args[ 'element' ]

    if 'elementType' not in self.args:
      return S_ERROR( '"elementType" not found in self.args' )
    elementType = self.args[ 'elementType' ]

    if not element in [ 'Site', 'Resource' ]:
      return S_ERROR( 'element is not Site nor Resource' )

    hours = None
    if 'hours' in self.args:
      hours = self.args[ 'hours' ]

    gocdbServiceType = None

    # Transform DIRAC site names into GOCDB topics
    if element == 'Site':

      gocSite = getGOCSiteName( elementName )
      if not gocSite[ 'OK' ]:
        return gocSite
      elementName = gocSite[ 'Value' ]

    # The DIRAC se names mean nothing on the grid, but their hosts do mean.
    elif elementType == 'StorageElement':
      # We need to distinguish if it's tape or disk
      if getStorageElementOptions( elementName )['Value']['TapeSE']:
        gocdbServiceType = "srm"
      elif getStorageElementOptions( elementName )['Value']['DiskSE']:
        gocdbServiceType = "srm.nearline"

      seHost = CSHelpers.getSEHost( elementName )
      if not seHost:
        return S_ERROR( 'No seHost for %s' % elementName )
      elementName = seHost

    return S_OK( ( element, elementName, hours, gocdbServiceType ) )

  def doNew( self, masterParams = None ):
    '''
      Gets the parameters to run, either from the master method or from its
      own arguments.

      For every elementName, unless it is given a list, in which case it contacts
#.........这里部分代码省略.........

class DowntimeCommand( Command ):
  '''
    Downtime "master" Command.    
  '''

  def __init__( self, args = None, clients = None ):
    
    super( DowntimeCommand, self ).__init__( args, clients )

    if 'GOCDBClient' in self.apis:
      self.gClient = self.apis[ 'GOCDBClient' ]
    else:
      self.gClient = GOCDBClient() 

    if 'ResourceManagementClient' in self.apis:
      self.rmClient = self.apis[ 'ResourceManagementClient' ]
    else:
      self.rmClient = ResourceManagementClient()
      
  def _storeCommand( self, result ):
    '''
      Stores the results of doNew method on the database.
    '''

    for dt in result:
      
      resQuery = self.rmClient.addOrModifyDowntimeCache( dt[ 'DowntimeID' ], 
                                                         dt[ 'Element' ], 
                                                         dt[ 'Name' ], 
                                                         dt[ 'StartDate' ], 
                                                         dt[ 'EndDate' ], 
                                                         dt[ 'Severity' ], 
                                                         dt[ 'Description' ], 
                                                         dt[ 'Link' ] )  
      if not resQuery[ 'OK' ]:
        return resQuery
    return S_OK()
  
  def _prepareCommand( self ):
    '''
      DowntimeCommand requires three arguments:
      - name : <str>
      - element : Site / Resource
      - elementType: <str>  
      
      If the elements are Site(s), we need to get their GOCDB names. They may
      not have, so we ignore them if they do not have.
    '''
    
    if 'name' not in self.args:
      return S_ERROR( '"name" not found in self.args' )
    elementName = self.args[ 'name' ]      
    
    if 'element' not in self.args:
      return S_ERROR( '"element" not found in self.args' )
    element = self.args[ 'element' ]
    
    if 'elementType' not in self.args:
      return S_ERROR( '"elementType" not found in self.args' )
    elementType = self.args[ 'elementType' ]
    
    if not element in [ 'Site', 'Resource' ]:
      return S_ERROR( 'element is not Site nor Resource' )   

    hours = None
    if 'hours' in self.args:
      hours = self.args[ 'hours' ]
    
    # Transform DIRAC site names into GOCDB topics
    if element == 'Site':

      gocSite = getGOCSiteName( elementName )
      if not gocSite[ 'OK' ]:
        return gocSite
      elementName = gocSite[ 'Value' ]
          
    # The DIRAC se names mean nothing on the grid, but their hosts do mean.
    elif elementType == 'StorageElement':
      
      seHost = CSHelpers.getSEHost( elementName )
      if not seHost:
        return S_ERROR( 'No seHost for %s' % elementName )
      elementName = seHost
             
    return S_OK( ( element, elementName, hours ) )

  def doNew( self, masterParams = None ):
    '''
      Gets the parameters to run, either from the master method or from its
      own arguments.
      
      For every elementName, unless it is given a list, in which case it contacts 
      the gocdb client. The server is not very stable, so in case of failure tries
      a second time.
      
      If there are downtimes, are recorded and then returned.
    '''
    
    if masterParams is not None:
      element, elementNames = masterParams
#.........这里部分代码省略.........