Python GOCDBClient.getServiceEndpointInfo方法代碼示例

# 需要導入模塊: from DIRAC.Core.LCG.GOCDBClient import GOCDBClient [as 別名]
# 或者: from DIRAC.Core.LCG.GOCDBClient.GOCDBClient import getServiceEndpointInfo [as 別名]

#.........這裏部分代碼省略.........
        else:
          gt = getGOCTier( DIRACSitesOfGridSites )

        #Utils.protect2(self.rsClient.addOrModifyGridSite, gridSiteName, gt)
        res = self.rsClient.addOrModifyGridSite( gridSiteName, gt )
        if not res[ 'OK' ]:
          gLogger.error( res[ 'Message' ] )
          return res
        
        #Utils.protect2(self.rsClient.addOrModifySite, site, tier, gridSiteName )
        res = self.rsClient.addOrModifySite( site, tier, gridSiteName )
        if not res[ 'OK' ]:
          gLogger.error( res[ 'Message' ] )
          return res

      elif siteType == "DIRAC":
        #Utils.protect2(self.rsClient.addOrModifySite, site, tier, "NULL" )
        res = self.rsClient.addOrModifySite( site, tier, "NULL" )
        if not res[ 'OK' ]:
          gLogger.error( res[ 'Message' ] )
          return res

################################################################################
# _syncResources HELPER functions

  def __updateService(self, site, type_):
    service = type_ + '@' + site
    #Utils.protect2(self.rsClient.addOrModifyService, service, type_, site )
    res = self.rsClient.addOrModifyService( service, type_, site )
    if not res[ 'OK' ]:
      gLogger.error( res[ 'Message' ] )
      return res

  def __getServiceEndpointInfo(self, node):
    #res = Utils.unpack( self.GOCDBClient.getServiceEndpointInfo( 'hostname', node ) )
    res = self.GOCDBClient.getServiceEndpointInfo( 'hostname', node )
    if res['OK']:
      res = res[ 'Value' ]
    else:
      gLogger.warn( 'Error getting hostname info for %s' % node )
      return []
        
    if res == []:
      #res = Utils.unpack( self.GOCDBClient.getServiceEndpointInfo('hostname', Utils.canonicalURL(node)) )
      url = Utils.canonicalURL(node)
      res = self.GOCDBClient.getServiceEndpointInfo('hostname', url )
      if res['OK']:
        res = res[ 'Value' ]
      else:
        gLogger.warn( 'Error getting canonical hostname info for %s' % node )
        res = []
      
    return res

  def __syncNode(self, NodeInCS, resourcesInDB, resourceType, serviceType, site = "NULL"):

    nodesToUpdate = NodeInCS - resourcesInDB
    if len(nodesToUpdate) > 0:
      gLogger.debug(str(NodeInCS))
      gLogger.debug(str(nodesToUpdate))

    # Update Service table
    siteInGOCDB = [self.__getServiceEndpointInfo(node) for node in nodesToUpdate]
    siteInGOCDB = Utils.list_sanitize(siteInGOCDB)
    #sites = [Utils.unpack(getDIRACSiteName(s[0]['SITENAME'])) for s in siteInGOCDB]
    

# 需要導入模塊: from DIRAC.Core.LCG.GOCDBClient import GOCDBClient [as 別名]
# 或者: from DIRAC.Core.LCG.GOCDBClient.GOCDBClient import getServiceEndpointInfo [as 別名]
class GOCDB2CSAgent (AgentModule):
  """ Class to retrieve information about service endpoints
      from GOCDB and update configuration stored by CS
  """

  def __init__(self, *args, **kwargs):
    """ c'tor
    """
    super(GOCDB2CSAgent, self).__init__(*args, **kwargs)
    self.GOCDBClient = None
    self.csAPI = None
    self.dryRun = False

  def initialize(self):
    """ Run at the agent initialization (normally every 500 cycles)
    """
    # client to connect to GOCDB
    self.GOCDBClient = GOCDBClient()
    self.dryRun = self.am_getOption('DryRun', self.dryRun)

    # API needed to update configuration stored by CS
    self.csAPI = CSAPI()
    return self.csAPI.initialize()

  def execute(self):
    """
    Execute GOCDB queries according to the function map
    and user request (options in configuration).
    """

    # __functionMap is at the end of the class definition
    for option, functionCall in GOCDB2CSAgent.__functionMap.iteritems():
      optionValue = self.am_getOption(option, True)
      if optionValue:
        result = functionCall(self)
        if not result['OK']:
          self.log.error("%s() failed with message: %s" % (functionCall.__name__, result['Message']))
        else:
          self.log.info("Successfully executed %s" % functionCall.__name__)

    return S_OK()

  def updatePerfSONARConfiguration(self):
    """
    Get current status of perfSONAR endpoints from GOCDB
    and update CS configuration accordingly.
    """
    log = self.log.getSubLogger('updatePerfSONAREndpoints')
    log.debug('Begin function ...')

    # get endpoints
    result = self.__getPerfSONAREndpoints()
    if not result['OK']:
      log.error("__getPerfSONAREndpoints() failed with message: %s" % result['Message'])
      return S_ERROR('Unable to fetch perfSONAR endpoints from GOCDB.')
    endpointList = result['Value']

    # add DIRAC site name
    result = self.__addDIRACSiteName(endpointList)
    if not result['OK']:
      log.error("__addDIRACSiteName() failed with message: %s" % result['Message'])
      return S_ERROR('Unable to extend the list with DIRAC site names.')
    extendedEndpointList = result['Value']

    # prepare dictionary with new configuration
    result = self.__preparePerfSONARConfiguration(extendedEndpointList)
    if not result['OK']:
      log.error("__preparePerfSONARConfiguration() failed with message: %s" % result['Message'])
      return S_ERROR('Unable to prepare a new perfSONAR configuration.')
    finalConfiguration = result['Value']

    # update configuration according to the final status of endpoints
    self.__updateConfiguration(finalConfiguration)
    log.debug("Configuration updated succesfully")

    log.debug('End function.')
    return S_OK()

  def __getPerfSONAREndpoints(self):
    """
    Retrieve perfSONAR endpoint information directly from GOCDB.

    :return: List of perfSONAR endpoints (dictionaries) as stored by GOCDB.
    """

    log = self.log.getSubLogger('__getPerfSONAREndpoints')
    log.debug('Begin function ...')

    # get perfSONAR endpoints (latency and bandwidth) form GOCDB
    endpointList = []
    for endpointType in ['Latency', 'Bandwidth']:
      result = self.GOCDBClient.getServiceEndpointInfo('service_type', 'net.perfSONAR.%s' % endpointType)

      if not result['OK']:
        log.error("getServiceEndpointInfo() failed with message: %s" % result['Message'])
        return S_ERROR('Could not fetch %s endpoints from GOCDB' % endpointType.lower())

      log.debug('Number of %s endpoints: %s' % (endpointType.lower(), len(result['Value'])))
      endpointList.extend(result['Value'])

#.........這裏部分代碼省略.........

# 需要導入模塊: from DIRAC.Core.LCG.GOCDBClient import GOCDBClient [as 別名]
# 或者: from DIRAC.Core.LCG.GOCDBClient.GOCDBClient import getServiceEndpointInfo [as 別名]

#.........這裏部分代碼省略.........
      fts = getFTSEndpoint( site )['Value']
      if fts is None or fts == []:
        continue
      fts = fts[0]
      if fts not in FTSNodeList:
        FTSNodeList.append( fts )

    # VOMS Nodes in CS now
    VOMSNodeList = getVOMSEndpoints()['Value']


    # complete list of resources in CS now
    resourcesList = CEList + SENodeList + LFCNodeList_L + LFCNodeList_C + FTSNodeList + VOMSNodeList

    # list of services in CS now (to be done)
    servicesList = []

    #remove resources no more in the CS
    for res in resourcesIn:
      if res not in resourcesList:
        self.rsDB.removeResource( res )
        self.rsDB.removeStorageElement( resourceName = res )


    # add to DB what is in CS now and wasn't before

    # CEs
    for site in siteCE.keys():
      if site == 'LCG.Dummy.ch':
        continue
      for ce in siteCE[site]:
        if ce is None:
          continue
        siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', ce )
        if not siteInGOCDB['OK']:
          raise RSSException, siteInGOCDB['Message']
        if siteInGOCDB['Value'] == []:
          trueName = socket.gethostbyname_ex( ce )[0]
          siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName )
        try:
          siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
        except IndexError:
          continue
        serviceType = 'Computing'
        service = serviceType + '@' + site

        if service not in servicesList:
          servicesList.append( service )
        if service not in servicesIn:
          self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init',
                                       datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
                                       datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
          servicesIn.append( service )

        if ce not in resourcesIn:
          CEType = getCEType( site, ce )['Value']
          ceType = 'CE'
          if CEType == 'CREAM':
            ceType = 'CREAMCE'
          self.rsDB.addOrModifyResource( ce, ceType, serviceType, site, siteInGOCDB, 'Active', 'init',
                                        datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
                                        datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
          resourcesIn.append( ce )

    # SRMs
    for srm in SENodeList:

# 需要導入模塊: from DIRAC.Core.LCG.GOCDBClient import GOCDBClient [as 別名]
# 或者: from DIRAC.Core.LCG.GOCDBClient.GOCDBClient import getServiceEndpointInfo [as 別名]
class AutoVac2CSAgent(AgentModule):
    """
    AutoBdii2CSAgent.

    Automatically updates the CS automatically for CEs and SEs.
    """

    max_cputime_map = {'VAC': 400000, 'CLOUD': 24000000}
    cc_regex = re.compile(r'\.([a-zA-Z]{2})$')
    cc_mappings = {'.gov': 'us',
                   '.edu': 'us',
                   'efda.org': 'uk',
                   'atlas-swt2.org': 'us'}

    def initialize(self, *args, **kwargs):
        """
        Initialize.

        Initialise method pulls in some extra configuration options
        These include:
        VOKeys            - List of VO identifiers
        """
        self.vokeys = self.am_getOption('VOKeys', ['GridPP'])
        self.removal_threshold = self.am_getOption('RemovalThreshold', 5)
        self.gocdb_client = GOCDBClient()
        return S_OK()

    def execute(self):
        """General agent execution method."""
        cfg_system = ConfigurationSystem()
        cfg_system.initialize()

        # Get VAC sites.
        # ##############
        result = self.gocdb_client.getServiceEndpointInfo('service_type', "uk.ac.gridpp.vac")
        if not result['OK']:
            self.log.error("Problem getting GOCDB VAC information")
            return result

        try:
            self.process_gocdb_results(result['Value'], 'VAC', cfg_system)
        except:
            self.log.exception("Problem processing GOCDB VAC information")
            return S_ERROR("Problem processing GOCDB VAC information")

        # Get CLOUD (vcycle) sites.
        # #########################
        result = self.gocdb_client.getServiceEndpointInfo('service_type', "uk.ac.gridpp.vcycle")
        if not result['OK']:
            self.log.error("Problem getting GOCDB CLOUD (vcycle) information")
            return result

        try:
            self.process_gocdb_results(result['Value'], 'CLOUD', cfg_system)
        except:
            self.log.exception("Problem processing GOCDB CLOUD (vcycle) information")
            return S_ERROR("Problem processing GOCDB CLOUD (vcycle) information")

        cfg_system.commit()

        # Remove old hosts/sites
        # ######################
        try:
            self.remove_old(self.removal_threshold)
        except:
            self.log.exception("Problem removing old hosts/sites.")
            return S_ERROR("Problem processing GOCDB CLOUD (vcycle) information")

        return S_OK()

    def process_gocdb_results(self, services, site_path_prefix, cfg_system, country_default='xx'):
        """
        Process GOCDB results.

        Args:
            services (list): List of services returned from GOCDB query.
            site_path_prefix (str): The CS path prefix (VAC or CLOUD) for the type of
                                    service that we are processing.
            cfg_system (ConfigurationSystem): A ConfigurationSystem instance used to update
                                              the CS.
        """
        for service in services:
            # Resources
            sitename = service.get('SITENAME')
            hostname = service.get('HOSTNAME')
            country_code = AutoVac2CSAgent.extract_cc(hostname) or country_default
            if sitename is None or hostname is None:
                self.log.warn("Missing sitename or hostname for service:\n%s" % pformat(service))
                continue

            site_path = cfgPath(SITES_BASE, site_path_prefix, "%s.%s.%s"
                                % (site_path_prefix, sitename, country_code))
            ce_path = cfgPath(site_path, 'CEs', hostname)
            queue_path = cfgPath(ce_path, 'Queues', 'default')
            cfg_system.add(site_path, 'Name', sitename)
            cfg_system.append_unique(site_path, 'CE', hostname)
            cfg_system.add(ce_path, 'CEType', site_path_prefix.capitalize())
            cfg_system.add(ce_path, 'Architecture', 'x86_64')
            cfg_system.add(ce_path, 'OS', 'EL6')
            cfg_system.add(ce_path, 'LastSeen', date.today().strftime('%d/%m/%Y'))
#.........這裏部分代碼省略.........