Python Utilities.CSHelpers类代码示例

 def export_getTestHistory(self, elementType, element, fromDate, toDate):
   gLogger.info('getTestHistory')
   
   if fromDate > toDate:
     return S_ERROR('from date can not be after the to date.')
 
   selectElements = []
   if elementType == 'Site':  
     if element.split('.')[ 0 ] == 'CLOUD':
       selectElements.append( element )
     else:
       selectElements += CSHelpers.getSiteComputingElements(element)
     selectElements += CSHelpers.getSiteStorageElements(element)
   else:
     selectElements = [ element ]
       
   queryRes = rmClient.selectSAMResultLog(
                                          elementName = selectElements,
                                          meta = { 'newer' : ['LastCheckTime', fromDate ],
                                                  'older' : [ 'LastCheckTime', toDate ],
                                                  'columns' : [ 'ElementName', 'TestType', 'Status', 'LastCheckTime' ] }
                                          )
   if not queryRes[ 'OK' ]:
     return queryRes
   records = queryRes[ 'Value' ]
   
   testHistory = {}
   for record in records:
     key = record[ 0 ] + '-' + record[ 1 ]
     if key not in testHistory:
       testHistory[ key ] = []
     testHistory[ key ].append(( record[ 3 ], record[ 2 ] ))
   
   return S_OK(testHistory)

  def export_getSAMSummary(self, siteName, vo):
    """
    Returns SAM tests status for the elements of the given site.
    
    :return: S_OK( { element : { 'ElementType' :
                                                            'WMSTest' : 
                                                            'CVMFSTest' : 
                                                            'BOSSTest' :
                                                            'SETest' : } } ) / S_ERROR
    
    """
    
    gLogger.info('getSAMSummary')

    siteType = siteName.split('.')[ 0 ]
    if 'CLOUD' == siteType:
      ces = [ siteName ]
    else:
      ces = CSHelpers.getSiteComputingElements(siteName)
    ses = CSHelpers.getSiteStorageElements(siteName)

    samSummary = {}
    for ce in ces:
      samSummary[ ce ] = { 'ElementType' : 'ComputingElement' }
    for se in ses:
      samSummary[ se ] = { 'ElementType' : 'StorageElement' }

    lastCheckTime = datetime.utcnow().replace(microsecond = 0) - timedelta(hours = 24)

    queryRes = rmClient.selectResourceSAMStatus(elementName = ces, vO = vo,
                                                meta = { 'newer' : [ 'LastCheckTime', lastCheckTime ] })
    if not queryRes[ 'OK' ]:
      return queryRes
    records = queryRes[ 'Value' ]
    columns = queryRes[ 'Columns' ]

    if ses != []:
      queryRes = rmClient.selectResourceSAMStatus(elementName = ses,
                                                  meta = { 'newer' : [ 'LastCheckTime', lastCheckTime ] })
      if not queryRes[ 'OK' ]:
        return queryRes
      records += queryRes[ 'Value' ]

    for record in records:
      samDict = dict(zip(columns, record))
      elementName = samDict[ 'ElementName' ]
      samSummary[ elementName ][ 'Status' ] = samDict[ 'Status' ]
      tests = [ test.strip() for test in samDict[ 'Tests' ].split(',') ]
      queryRes = rmClient.selectSAMResult(elementName = elementName, testType = tests,
                                          meta = { 'newer' : [ 'LastCheckTime', lastCheckTime ] })
      if not queryRes[ 'OK' ]:
        return queryRes
      testRecords = queryRes[ 'Value' ]
      testColumns = queryRes[ 'Columns' ]
      for testRecord in testRecords:
        testDict = dict(zip(testColumns, testRecord))
        samSummary[ elementName ][ testDict[ 'TestType' ] ] = testDict[ 'Status' ]
              
    return S_OK(samSummary)

    def __init__(self, rStatus=None, rManagement=None):

        # Warm up local CS
        CSHelpers.warmUp()

        if rStatus is None:
            self.rStatus = ResourceStatusClient.ResourceStatusClient()
        if rManagement is None:
            self.rManagement = ResourceManagementClient.ResourceManagementClient()

        self.rssConfig = RssConfiguration()

  def doMaster( self ):
    '''
      Master method, which looks little bit spaguetti code, sorry !
      - It gets all sites and transforms them into gocSites.
      - It gets all the storage elements and transforms them into their hosts
      - It gets the fts, the ces and file catalogs.
    '''
        
    gocSites = CSHelpers.getGOCSites()
    if not gocSites[ 'OK' ]:
      return gocSites
    gocSites = gocSites[ 'Value' ]
  
    sesHosts = CSHelpers.getStorageElementsHosts()
    if not sesHosts[ 'OK' ]:
      return sesHosts      
    sesHosts = sesHosts[ 'Value' ]  
    
    resources = sesHosts      
              
    #
    #
    #FIXME: file catalogs need also to use their hosts
    # something similar applies to FTS Channels
    #
    #fts = CSHelpers.getFTS()
    #if fts[ 'OK' ]:
    #  resources = resources + fts[ 'Value' ]
    #fc = CSHelpers.getFileCatalogs()
    #if fc[ 'OK' ]:
    #  resources = resources + fc[ 'Value' ]
    
    ce = CSHelpers.getComputingElements() 
    if ce[ 'OK' ]:
      resources = resources + ce[ 'Value' ]
    
    gLogger.info( 'Processing Sites: %s' % ', '.join( gocSites ) )
    
    siteRes = self.doNew( ( 'Site', gocSites ) )
    if not siteRes[ 'OK' ]:
      self.metrics[ 'failed' ].append( siteRes[ 'Message' ] )

    gLogger.info( 'Processing Resources: %s' % ', '.join( resources ) )

    resourceRes = self.doNew( ( 'Resource', resources ) ) 
    if not resourceRes[ 'OK' ]:
      self.metrics[ 'failed' ].append( resourceRes[ 'Message' ] )
    
    return S_OK( self.metrics )

################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF

  def doMaster( self ):
    ''' Master method, which looks little bit spaghetti code, sorry !
        - It gets all sites and transforms them into gocSites.
        - It gets all the storage elements and transforms them into their hosts
        - It gets the the CEs (FTS and file catalogs will come).
    '''

    gocSites = CSHelpers.getGOCSites()
    if not gocSites[ 'OK' ]:
      return gocSites
    gocSites = gocSites[ 'Value' ]

    sesHosts = CSHelpers.getStorageElementsHosts()
    if not sesHosts[ 'OK' ]:
      return sesHosts
    sesHosts = sesHosts[ 'Value' ]

    resources = sesHosts
    
    ftsServer = getFTS3Servers()
    if ftsServer[ 'OK' ]:
      resources.extend( ftsServer[ 'Value' ] )
      
    #TODO: file catalogs need also to use their hosts
   
    #fc = CSHelpers.getFileCatalogs()
    #if fc[ 'OK' ]:
    #  resources = resources + fc[ 'Value' ]

    ce = CSHelpers.getComputingElements()
    if ce[ 'OK' ]:
      resources.extend( ce[ 'Value' ] )
       

    gLogger.verbose( 'Processing Sites: %s' % ', '.join( gocSites ) )

    siteRes = self.doNew( ( 'Site', gocSites ) )
    if not siteRes[ 'OK' ]:
      self.metrics[ 'failed' ].append( siteRes[ 'Message' ] )

    gLogger.verbose( 'Processing Resources: %s' % ', '.join( resources ) )

    resourceRes = self.doNew( ( 'Resource', resources ) )
    if not resourceRes[ 'OK' ]:
      self.metrics[ 'failed' ].append( resourceRes[ 'Message' ] )

    return S_OK( self.metrics )

################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF

  def export_getFreeDiskSpace(self, site, token):
    """ Exporting to web the
    """

    endpoint2Site = {}

    ses = CSHelpers.getStorageElements()
    if not ses['OK']:
      gLogger.error(ses['Message'])
      return ses

    for seName in ses['Value']:
      res = CSHelpers.getStorageElementEndpoint(seName)
      if not res['OK']:
        continue

      if not res['Value'] in endpoint2Site:
        endpoint2Site[res['Value']] = seName.split('-', 1)[0]

    endpointSet = set()

    if site:
      if isinstance(site, basestring):
        site = [site]

      for ep, siteName in endpoint2Site.items():
        if siteName in site:
          endpointSet.add(ep)

    if endpointSet:
      endpoint = list(endpointSet)
    else:
      endpoint = None

    res = rmClient.selectSpaceTokenOccupancyCache(endpoint=endpoint, token=token)
    if not res['OK']:
      return res

    spList = [dict(zip(res['Columns'], sp)) for sp in res['Value']]

    for spd in spList:

      try:
        spd['Site'] = endpoint2Site[spd['Endpoint']]
      except KeyError:
        spd['Site'] = 'Unknown'

    return S_OK(spList)

  def export_getCachedDowntimes( self, element, elementType, name, severity, startDate, endDate ):

    if elementType == 'StorageElement':
      name = CSHelpers.getSEHost( name )
      if not name['OK']:
        return name
      name = name['Value']

    if startDate > endDate:
      return S_ERROR( 'startDate > endDate' )

    res = rmClient.selectDowntimeCache( element = element, name = name, severity = severity,
                                        meta = { 'columns' : [ 'Element', 'Name', 'StartDate',
                                                               'EndDate', 'Severity',
                                                               'Description', 'Link' ] } )
    if not res[ 'OK' ]:
      return res

    downtimes = []

    for dt in res[ 'Value' ]:

      dtDict = dict( zip( res[ 'Columns' ], dt ) )

      if dtDict[ 'StartDate' ] < endDate and dtDict[ 'EndDate' ] > startDate:
        downtimes.append( dt )

    result = S_OK( downtimes )
    result[ 'Columns' ] = res[ 'Columns' ]

    return result

    def doMaster(self):
        """
      Master method, which looks little bit spaguetti code, sorry !
      - It gets all gocSites.
      
      As there is no bulk query, it compares with what we have on the database.
      It queries a portion of them.
    """

        gocSites = CSHelpers.getGOCSites()
        if not gocSites["OK"]:
            return gocSites
        gocSites = gocSites["Value"]

        #    resQuery = self.rmClient.selectGGUSTicketsCache( meta = { 'columns' : [ 'GocSite' ] } )
        #    if not resQuery[ 'OK' ]:
        #      return resQuery
        #    resQuery = [ element[0] for element in resQuery[ 'Value' ] ]
        #
        #    gocNamesToQuery = set( gocSites ).difference( set( resQuery ) )

        gLogger.info("Processing %s" % ", ".join(gocSites))

        for gocNameToQuery in gocSites:

            #    if gocNameToQuery is None:
            #      self.metrics[ 'failed' ].append( 'None result' )
            #      continue

            result = self.doNew(gocNameToQuery)

            if not result["OK"]:
                self.metrics["failed"].append(result)

        return S_OK(self.metrics)

  def doCommand( self ):
    """ 
    Returns running and runned jobs, querying the WMSHistory  
    for the last self.args[0] hours 
        
    :params:
      :attr:`sites`: list of sites (when not given, take every sites)

    :returns:
      
    """

    if not 'hours' in self.args:
      return S_ERROR( 'Number of hours not specified' )
    hours = self.args[ 'hours' ]

    sites = None
    if 'sites' in self.args:
      sites = self.args[ 'sites' ] 
    if sites is None:      
#FIXME: pointing to the CSHelper instead     
#      sources = self.rsClient.getSite( meta = {'columns': 'SiteName'} )
#      if not sources[ 'OK' ]:
#        return sources 
#      sources = [ si[0] for si in sources[ 'Value' ] ]
      sites = CSHelpers.getSites()      
      if not sites[ 'OK' ]:
        return sites
      sites = sites[ 'Value' ]
    
    if not sites:
      return S_ERROR( 'Sites is empty' )   

    fromD = datetime.utcnow() - timedelta( hours = hours )
    toD   = datetime.utcnow()

    runJobs = self.rClient.getReport( 'WMSHistory', 'NumberOfJobs', fromD, toD, 
                                       {}, 'Site')
    if not runJobs[ 'OK' ]:
      return runJobs 
    runJobs    = runJobs[ 'Value' ]
    
    if not 'data' in runJobs:
      return S_ERROR( 'Missing data key' )
    if not 'granularity' in runJobs:
      return S_ERROR( 'Missing granularity key' )
    
    singlePlots = {}
    
    for site, value in runJobs[ 'data' ].items():
      if site in sites:
        plot                  = {}
        plot[ 'data' ]        = { site: value }
        plot[ 'granularity' ] = runJobs[ 'granularity' ]
        singlePlots[ site ]   = plot
    
    return S_OK( singlePlots )

################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF

    def __init__(self, rStatus=None, rManagement=None, defaultStatus="Unknown"):

        # Warm up local CS
        CSHelpers.warmUp()

        if rStatus is None:
            self.rStatus = ResourceStatusClient()
        if rManagement is None:
            self.rManagement = ResourceManagementClient()
        self.defaultStatus = defaultStatus

        self.rssConfig = RssConfiguration()
        self.tokenOwner = "rs_svc"
        result = getProxyInfo()
        if result['OK']:
            self.tokenOwner = result['Value']['username']

 def export_getSitesResources( self, siteNames ):
   
   resources = Resources.Resources()
   
   if siteNames is None:
     siteNames = Resources.getSites()
     if not siteNames[ 'OK' ]:
       return siteNames
     siteNames = siteNames[ 'Value' ]
   
   if isinstance( siteNames, str ):
     siteNames = [ siteNames ]
   
   sitesRes = {}
   
   for siteName in siteNames:
     
     res = {}         
     res[ 'ces' ] = resources.getEligibleResources( 'Computing', { 'Site': siteName } )
     ses          = resources.getEligibleStorageElements( { 'Site': siteName } )
     sesHosts = CSHelpers.getStorageElementsHosts( ses )
     if not sesHosts[ 'OK' ]:
       return sesHosts
     res[ 'ses' ] = list( set( sesHosts[ 'Value' ] ) )
         
     sitesRes[ siteName ] = res
   
   return S_OK( sitesRes )

  def doCache(self):

    if not self.args['site']:
      return S_ERROR('site was not found in args')

    site = self.args['site']

    elements = CSHelpers.getSiteElements(site)

    statusList = []

    if elements['OK']:
      for element in elements['Value']:
        status = self.rssClient.selectStatusElement("Resource", "Status", element, meta={'columns': ['Status']})
        if not status['OK']:
          return status

        if status['Value']:
          statusList.append(status['Value'][0][0])
        else:  # forcing in the case the resource has no status (yet)
          statusList.append('Active')

      if 'Active' in statusList:
        return S_OK({'Status': 'Active', 'Reason': 'An element that belongs to the site is Active'})

      if 'Degraded' in statusList:
        return S_OK({'Status': 'Degraded', 'Reason': 'An element that belongs to the site is Degraded'})

    return S_OK({'Status': 'Banned', 'Reason': 'There is no Active element in the site'})

  def export_getTree(self, elementType, elementName):
    """
    Given an element type and name,
    finds its parent site and returns all descendants of that site.
    """

    gLogger.info('getTree')

    site = self.getSite(elementType, elementName)
    if not site:
      return S_ERROR('No site')

    siteStatus = rsClient.selectStatusElement('Site', 'Status', name=site,
                                              meta={'columns': ['StatusType', 'Status']})
    if not siteStatus['OK']:
      return siteStatus

    tree = {site: {'statusTypes': dict(siteStatus['Value'])}}

    ces = CSHelpers.getSiteComputingElements(site)
    cesStatus = rsClient.selectStatusElement('Resource', 'Status', name=ces,
                                             meta={'columns': ['Name', 'StatusType', 'Status']})
    if not cesStatus['OK']:
      return cesStatus

    ses = CSHelpers.getSiteStorageElements(site)
    sesStatus = rsClient.selectStatusElement('Resource', 'Status', name=ses,
                                             meta={'columns': ['Name', 'StatusType', 'Status']})
    if not sesStatus['OK']:
      return sesStatus

    def feedTree(elementsList):

      elements = {}
      for elementTuple in elementsList['Value']:
        name, statusType, status = elementTuple

        if name not in elements:
          elements[name] = {}
        elements[name][statusType] = status

      return elements

    tree[site]['ces'] = feedTree(cesStatus)
    tree[site]['ses'] = feedTree(sesStatus)

    return S_OK(tree)

  def doCommand( self ):
    """
    Returns failed jobs using the DIRAC accounting system for every site
    for the last self.args[0] hours

    :params:
      :attr:`sites`: list of sites (when not given, take every site)

    :returns:

    """

    if 'hours' not in self.args:
      return S_ERROR( 'Number of hours not specified' )
    hours = self.args[ 'hours' ]

    sites = None
    if 'sites' in self.args:
      sites = self.args[ 'sites' ]
    if sites is None:
#FIXME: pointing to the CSHelper instead
#      sources = self.rsClient.getSite( meta = {'columns': 'SiteName'} )
#      if not sources[ 'OK' ]:
#        return sources
#      sources = [ si[0] for si in sources[ 'Value' ] ]
      sites = CSHelpers.getSites()
      if not sites[ 'OK' ]:
        return sites
      sites = sites[ 'Value' ]

    if not sites:
      return S_ERROR( 'Sites is empty' )

    fromD = datetime.utcnow() - timedelta( hours = hours )
    toD   = datetime.utcnow()

    failedPilots = self.rClient.getReport( 'Pilot', 'NumberOfPilots', fromD, toD,
                                           { 'GridStatus' : [ 'Aborted' ],
                                              'Site'       : sites
                                           }, 'Site' )
    if not failedPilots[ 'OK' ]:
      return failedPilots
    failedPilots = failedPilots[ 'Value' ]

    if not 'data' in failedPilots:
      return S_ERROR( 'Missing data key' )
    if not 'granularity' in failedPilots:
      return S_ERROR( 'Missing granularity key' )

    singlePlots = {}

    for site, value in failedPilots[ 'data' ].items():
      if site in sites:
        plot                  = {}
        plot[ 'data' ]        = { site: value }
        plot[ 'granularity' ] = failedPilots[ 'granularity' ]
        singlePlots[ site ]   = plot

    return S_OK( singlePlots )

 def export_getComputingElements(self):
   """
   Returns the list of all CEs.
   """
   
   gLogger.info('getComputingElements')
   
   return CSHelpers.getComputingElements()