Python ResourceManagementClient.selectSpaceTokenOccupancyCache方法代码示例

# 需要导入模块: from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient [as 别名]
# 或者: from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient.ResourceManagementClient import selectSpaceTokenOccupancyCache [as 别名]
class FreeDiskSpaceCommand(Command):
    """
  Uses diskSpace method to get the free space
  """

    def __init__(self, args=None, clients=None):

        super(FreeDiskSpaceCommand, self).__init__(args, clients=clients)

        self.rpc = None
        self.rsClient = ResourceManagementClient()

    def _prepareCommand(self):
        """
      FreeDiskSpaceCommand requires one argument:
      - name : <str>
    """

        if "name" not in self.args:
            return S_ERROR('"name" not found in self.args')
        elementName = self.args["name"]

        return S_OK(elementName)

    def doNew(self, masterParams=None):
        """
    Gets the total and the free disk space of a DIPS storage element that
    is found in the CS and inserts the results in the SpaceTokenOccupancyCache table
    of ResourceManagementDB database.
    """

        if masterParams is not None:
            elementName = masterParams
        else:
            elementName = self._prepareCommand()
            if not elementName["OK"]:
                return elementName

        se = StorageElement(elementName)

        elementURL = se.getStorageParameters(protocol="dips")

        if elementURL["OK"]:
            elementURL = se.getStorageParameters(protocol="dips")["Value"]["URLBase"]
        else:
            gLogger.verbose("Not a DIPS storage element, skipping...")
            return S_OK()

        self.rpc = RPCClient(elementURL, timeout=120)

        free = self.rpc.getFreeDiskSpace("/")

        if not free["OK"]:
            return free
        free = free["Value"]

        total = self.rpc.getTotalDiskSpace("/")

        if not total["OK"]:
            return total
        total = total["Value"]

        if free and free < 1:
            free = 1
        if total and total < 1:
            total = 1

        result = self.rsClient.addOrModifySpaceTokenOccupancyCache(
            endpoint=elementURL, lastCheckTime=datetime.utcnow(), free=free, total=total, token=elementName
        )
        if not result["OK"]:
            return result

        return S_OK()

    def doCache(self):
        """
    This is a method that gets the element's details from the spaceTokenOccupancy cache.
    """

        elementName = self._prepareCommand()
        if not elementName["OK"]:
            return elementName

        result = self.rsClient.selectSpaceTokenOccupancyCache(token=elementName)

        if not result["OK"]:
            return result

        return S_OK(result)

    def doMaster(self):
        """
    This method calls the doNew method for each storage element
    that exists in the CS.
    """

        elements = CSHelpers.getStorageElements()

        for name in elements["Value"]:
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient [as 别名]
# 或者: from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient.ResourceManagementClient import selectSpaceTokenOccupancyCache [as 别名]

#.........这里部分代码省略.........

      It queries the srm interface, and hopefully it will not crash. Out of the
      results, we keep totalsize, guaranteedsuze, and unusedsize.

      Then, they are recorded and returned.
    """

        if masterParams is not None:
            spaceTokenEndpoint, spaceToken = masterParams
        else:
            params = self._prepareCommand()
            if not params["OK"]:
                return params
            spaceTokenEndpoint, spaceToken = params["Value"]

        # 10 secs of timeout. If it works, the reply is immediate.
        occupancy = pythonCall(10, lcg_util.lcg_stmd, spaceToken, spaceTokenEndpoint, True, 0)
        if not occupancy["OK"]:
            return occupancy
        occupancy = occupancy["Value"]

        # Timeout does not work here...
        # occupancy = lcg_util.lcg_stmd( spaceToken, spaceTokenEndpoint, True, 0 )

        if occupancy[0] != 0:
            return S_ERROR(occupancy)
        output = occupancy[1][0]

        sTokenDict = {}
        sTokenDict["Endpoint"] = spaceTokenEndpoint
        sTokenDict["Token"] = spaceToken
        sTokenDict["Total"] = float(output.get("totalsize", "0")) / 1e12  # Bytes to Terabytes
        sTokenDict["Guaranteed"] = float(output.get("guaranteedsize", "0")) / 1e12
        sTokenDict["Free"] = float(output.get("unusedsize", "0")) / 1e12

        storeRes = self._storeCommand([sTokenDict])
        if not storeRes["OK"]:
            return storeRes

        return S_OK([sTokenDict])

    def doCache(self):
        """
      Method that reads the cache table and tries to read from it. It will
      return a list of dictionaries if there are results.
    """

        params = self._prepareCommand()
        if not params["OK"]:
            return params
        spaceTokenEndpoint, spaceToken = params["Value"]

        result = self.rmClient.selectSpaceTokenOccupancyCache(spaceTokenEndpoint, spaceToken)
        if result["OK"]:
            result = S_OK([dict(zip(result["Columns"], res)) for res in result["Value"]])

        return result

    def doMaster(self):
        """
      Master method. Gets all endpoints from the storage elements and all
      the spaceTokens. Could have taken from Shares/Disk as well.
      It queries for all their possible combinations, unless there are records
      in the database for those combinations, which then are not queried.
    """

        self.log.verbose("Getting all SEs defined in the CS")
        storageElementNames = CSHelpers.getStorageElements()
        if not storageElementNames["OK"]:
            self.log.warn(storageElementNames["Message"])
            return storageElementNames
        storageElementNames = storageElementNames["Value"]

        endpointTokenSet = set()

        for storageElementName in storageElementNames:

            endpoint = CSHelpers.getStorageElementEndpoint(storageElementName)
            if not endpoint["OK"]:
                self.log.warn(endpoint["Message"])
                continue
            endpoint = endpoint["Value"]

            spaceToken = CSHelpers.getSEToken(storageElementName)
            if not spaceToken["OK"]:
                self.log.warn(spaceToken["Message"])
                continue
            spaceToken = spaceToken["Value"]

            endpointTokenSet.add((endpoint, spaceToken))

        self.log.verbose("Processing %s" % endpointTokenSet)

        for elementToQuery in endpointTokenSet:

            result = self.doNew(elementToQuery)
            if not result["OK"]:
                self.metrics["failed"].append(result)

        return S_OK(self.metrics)

# 需要导入模块: from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient [as 别名]
# 或者: from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient.ResourceManagementClient import selectSpaceTokenOccupancyCache [as 别名]

#.........这里部分代码省略.........
      It queries the srm interface, and hopefully it will not crash. Out of the
      results, we keep totalsize, guaranteedsuze, and unusedsize.
      
      Then, they are recorded and returned.
    '''   
     
    if masterParams is not None:
      spaceTokenEndpoint, spaceToken = masterParams
    else:
      params = self._prepareCommand()
      if not params[ 'OK' ]:
        return params
      spaceTokenEndpoint, spaceToken = params[ 'Value' ] 
      
    # 10 secs of timeout. If it works, the reply is immediate.  
    occupancy = pythonCall( 10, lcg_util.lcg_stmd, spaceToken, spaceTokenEndpoint, True, 0 )
    if not occupancy[ 'OK' ]:
      return occupancy
    occupancy = occupancy[ 'Value' ]
    
    #Timeout does not work here...
    #occupancy = lcg_util.lcg_stmd( spaceToken, spaceTokenEndpoint, True, 0 )
    
    if occupancy[ 0 ] != 0:
      return S_ERROR( occupancy )
    output = occupancy[ 1 ][ 0 ]

    sTokenDict = {} 
    sTokenDict[ 'Endpoint' ]   = spaceTokenEndpoint
    sTokenDict[ 'Token' ]      = spaceToken
    sTokenDict[ 'Total' ]      = float( output.get( 'totalsize', '0' ) ) / 1e12 # Bytes to Terabytes
    sTokenDict[ 'Guaranteed' ] = float( output.get( 'guaranteedsize', '0' ) ) / 1e12
    sTokenDict[ 'Free' ]       = float( output.get( 'unusedsize', '0' ) ) / 1e12                       
    
    storeRes = self._storeCommand( [ sTokenDict ] )
    if not storeRes[ 'OK' ]:
      return storeRes
           
    return S_OK( [ sTokenDict ] )                                 

  def doCache( self ):
    '''
      Method that reads the cache table and tries to read from it. It will 
      return a list of dictionaries if there are results.   
    '''
    
    params = self._prepareCommand()
    if not params[ 'OK' ]:
      return params
    spaceTokenEndpoint, spaceToken = params[ 'Value' ] 
      
    result = self.rmClient.selectSpaceTokenOccupancyCache( spaceTokenEndpoint, spaceToken )
    if result[ 'OK' ]:
      result = S_OK( [ dict( zip( result[ 'Columns' ], res ) ) for res in result[ 'Value' ] ] )
           
    return result    

  def doMaster( self ):
    '''
      Master method. Gets all endpoints from the storage elements and all 
      the spaceTokens. Could have taken from Shares/Disk as well. 
      It queries for all their possible combinations, unless there are records
      in the database for those combinations, which then are not queried. 
    '''
    
    spaceTokens = CSHelpers.getSpaceTokens() 
    if not spaceTokens[ 'OK' ]:
      return spaceTokens
    spaceTokens = spaceTokens[ 'Value' ]

    elementsToCheck = []

    seEndpoints = CSHelpers.getStorageElementEndpoints()
    if not seEndpoints[ 'OK' ]:
      return seEndpoints
    seEndpoints = seEndpoints[ 'Value' ]   

    for seEndpoint in seEndpoints:
      for spaceToken in spaceTokens:
        elementsToCheck.append( ( seEndpoint, spaceToken ) )
                                  
#    resQuery = self.rmClient.selectSpaceTokenOccupancyCache( meta = { 'columns' : [ 'Endpoint', 'Token' ] } )
#    if not resQuery[ 'OK' ]:
#      return resQuery
#    resQuery = resQuery[ 'Value' ]                                  
#
#    elementsToQuery = list( set( elementsToCheck ).difference( set( resQuery ) ) )
    
    gLogger.verbose( 'Processing %s' % elementsToCheck )
    
    for elementToQuery in elementsToCheck:

      result = self.doNew( elementToQuery  ) 
      if not result[ 'OK' ]:
        self.metrics[ 'failed' ].append( result )      
       
    return S_OK( self.metrics )
      
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF

# 需要导入模块: from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient [as 别名]
# 或者: from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient.ResourceManagementClient import selectSpaceTokenOccupancyCache [as 别名]
class FreeDiskSpaceCommand(Command):
  '''
  Uses diskSpace method to get the free space
  '''

  def __init__(self, args=None, clients=None):

    super(FreeDiskSpaceCommand, self).__init__(args, clients=clients)

    self.rmClient = ResourceManagementClient()

  def _prepareCommand(self):
    '''
      FreeDiskSpaceCommand requires one argument:
      - name : <str>
    '''

    if 'name' not in self.args:
      return S_ERROR('"name" not found in self.args')
    elementName = self.args['name']

    # We keep TB as default as this is what was used (and will still be used)
    # in the policy for "space tokens" ("real", "data" SEs)
    unit = self.args.get('unit', 'TB')

    return S_OK((elementName, unit))

  def doNew(self, masterParams=None):
    """
    Gets the parameters to run, either from the master method or from its
    own arguments.

    Gets the total and the free disk space of a storage element
    and inserts the results in the SpaceTokenOccupancyCache table
    of ResourceManagementDB database.

    The result is also returned to the caller, not only inserted.
    What is inserted in the DB will normally be in MB,
    what is returned will be in the specified unit.
    """

    if masterParams is not None:
      elementName, unit = masterParams
    else:
      params = self._prepareCommand()
      if not params['OK']:
        return params
      elementName, unit = params['Value']

    endpointResult = CSHelpers.getStorageElementEndpoint(elementName)
    if not endpointResult['OK']:
      return endpointResult

    se = StorageElement(elementName)
    occupancyResult = se.getOccupancy(unit=unit)
    if not occupancyResult['OK']:
      return occupancyResult
    occupancy = occupancyResult['Value']
    free = occupancy['Free']
    total = occupancy['Total']

    results = {'Endpoint': endpointResult['Value'],
               'Free': free,
               'Total': total,
               'ElementName': elementName}
    result = self._storeCommand(results)
    if not result['OK']:
      return result

    return S_OK({'Free': free, 'Total': total})

  def _storeCommand(self, results):
    """ Here purely for extensibility
    """
    return self.rmClient.addOrModifySpaceTokenOccupancyCache(endpoint=results['Endpoint'],
                                                             lastCheckTime=datetime.utcnow(),
                                                             free=results['Free'],
                                                             total=results['Total'],
                                                             token=results['ElementName'])

  def doCache(self):
    """
    This is a method that gets the element's details from the spaceTokenOccupancyCache DB table.
    It will return a dictionary with th results, converted to "correct" unit.
    """

    params = self._prepareCommand()
    if not params['OK']:
      return params
    elementName, unit = params['Value']

    result = self.rmClient.selectSpaceTokenOccupancyCache(token=elementName)

    if not result['OK']:
      return result
    if not result['Value']:
      return S_ERROR(errno.ENODATA, "No occupancy recorded")

    # results are normally in 'MB'
    free = result['Value'][0][3]
#.........这里部分代码省略.........