Python ResourceManagementClient.addOrModifySpaceTokenOccupancyCache方法代码示例

# 需要导入模块: from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient [as 别名]
# 或者: from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient.ResourceManagementClient import addOrModifySpaceTokenOccupancyCache [as 别名]
class FreeDiskSpaceCommand(Command):
    """
  Uses diskSpace method to get the free space
  """

    def __init__(self, args=None, clients=None):

        super(FreeDiskSpaceCommand, self).__init__(args, clients=clients)

        self.rpc = None
        self.rsClient = ResourceManagementClient()

    def _prepareCommand(self):
        """
      FreeDiskSpaceCommand requires one argument:
      - name : <str>
    """

        if "name" not in self.args:
            return S_ERROR('"name" not found in self.args')
        elementName = self.args["name"]

        return S_OK(elementName)

    def doNew(self, masterParams=None):
        """
    Gets the total and the free disk space of a DIPS storage element that
    is found in the CS and inserts the results in the SpaceTokenOccupancyCache table
    of ResourceManagementDB database.
    """

        if masterParams is not None:
            elementName = masterParams
        else:
            elementName = self._prepareCommand()
            if not elementName["OK"]:
                return elementName

        se = StorageElement(elementName)

        elementURL = se.getStorageParameters(protocol="dips")

        if elementURL["OK"]:
            elementURL = se.getStorageParameters(protocol="dips")["Value"]["URLBase"]
        else:
            gLogger.verbose("Not a DIPS storage element, skipping...")
            return S_OK()

        self.rpc = RPCClient(elementURL, timeout=120)

        free = self.rpc.getFreeDiskSpace("/")

        if not free["OK"]:
            return free
        free = free["Value"]

        total = self.rpc.getTotalDiskSpace("/")

        if not total["OK"]:
            return total
        total = total["Value"]

        if free and free < 1:
            free = 1
        if total and total < 1:
            total = 1

        result = self.rsClient.addOrModifySpaceTokenOccupancyCache(
            endpoint=elementURL, lastCheckTime=datetime.utcnow(), free=free, total=total, token=elementName
        )
        if not result["OK"]:
            return result

        return S_OK()

    def doCache(self):
        """
    This is a method that gets the element's details from the spaceTokenOccupancy cache.
    """

        elementName = self._prepareCommand()
        if not elementName["OK"]:
            return elementName

        result = self.rsClient.selectSpaceTokenOccupancyCache(token=elementName)

        if not result["OK"]:
            return result

        return S_OK(result)

    def doMaster(self):
        """
    This method calls the doNew method for each storage element
    that exists in the CS.
    """

        elements = CSHelpers.getStorageElements()

        for name in elements["Value"]:
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient [as 别名]
# 或者: from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient.ResourceManagementClient import addOrModifySpaceTokenOccupancyCache [as 别名]
class SpaceTokenOccupancyCommand( Command ):
  '''
  Uses lcg_util to query status of endpoint for a given token.
  ''' 

  def __init__( self, args = None, clients = None ):
    
    super( SpaceTokenOccupancyCommand, self ).__init__( args, clients )
    
    if 'ResourceManagementClient' in self.apis:
      self.rmClient = self.apis[ 'ResourceManagementClient' ]
    else:
      self.rmClient = ResourceManagementClient()

  def _storeCommand( self, results ):
    '''
      Stores the results of doNew method on the database.
    '''
    
    for result in results:
      
      resQuery = self.rmClient.addOrModifySpaceTokenOccupancyCache( result[ 'Endpoint' ], 
                                                                    result[ 'Token' ], 
                                                                    result[ 'Total' ], 
                                                                    result[ 'Guaranteed' ], 
                                                                    result[ 'Free' ] )
      if not resQuery[ 'OK' ]:
        return resQuery
    
    return S_OK()  

  def _prepareCommand( self ):
    '''
      SpaceTokenOccupancy requires one argument:
      - elementName : <str>
    
      Given a (storage)elementName, we calculate its endpoint and spaceToken,
      which are used to query the srm interface.
    '''

    if not 'name' in self.args:
      return S_ERROR( '"name" not found in self.args' )
    elementName = self.args[ 'name' ]
    
    endpoint = CSHelpers.getStorageElementEndpoint( elementName )
    if not endpoint[ 'OK' ]:
      return endpoint
    endpoint = endpoint[ 'Value' ]
    
    spaceToken = CSHelpers.getStorageElementSpaceToken( elementName )
    if not spaceToken[ 'OK' ]:
      return spaceToken
    spaceToken = spaceToken[ 'Value']
  
    return S_OK( ( endpoint, spaceToken ) )

  def doNew( self, masterParams = None ):
    '''
      Gets the parameters to run, either from the master method or from its
      own arguments.
      
      It queries the srm interface, and hopefully it will not crash. Out of the
      results, we keep totalsize, guaranteedsuze, and unusedsize.
      
      Then, they are recorded and returned.
    '''   
     
    if masterParams is not None:
      spaceTokenEndpoint, spaceToken = masterParams
    else:
      params = self._prepareCommand()
      if not params[ 'OK' ]:
        return params
      spaceTokenEndpoint, spaceToken = params[ 'Value' ] 
      
    # 10 secs of timeout. If it works, the reply is immediate.  
    occupancy = pythonCall( 10, lcg_util.lcg_stmd, spaceToken, spaceTokenEndpoint, True, 0 )
    if not occupancy[ 'OK' ]:
      return occupancy
    occupancy = occupancy[ 'Value' ]
    
    #Timeout does not work here...
    #occupancy = lcg_util.lcg_stmd( spaceToken, spaceTokenEndpoint, True, 0 )
    
    if occupancy[ 0 ] != 0:
      return S_ERROR( occupancy )
    output = occupancy[ 1 ][ 0 ]

    sTokenDict = {} 
    sTokenDict[ 'Endpoint' ]   = spaceTokenEndpoint
    sTokenDict[ 'Token' ]      = spaceToken
    sTokenDict[ 'Total' ]      = float( output.get( 'totalsize', '0' ) ) / 1e12 # Bytes to Terabytes
    sTokenDict[ 'Guaranteed' ] = float( output.get( 'guaranteedsize', '0' ) ) / 1e12
    sTokenDict[ 'Free' ]       = float( output.get( 'unusedsize', '0' ) ) / 1e12                       
    
    storeRes = self._storeCommand( [ sTokenDict ] )
    if not storeRes[ 'OK' ]:
      return storeRes
           
    return S_OK( [ sTokenDict ] )                                 
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient [as 别名]
# 或者: from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient.ResourceManagementClient import addOrModifySpaceTokenOccupancyCache [as 别名]

#.........这里部分代码省略.........

        serviceUp = res[ 'serviceUpTime' ]
        machineUp = res[ 'machineUpTime' ]
        site      = res[ 'site' ]
        system    = res[ 'system' ]
       
        resQuery = self.rmClient.addOrModifyVOBOXCache( site, system, serviceUp, 
                                                        machineUp, now )    
        if not resQuery[ 'OK' ]:
          self.log.error( str( resQuery[ 'Message' ] ) ) 

      #SpaceTokenOccupancy
      for co in self.commandObjectsSpaceTokenOccupancy:
        
        commandName = co[0][1].split( '_' )[0]
        self.log.info( 'Executed %s with %s' % ( commandName, str( co[2] ) ) )

        co[1].setArgs( co[2] )
        self.clientsInvoker.setCommand( co[1] )
        res = self.clientsInvoker.doCommand()[ 'Result' ]
        
        if not res[ 'OK' ]:
          self.log.warn( res[ 'Message' ] )
          continue

        site, token = co[ 2 ]

        res = res[ 'Value' ]
        
        total      = res[ 'total' ]
        guaranteed = res[ 'guaranteed' ]
        free       = res[ 'free' ]
               
        resQuery = self.rmClient.addOrModifySpaceTokenOccupancyCache( site, token, 
                                                                      total, guaranteed,
                                                                      free, now )    
        if not resQuery[ 'OK' ]:
          self.log.error( str( resQuery[ 'Message' ] ) )                     

      for co in self.commandObjectsListClientsCache:

        commandName = co[0][1].split( '_' )[0]
        self.log.info( 'Executed %s' % commandName )
        try:
          self.clientsInvoker.setCommand( co[1] )
          res = self.clientsInvoker.doCommand()['Result']

          if not res['OK']:
            self.log.warn( res['Message'] )
            continue
          res = res[ 'Value' ]

          if not res or res is None:
            self.log.info('  returned empty...')
            continue
          self.log.debug( res )

          for key in res.keys():

            clientCache = ()

            if 'ID' in res[key].keys():

              for value in res[key].keys():
                if value != 'ID':
                  clientCache = ( key.split()[1], commandName, res[key]['ID'],

# 需要导入模块: from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient [as 别名]
# 或者: from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient.ResourceManagementClient import addOrModifySpaceTokenOccupancyCache [as 别名]
class SpaceTokenOccupancyCommand(Command):
    """
  Uses lcg_util to query status of endpoint for a given token.
  """

    def __init__(self, args=None, clients=None):

        super(SpaceTokenOccupancyCommand, self).__init__(args, clients)

        if "ResourceManagementClient" in self.apis:
            self.rmClient = self.apis["ResourceManagementClient"]
        else:
            self.rmClient = ResourceManagementClient()

    def _storeCommand(self, results):
        """
      Stores the results of doNew method on the database.
    """

        for result in results:

            resQuery = self.rmClient.addOrModifySpaceTokenOccupancyCache(
                result["Endpoint"], result["Token"], result["Total"], result["Guaranteed"], result["Free"]
            )
            if not resQuery["OK"]:
                return resQuery

        return S_OK()

    def _prepareCommand(self):
        """
      SpaceTokenOccupancy requires one argument:
      - elementName : <str>

      Given a (storage)elementName, we calculate its endpoint and spaceToken,
      which are used to query the srm interface.
    """

        if "name" not in self.args:
            return S_ERROR('"name" not found in self.args')
        elementName = self.args["name"]

        endpoint = CSHelpers.getStorageElementEndpoint(elementName)
        if not endpoint["OK"]:
            return endpoint
        endpoint = endpoint["Value"]

        spaceToken = CSHelpers.getSEToken(elementName)
        if not spaceToken["OK"]:
            return spaceToken
        spaceToken = spaceToken["Value"]

        return S_OK((endpoint, spaceToken))

    def doNew(self, masterParams=None):
        """
      Gets the parameters to run, either from the master method or from its
      own arguments.

      It queries the srm interface, and hopefully it will not crash. Out of the
      results, we keep totalsize, guaranteedsuze, and unusedsize.

      Then, they are recorded and returned.
    """

        if masterParams is not None:
            spaceTokenEndpoint, spaceToken = masterParams
        else:
            params = self._prepareCommand()
            if not params["OK"]:
                return params
            spaceTokenEndpoint, spaceToken = params["Value"]

        # 10 secs of timeout. If it works, the reply is immediate.
        occupancy = pythonCall(10, lcg_util.lcg_stmd, spaceToken, spaceTokenEndpoint, True, 0)
        if not occupancy["OK"]:
            return occupancy
        occupancy = occupancy["Value"]

        # Timeout does not work here...
        # occupancy = lcg_util.lcg_stmd( spaceToken, spaceTokenEndpoint, True, 0 )

        if occupancy[0] != 0:
            return S_ERROR(occupancy)
        output = occupancy[1][0]

        sTokenDict = {}
        sTokenDict["Endpoint"] = spaceTokenEndpoint
        sTokenDict["Token"] = spaceToken
        sTokenDict["Total"] = float(output.get("totalsize", "0")) / 1e12  # Bytes to Terabytes
        sTokenDict["Guaranteed"] = float(output.get("guaranteedsize", "0")) / 1e12
        sTokenDict["Free"] = float(output.get("unusedsize", "0")) / 1e12

        storeRes = self._storeCommand([sTokenDict])
        if not storeRes["OK"]:
            return storeRes

        return S_OK([sTokenDict])

    def doCache(self):
#.........这里部分代码省略.........

# 需要导入模块: from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient import ResourceManagementClient [as 别名]
# 或者: from DIRAC.ResourceStatusSystem.Client.ResourceManagementClient.ResourceManagementClient import addOrModifySpaceTokenOccupancyCache [as 别名]
class FreeDiskSpaceCommand(Command):
  '''
  Uses diskSpace method to get the free space
  '''

  def __init__(self, args=None, clients=None):

    super(FreeDiskSpaceCommand, self).__init__(args, clients=clients)

    self.rmClient = ResourceManagementClient()

  def _prepareCommand(self):
    '''
      FreeDiskSpaceCommand requires one argument:
      - name : <str>
    '''

    if 'name' not in self.args:
      return S_ERROR('"name" not found in self.args')
    elementName = self.args['name']

    # We keep TB as default as this is what was used (and will still be used)
    # in the policy for "space tokens" ("real", "data" SEs)
    unit = self.args.get('unit', 'TB')

    return S_OK((elementName, unit))

  def doNew(self, masterParams=None):
    """
    Gets the parameters to run, either from the master method or from its
    own arguments.

    Gets the total and the free disk space of a storage element
    and inserts the results in the SpaceTokenOccupancyCache table
    of ResourceManagementDB database.

    The result is also returned to the caller, not only inserted.
    What is inserted in the DB will normally be in MB,
    what is returned will be in the specified unit.
    """

    if masterParams is not None:
      elementName, unit = masterParams
    else:
      params = self._prepareCommand()
      if not params['OK']:
        return params
      elementName, unit = params['Value']

    endpointResult = CSHelpers.getStorageElementEndpoint(elementName)
    if not endpointResult['OK']:
      return endpointResult

    se = StorageElement(elementName)
    occupancyResult = se.getOccupancy(unit=unit)
    if not occupancyResult['OK']:
      return occupancyResult
    occupancy = occupancyResult['Value']
    free = occupancy['Free']
    total = occupancy['Total']

    results = {'Endpoint': endpointResult['Value'],
               'Free': free,
               'Total': total,
               'ElementName': elementName}
    result = self._storeCommand(results)
    if not result['OK']:
      return result

    return S_OK({'Free': free, 'Total': total})

  def _storeCommand(self, results):
    """ Here purely for extensibility
    """
    return self.rmClient.addOrModifySpaceTokenOccupancyCache(endpoint=results['Endpoint'],
                                                             lastCheckTime=datetime.utcnow(),
                                                             free=results['Free'],
                                                             total=results['Total'],
                                                             token=results['ElementName'])

  def doCache(self):
    """
    This is a method that gets the element's details from the spaceTokenOccupancyCache DB table.
    It will return a dictionary with th results, converted to "correct" unit.
    """

    params = self._prepareCommand()
    if not params['OK']:
      return params
    elementName, unit = params['Value']

    result = self.rmClient.selectSpaceTokenOccupancyCache(token=elementName)

    if not result['OK']:
      return result
    if not result['Value']:
      return S_ERROR(errno.ENODATA, "No occupancy recorded")

    # results are normally in 'MB'
    free = result['Value'][0][3]
#.........这里部分代码省略.........