Python RequestClient.RequestClient类代码示例

  def __setRemovalRequest( self, lfn, ownerDN, ownerGroup ):
    """ Set removal request with the given credentials
    """
    request = RequestContainer()
    request.setRequestAttributes( { 'OwnerDN':ownerDN, 'OwnerGroup':ownerGroup } )
    requestName = os.path.basename( lfn ).strip()+'_removal_request.xml'
    request.setRequestName( requestName )
    request.setSourceComponent( 'JobCleaningAgent' )

    removalDict = {'Attributes':{ 'Operation':'removeFile',
                                  'TargetSE':'',
                                  'ExecutionOrder':0
                                }
                   }
    result = request.addSubRequest( removalDict, 'removal' )
    if not result['OK']:
      return result

    index = result['Value']
    fileDict = { 'LFN':lfn, 'PFN':'', 'Status':'Waiting' }
    request.setSubRequestFiles( index, 'removal', [fileDict] )

    client = RequestClient()
    result = request.toXML()
    if not result['OK']:
      return result
    xmlRequest = result['Value']
    result = client.setRequest( requestName, xmlRequest )
    return result

def _sendToFailover( rpcStub ):
  requestClient = RequestClient()
  request = RequestContainer()
  request.setDISETRequest( rpcStub )

  requestStub = request.toXML()['Value']
  return requestClient.setRequest( "Accounting.DataStore.%s.%s" % ( time.time(), random.random() ),
                                   requestStub )

class RequestManagerHandlerTests( unittest.TestCase ):
  """
  .. class:: RequestManagerHandlerTests

  """

  def setUp( self ):
    """ test setup

    :param self: self reference
    """
    self.request = Request()
    self.request.RequestName = "RequestManagerHandlerTests"
    self.request.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba"
    self.request.OwnerGroup = "dirac_user"
    self.operation = Operation()
    self.operation.Type = "ReplicateAndRegister"
    self.operation.TargetSE = "CERN-USER"
    self.file = File()
    self.file.LFN = "/lhcb/user/c/cibak/testFile"
    self.file.Checksum = "123456"
    self.file.ChecksumType = "ADLER32"
    self.request.addOperation( self.operation )
    self.operation.addFile( self.file )
    # # xml representation of a whole request
    self.xmlStr = self.request.toXML( True )["Value"]
    # # request client
    self.requestClient = RequestClient()


  def tearDown( self ):
    """ test case tear down """
    del self.request
    del self.operation
    del self.file
    del self.xmlStr

  def test01PutRequest( self ):
    """ test set request """
    put = self.requestClient.putRequest( self.request )
    self.assertEqual( put["OK"], True, "put failed" )

  def test02GetRequest( self ):
    """ test get request """
    get = self.requestClient.getRequest( self.request.RequestName )
    self.assertEqual( get["OK"], True, "get failed" )

  def test03DeleteRequest( self ):
    """ test delete request """
    delete = self.requestClient.deleteRequest( "test" )
    self.assertEqual( delete["OK"], True, "delete failed" )

class RequestCleaningAgent(AgentModule):

  def initialize(self):

    self.graceRemovalPeriod = self.am_getOption('GraceRemovalPeriod',7)
    self.checkAssigned = self.am_getOption('CheckAssigned',True)
    self.assignedResetDelay = self.am_getOption('AssignedResetDelay',7200)
    self.ftsCleaning = self.am_getOption('FTSCleaning',True)
    self.requestClient = RequestClient()
    
    return S_OK()

  def execute(self):
    """ Main execution method
    """
    
    toDate = dateTime() - day*self.graceRemovalPeriod
    result = self.requestClient.selectRequests({'Status':'Done','ToDate':str(toDate)})
    if not result['OK']:
      return result
    requestDict = result['Value']
    for rID,rName in requestDict.items():

      gLogger.verbose("Removing request %s" % rName)
      result = self.requestClient.deleteRequest(rName)
      if not result['OK']:
        gLogger.error('Failed to delete request %s' % rName, result['Message'])
      else:
        gLogger.info('Successfully removed request %d/%s' % (rID,rName) )
      
    if self.checkAssigned:
      toDate = dateTime() - second*self.assignedResetDelay
      result = self.requestClient.selectRequests({'Status':'Assigned','ToDate':str(toDate)})
      if not result['OK']:
        return result
      requestDict = result['Value']
      for rID,rName in requestDict.items():
        gLogger.verbose("Resetting request %s to Waiting" % rName)
        result = self.requestClient.setRequestStatus(rName,'Waiting')
        if not result['OK']:
          gLogger.error('Failed to reset request %s to Waiting' % rName, result['Message'])
        else:
          gLogger.info('Successfully reset request %d/%s to Waiting' % (rID,rName) )

    if self.ftsCleaning:
      pass
    
    return S_OK()  

  def initialize( self ):
    """Sets defaults """
    self.replicaManager = ReplicaManager()
    self.transClient = TransformationClient()
    self.wmsClient = WMSClient()
    self.requestClient = RequestClient()
    self.metadataClient = FileCatalogClient()
    self.storageUsageClient = StorageUsageClient()

    # This sets the Default Proxy to used as that defined under 
    # /Operations/Shifter/DataManager
    # the shifterProxy option in the Configuration can be used to change this default.
    self.am_setOption( 'shifterProxy', 'DataManager' )

    self.transformationTypes = sortList( self.am_getOption( 'TransformationTypes', ['MCSimulation', 'DataReconstruction', 'DataStripping', 'MCStripping', 'Merge', 'Replication'] ) )
    gLogger.info( "Will consider the following transformation types: %s" % str( self.transformationTypes ) )
    self.directoryLocations = sortList( self.am_getOption( 'DirectoryLocations', ['TransformationDB', 'StorageUsage', 'MetadataCatalog'] ) )
    gLogger.info( "Will search for directories in the following locations: %s" % str( self.directoryLocations ) )
    self.transfidmeta = self.am_getOption( 'TransfIDMeta', "TransformationID" )
    gLogger.info( "Will use %s as metadata tag name for TransformationID" % self.transfidmeta )
    self.archiveAfter = self.am_getOption( 'ArchiveAfter', 7 ) # days
    gLogger.info( "Will archive Completed transformations after %d days" % self.archiveAfter )
    self.activeStorages = sortList( self.am_getOption( 'ActiveSEs', [] ) )
    gLogger.info( "Will check the following storage elements: %s" % str( self.activeStorages ) )
    self.logSE = self.am_getOption( 'TransformationLogSE', 'LogSE' )
    gLogger.info( "Will remove logs found on storage element: %s" % self.logSE )
    return S_OK()

  def __init__( self, *args, **kwargs ):
    ''' c'tor
    '''
    AgentModule.__init__( self, *args, **kwargs )
    # # replica manager
    self.replicaManager = ReplicaManager()
    # # transformation client
    self.transClient = TransformationClient()
    # # wms client
    self.wmsClient = WMSClient()
    # # request client
    self.requestClient = RequestClient()
    # # file catalog clinet
    self.metadataClient = FileCatalogClient()

    # # placeholders for CS options

    # # transformations types
    self.transformationTypes = None
    # # directory locations
    self.directoryLocations = None
    # # transformation metadata
    self.transfidmeta = None
    # # archive periof in days
    self.archiveAfter = None
    # # active SEs
    self.activeStorages = None
    # # transformation log SEs
    self.logSE = None
    # # enable/disable execution
    self.enableFlag = None

    def initialize(self):

        self.RequestDBClient = RequestClient()
        backend = self.am_getOption('Backend', '')
        self.RequestDB = False
        if backend == 'mysql':
            from DIRAC.RequestManagementSystem.DB.RequestDBMySQL import RequestDBMySQL
            requestDB = RequestDBMySQL()
            if requestDB._connected:
                self.RequestDB = requestDB

        gMonitor.registerActivity("Iteration", "Agent Loops",
                                  "DISETForwardingAgent", "Loops/min",
                                  gMonitor.OP_SUM)
        gMonitor.registerActivity("Attempted", "Request Processed",
                                  "DISETForwardingAgent", "Requests/min",
                                  gMonitor.OP_SUM)
        gMonitor.registerActivity("Successful", "Request Forward Successful",
                                  "DISETForwardingAgent", "Requests/min",
                                  gMonitor.OP_SUM)
        gMonitor.registerActivity("Failed", "Request Forward Failed",
                                  "DISETForwardingAgent", "Requests/min",
                                  gMonitor.OP_SUM)

        #self.local = PathFinder.getServiceURL( "RequestManagement/localURL" )
        #if not self.local:
        #  self.local = AgentModule.am_getOption( self, 'localURL', '' )
        #if not self.local:
        #  errStr = 'The RequestManagement/localURL option must be defined.'
        #  gLogger.fatal( errStr )
        #  return S_ERROR( errStr )
        return S_OK()

  def initialize(self):

    self.graceRemovalPeriod = self.am_getOption('GraceRemovalPeriod',7)
    self.checkAssigned = self.am_getOption('CheckAssigned',True)
    self.ftsCleaning = self.am_getOption('FTSCleaning',True)
    self.requestClient = RequestClient()
    
    return S_OK()

  def initialize( self ):

    self.RequestDBClient = RequestClient()
    self.ReplicaManager = ReplicaManager()
    # This sets the Default Proxy to used as that defined under 
    # /Operations/Shifter/DataManager
    # the shifterProxy option in the Configuration can be used to change this default.
    self.am_setOption( 'shifterProxy', 'DataManager' )

    return S_OK()

  def __init__( self, transClient = None, logger = None, requestClient = None ):

    if not logger:
      logger = gLogger.getSubLogger( 'RequestTasks' )

    super( RequestTasks, self ).__init__( transClient, logger )

    if not requestClient:
      self.requestClient = RequestClient()
    else:
      self.requestClient = requestClient

  def __init__( self, transClient = None, logger = None, requestClient = None ):

    if not logger:
      logger = gLogger.getSubLogger( 'RequestTasks' )

    super( RequestTasks, self ).__init__( transClient, logger )

    if not requestClient:
      from DIRAC.RequestManagementSystem.Client.RequestClient import RequestClient
      self.requestClient = RequestClient()
    else:
      self.requestClient = requestClient

  def initialize( self ):
    """
      Called by the framework upon startup, before any cycle (execute method bellow)
    """
    self.requestDBClient = RequestClient()
    # the RequestAgentMixIn needs the capitalized version, until is is fixed keep this.
    self.RequestDBClient = self.requestDBClient
    self.replicaManager = ReplicaManager()

    gMonitor.registerActivity( "Iteration", "Agent Loops", "RemovalAgent", "Loops/min", gMonitor.OP_SUM )
    gMonitor.registerActivity( "Execute", "Request Processed", "RemovalAgent", "Requests/min", gMonitor.OP_SUM )
    gMonitor.registerActivity( "Done", "Request Completed", "RemovalAgent", "Requests/min", gMonitor.OP_SUM )

    gMonitor.registerActivity( "PhysicalRemovalAtt", "Physical removals attempted",
                               "RemovalAgent", "Removal/min", gMonitor.OP_SUM )
    gMonitor.registerActivity( "PhysicalRemovalDone", "Successful physical removals",
                               "RemovalAgent", "Removal/min", gMonitor.OP_SUM )
    gMonitor.registerActivity( "PhysicalRemovalFail", "Failed physical removals",
                               "RemovalAgent", "Removal/min", gMonitor.OP_SUM )
    gMonitor.registerActivity( "PhysicalRemovalSize", "Physically removed size",
                               "RemovalAgent", "Bytes", gMonitor.OP_ACUM )

    gMonitor.registerActivity( "ReplicaRemovalAtt", "Replica removal attempted",
                               "RemovalAgent", "Removal/min", gMonitor.OP_SUM )
    gMonitor.registerActivity( "ReplicaRemovalDone", "Successful replica removals",
                               "RemovalAgent", "Removal/min", gMonitor.OP_SUM )
    gMonitor.registerActivity( "ReplicaRemovalFail", "Failed replica removals",
                               "RemovalAgent", "Removal/min", gMonitor.OP_SUM )

    gMonitor.registerActivity( "RemoveFileAtt", "File removal attempted",
                               "RemovalAgent", "Removal/min", gMonitor.OP_SUM )
    gMonitor.registerActivity( "RemoveFileDone", "File removal done",
                               "RemovalAgent", "Removal/min", gMonitor.OP_SUM )
    gMonitor.registerActivity( "RemoveFileFail", "File removal failed",
                               "RemovalAgent", "Removal/min", gMonitor.OP_SUM )

    self.maxNumberOfThreads = self.am_getOption( 'NumberOfThreads', self.maxNumberOfThreads )
    self.maxRequestsInQueue = self.am_getOption( 'RequestsInQueue', self.maxRequestsInQueue )
    self.threadPool = ThreadPool( 1, self.maxNumberOfThreads, self.maxRequestsInQueue )

    # Set the ThreadPool in daemon mode to process new ThreadedJobs as they are inserted
    self.threadPool.daemonize()

    self.maxRequests = self.am_getOption( 'MaxRequestsPerCycle', 1200. )

    # This sets the Default Proxy to used as that defined under
    # /Operations/Shifter/DataManager
    # the shifterProxy option in the Configuration can be used to change this default.
    self.am_setOption( 'shifterProxy', 'DataManager' )

    return S_OK()

 def initialize(self):
   """Sets defaults
   """
   self.enableFlag = '' #defined below
   self.replicaManager = ReplicaManager()
   self.prodDB = TransformationClient()
   self.requestClient = RequestClient()
   self.taskIDName = 'TaskID' 
   self.externalStatus = 'ExternalStatus'
   self.externalID = 'ExternalID'
   self.am_setOption('PollingTime',2*60*60) #no stalled jobs are considered so can be frequent
   self.enableFlag = self.am_getOption('EnableFlag', False)
   self.am_setModuleParam("shifterProxy", "ProductionManager")
   self.ops = Operations()
   return S_OK()

  def initialize( self ):

    self.RequestDBClient = RequestClient()
    self.ReplicaManager = ReplicaManager()
    self.DataLog = DataLoggingClient()

    self.maxNumberOfThreads = self.am_getOption( 'NumberOfThreads', 1 )
    self.threadPoolDepth = self.am_getOption( 'ThreadPoolDepth', 1 )
    self.threadPool = ThreadPool( 1, self.maxNumberOfThreads )

    # This sets the Default Proxy to used as that defined under
    # /Operations/Shifter/DataManager
    # the shifterProxy option in the Configuration can be used to change this default.
    self.am_setOption( 'shifterProxy', 'DataManager' )

    return S_OK()

  def setUp( self ):
    """ test setup

    :param self: self reference
    """
    self.request = Request()
    self.request.RequestName = "RequestManagerHandlerTests"
    self.request.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba"
    self.request.OwnerGroup = "dirac_user"
    self.operation = Operation()
    self.operation.Type = "ReplicateAndRegister"
    self.operation.TargetSE = "CERN-USER"
    self.file = File()
    self.file.LFN = "/lhcb/user/c/cibak/testFile"
    self.file.Checksum = "123456"
    self.file.ChecksumType = "ADLER32"
    self.request.addOperation( self.operation )
    self.operation.addFile( self.file )
    # # xml representation of a whole request
    self.xmlStr = self.request.toXML( True )["Value"]
    # # request client
    self.requestClient = RequestClient()