Python Request.Request类代码示例

class OperationHandlerBaseTests( unittest.TestCase ):
  """
  .. class:: OperationHandlerBaseTests

  """

  def setUp( self ):
    """ test set up """
    self.req = Request()
    self.req.RequestName = "testRequest"
    self.op = Operation( {"Type" : "ForwardDISET", "Arguments" : "foobar" } )
    self.req.addOperation( self.op )
    self.baseOp = OperationHandlerBase()

  def tearDown( self ):
    """ test tear down """
    del self.baseOp
    del self.op
    del self.req

  def testOperationHandlerBase( self ):
    """ base op test """
    self.baseOp.setOperation( self.op )

    # # log is there
    self.assertEqual( "log" in dir( self.baseOp ), True, "log missing" )
    # # operation is there
    self.assertEqual( "operation" in dir( self.baseOp ), True, "operation is missing" )
    # # request is there
    self.assertEqual( "request" in dir( self.baseOp ), True, "request is missing" )
    # # __call__ not implemented
    self.assertRaises( NotImplementedError, self.baseOp )
    # # replica manager
    self.assertEqual( isinstance( self.baseOp.dm, DataManager ), True, "DataManager is missing" )

  def __deleteSandboxFromExternalBackend( self, SEName, SEPFN ):
    if self.getCSOption( "DelayedExternalDeletion", True ):
      gLogger.info( "Setting deletion request" )
      try:

        request = Request()
        request.RequestName = "RemoteSBDeletion:%s|%s:%s" % ( SEName, SEPFN, time.time() )
        physicalRemoval = Operation()
        physicalRemoval.Type = "PhysicalRemoval"
        physicalRemoval.TargetSE = SEName
        fileToRemove = File()
        fileToRemove.PFN = SEPFN
        physicalRemoval.addFile( fileToRemove )
        request.addOperation( physicalRemoval )
        return ReqClient().putRequest( request )
      except Exception as e:
        gLogger.exception( "Exception while setting deletion request" )
        return S_ERROR( "Cannot set deletion request: %s" % str( e ) )
    else:
      gLogger.info( "Deleting external Sandbox" )
      try:
        return StorageElement( SEName ).removeFile( SEPFN )
      except Exception as e:
        gLogger.exception( "RM raised an exception while trying to delete a remote sandbox" )
        return S_ERROR( "RM raised an exception while trying to delete a remote sandbox" )

class RequestTaskTests( unittest.TestCase ):
  """
  .. class:: RequestTaskTests

  """

  def setUp( self ):
    """ test case set up """
    self.handlersDict = { "ForwardDISET" : "DIRAC/RequestManagementSystem/private/ForwardDISET" }
    self.req = Request()
    self.req.RequestName = "foobarbaz"
    self.req.OwnerGroup = "dirac_user"
    self.req.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba"
    self.op = Operation( { "Type": "ForwardDISET", "Arguments" : "tts10:helloWorldee" } )
    self.req.addOperation( self.op )
    self.task = None  

  def tearDown( self ):
    """ test case tear down """
    del self.req
    del self.op
    del self.task

  def testAPI( self ):
    """ test API """
    self.task = RequestTask( self.req.toXML()["Value"], self.handlersDict )
    self.task.requestClient = Mock( return_value = Mock(spec=RequestClient) ) 
    self.task.requestClient().updateRequest = Mock()
    self.task.requestClient().updateRequest.return_value = { "OK" : True, "Value" : None }    
    ret = self.task()
    self.assertEqual( ret["OK"], True , "call failed")

def createRequest(requestID, opType, opStatus, fileStatus, lfnError=" ",
                  lfn="/ilc/fake/lfn"):
  """Create a request."""
  req = Request({"RequestID": requestID})
  op = Operation({"Type": opType, "Status": opStatus})
  op.addFile(File({"LFN": lfn, "Status": fileStatus, "Error": lfnError}))
  req.addOperation(op)
  return req

  def test06Dirty( self ):
    """ dirty records """
    db = RequestDB()

    r = Request()
    r.RequestName = "dirty"

    op1 = Operation( { "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"} )
    op1 += File( {"LFN": "/a/b/c/1", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" } )

    op2 = Operation( { "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"} )
    op2 += File( {"LFN": "/a/b/c/2", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" } )

    op3 = Operation( { "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"} )
    op3 += File( {"LFN": "/a/b/c/3", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" } )

    r += op1
    r += op2
    r += op3

    put = db.putRequest( r )
    self.assertEqual( put["OK"], True, "1. putRequest failed: %s" % put.get( "Message", "" ) )
    reqID = put['Value']

    r = db.getRequest( reqID )
    self.assertEqual( r["OK"], True, "1. getRequest failed: %s" % r.get( "Message", "" ) )
    r = r["Value"]

    del r[0]
    self.assertEqual( len( r ), 2, "1. len wrong" )

    put = db.putRequest( r )
    self.assertEqual( put["OK"], True, "2. putRequest failed: %s" % put.get( "Message", "" ) )
    reqID = put['Value']

    r = db.getRequest( reqID )
    self.assertEqual( r["OK"], True, "2. getRequest failed: %s" % r.get( "Message", "" ) )

    r = r["Value"]
    self.assertEqual( len( r ), 2, "2. len wrong" )

    op4 = Operation( { "Type": "ReplicateAndRegister", "TargetSE": "CERN-USER"} )
    op4 += File( {"LFN": "/a/b/c/4", "Status": "Scheduled", "Checksum": "123456", "ChecksumType": "ADLER32" } )

    r[0] = op4
    put = db.putRequest( r )
    self.assertEqual( put["OK"], True, "3. putRequest failed: %s" % put.get( "Message", "" ) )
    reqID = put['Value']

    r = db.getRequest( reqID )
    self.assertEqual( r["OK"], True, "3. getRequest failed: %s" % r.get( "Message", "" ) )
    r = r["Value"]

    self.assertEqual( len( r ), 2, "3. len wrong" )

    delete = db.deleteRequest( reqID )
    self.assertEqual( delete["OK"], True, delete['Message'] if 'Message' in delete else 'OK' )

  def __init__(self,request=None,init=True):

    # A common set of attributes that define requests.
    self.requestAttributes = ['SubRequestID','TargetSE','Status','Operation','SourceSE','Catalogue','SpaceToken','RequestType']
    # Possible keys to define the files in the request.
    self.fileAttributes = ['LFN','Size','PFN','GUID','Md5','Addler','Status','Attempt','FileID']
    # Possible keys to define the dataset in the request.
    self.datasetAttributes = ['Handle']

    Request.__init__(self,request,init)

def _sendToFailover( rpcStub ):
  """ Create a ForwardDISET operation for failover
  """
  request = Request()
  request.RequestName = "Accounting.DataStore.%s.%s" % ( time.time(), random.random() )
  forwardDISETOp = Operation()
  forwardDISETOp.Type = "ForwardDISET"
  forwardDISETOp.Arguments = DEncode.encode( rpcStub )
  request.addOperation( forwardDISETOp )

  return ReqClient().putRequest( request )

    def export_putRequest(cls, requestJSON):
        """ put a new request into RequestDB

    :param cls: class ref
    :param str requestJSON: request serialized to JSON format
    """
        requestDict = json.loads(requestJSON)
        requestName = requestDict.get("RequestID", requestDict.get("RequestName", "***UNKNOWN***"))
        request = Request(requestDict)
        optimized = request.optimize()
        if optimized.get("Value", False):
            gLogger.debug("putRequest: request was optimized")
        else:
            gLogger.debug("putRequest: request unchanged", optimized.get("Message", "Nothing could be optimized"))

        valid = cls.validate(request)
        if not valid["OK"]:
            gLogger.error("putRequest: request %s not valid: %s" % (requestName, valid["Message"]))
            return valid

        # If NotBefore is not set or user defined, we calculate its value

        now = datetime.datetime.utcnow().replace(microsecond=0)
        extraDelay = datetime.timedelta(0)
        if request.Status not in Request.FINAL_STATES and (not request.NotBefore or request.NotBefore < now):
            # We don't delay if it is the first insertion
            if getattr(request, "RequestID", 0):
                # If it is a constant delay, just set it
                if cls.constantRequestDelay:
                    extraDelay = datetime.timedelta(minutes=cls.constantRequestDelay)
                else:
                    # If there is a waiting Operation with Files
                    op = request.getWaiting().get("Value")
                    if op and len(op):
                        attemptList = [opFile.Attempt for opFile in op if opFile.Status == "Waiting"]
                        if attemptList:
                            maxWaitingAttempt = max([opFile.Attempt for opFile in op if opFile.Status == "Waiting"])
                            # In case it is the first attempt, extraDelay is 0
                            # maxWaitingAttempt can be None if the operation has no File, like the ForwardDiset
                            extraDelay = datetime.timedelta(
                                minutes=2 * math.log(maxWaitingAttempt) if maxWaitingAttempt else 0
                            )

                request.NotBefore = now + extraDelay

        gLogger.info(
            "putRequest: request %s not before %s (extra delay %s)"
            % (request.RequestName, request.NotBefore, extraDelay)
        )

        requestName = request.RequestName
        gLogger.info("putRequest: Attempting to set request '%s'" % requestName)
        return cls.__requestDB.putRequest(request)

  def _multiOperationsBody(self, transJson, taskDict, ownerDN, ownerGroup):
    """ deal with a Request that has multiple operations

    :param transJson: list of lists of string and dictionaries, e.g.:

      .. code :: python

        body = [ ( "ReplicateAndRegister", { "SourceSE":"FOO-SRM", "TargetSE":"BAR-SRM" }),
                 ( "RemoveReplica", { "TargetSE":"FOO-SRM" } ),
               ]

    :param dict taskDict: dictionary of tasks, modified in this function
    :param str ownerDN: certificate DN used for the requests
    :param str onwerGroup: dirac group used for the requests

    :returns: None
    """
    failedTasks = []
    for taskID, task in taskDict.items():
      transID = task['TransformationID']
      if not task.get('InputData'):
        self._logError("Error creating request for task", "%s, No input data" % taskID, transID=transID)
        taskDict.pop(taskID)
        continue
      files = []

      oRequest = Request()
      if isinstance(task['InputData'], list):
        files = task['InputData']
      elif isinstance(task['InputData'], basestring):
        files = task['InputData'].split(';')

      # create the operations from the json structure
      for operationTuple in transJson:
        op = Operation()
        op.Type = operationTuple[0]
        for parameter, value in operationTuple[1].iteritems():
          setattr(op, parameter, value)

        for lfn in files:
          opFile = File()
          opFile.LFN = lfn
          op.addFile(opFile)

        oRequest.addOperation(op)

      result = self._assignRequestToTask(oRequest, taskDict, transID, taskID, ownerDN, ownerGroup)
      if not result['OK']:
        failedTasks.append(taskID)
    # Remove failed tasks
    for taskID in failedTasks:
      taskDict.pop(taskID)

class RequestTaskTests( unittest.TestCase ):
  """
  .. class:: RequestTaskTests

  """

  def setUp( self ):
    """ test case set up """
    self.handlersDict = { "ForwardDISET" : "DIRAC/RequestManagementSystem/private/ForwardDISET" }
    self.req = Request()
    self.req.RequestName = "foobarbaz"
    self.req.OwnerGroup = "lhcb_user"
    self.req.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=chaen/CN=705305/CN=Christophe Haen"
    self.op = Operation( { "Type": "ForwardDISET", "Arguments" : "tts10:helloWorldee" } )
    self.req.addOperation( self.op )
    self.task = None
    self.mockRC = MagicMock()

    self.mockObjectOps = MagicMock()
    self.mockObjectOps.getSections.return_value = {'OK': True,
                                                   'Value': ['DataProcessing',
                                                             'DataManager']}
    self.mockObjectOps.getOptionsDict.return_value = {'OK': True,
                                                      'Value': {'Group': 'lhcb_user', 'User': 'fstagni'}}
    self.mockOps = MagicMock()
    self.mockOps.return_value = self.mockObjectOps


  def tearDown( self ):
    """ test case tear down """
    del self.req
    del self.op
    del self.task

  def testAPI( self ):
    """ test API
    """
    rt = importlib.import_module( 'DIRAC.RequestManagementSystem.private.RequestTask' )
    rt.gMonitor = MagicMock()
    rt.Operations = self.mockOps
    rt.CS = MagicMock()

    self.task = RequestTask( self.req.toJSON()["Value"], self.handlersDict, 'csPath', 'RequestManagement/RequestExecutingAgent',
                             requestClient = self.mockRC )
    self.task.requestClient = Mock( return_value = Mock( spec = ReqClient ) )
    self.task.requestClient().updateRequest = Mock()
    self.task.requestClient().updateRequest.return_value = { "OK" : True, "Value" : None }
    ret = self.task()
    self.assertEqual( ret["OK"], True , "call failed" )

    ret = self.task.setupProxy()
    print ret

class RequestManagerHandlerTests( unittest.TestCase ):
  """
  .. class:: RequestManagerHandlerTests

  """

  def setUp( self ):
    """ test setup

    :param self: self reference
    """
    self.request = Request()
    self.request.RequestName = "RequestManagerHandlerTests"
    self.request.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba"
    self.request.OwnerGroup = "dirac_user"
    self.operation = Operation()
    self.operation.Type = "ReplicateAndRegister"
    self.operation.TargetSE = "CERN-USER"
    self.file = File()
    self.file.LFN = "/lhcb/user/c/cibak/testFile"
    self.file.Checksum = "123456"
    self.file.ChecksumType = "ADLER32"
    self.request.addOperation( self.operation )
    self.operation.addFile( self.file )
    # # xml representation of a whole request
    self.xmlStr = self.request.toXML( True )["Value"]
    # # request client
    self.requestClient = RequestClient()


  def tearDown( self ):
    """ test case tear down """
    del self.request
    del self.operation
    del self.file
    del self.xmlStr

  def test01PutRequest( self ):
    """ test set request """
    put = self.requestClient.putRequest( self.request )
    self.assertEqual( put["OK"], True, "put failed" )

  def test02GetRequest( self ):
    """ test get request """
    get = self.requestClient.getRequest( self.request.RequestName )
    self.assertEqual( get["OK"], True, "get failed" )

  def test03DeleteRequest( self ):
    """ test delete request """
    delete = self.requestClient.deleteRequest( "test" )
    self.assertEqual( delete["OK"], True, "delete failed" )

class ReqClientTestCase( unittest.TestCase ):
  """
  .. class:: ReqClientTestCase

  """

  def setUp( self ):
    """ test case set up """

    gLogger.setLevel( 'NOTICE' )

    self.file = File()
    self.file.LFN = "/lhcb/user/c/cibak/testFile"
    self.file.Checksum = "123456"
    self.file.ChecksumType = "ADLER32"

    self.file2 = File()
    self.file2.LFN = "/lhcb/user/f/fstagni/testFile"
    self.file2.Checksum = "654321"
    self.file2.ChecksumType = "ADLER32"

    self.operation = Operation()
    self.operation.Type = "ReplicateAndRegister"
    self.operation.TargetSE = "CERN-USER"
    self.operation.addFile( self.file )
    self.operation.addFile( self.file2 )

    self.request = Request()
    self.request.RequestName = "RequestManagerHandlerTests"
    self.request.OwnerDN = "/DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=cibak/CN=605919/CN=Krzysztof Ciba"
    self.request.OwnerGroup = "dirac_user"
    self.request.JobID = 123
    self.request.addOperation( self.operation )

    # # JSON representation of a whole request
    self.jsonStr = self.request.toJSON()['Value']
    # # request client
    self.requestClient = ReqClient()

    self.stressRequests = 1000
    self.bulkRequest = 1000


  def tearDown( self ):
    """ clean up """
    del self.request
    del self.operation
    del self.file
    del self.jsonStr

class ReqClientTestCase(unittest.TestCase):
  """
  .. class:: ReqClientTestCase

  """

  def setUp(self):
    """ test case set up """

    gLogger.setLevel('INFO')

    self.file = File()
    self.file.LFN = "/lhcb/user/c/cibak/testFile"
    self.file.Checksum = "123456"
    self.file.ChecksumType = "ADLER32"

    self.file2 = File()
    self.file2.LFN = "/lhcb/user/f/fstagni/testFile"
    self.file2.Checksum = "654321"
    self.file2.ChecksumType = "ADLER32"

    self.operation = Operation()
    self.operation.Type = "ReplicateAndRegister"
    self.operation.TargetSE = "CERN-USER"
    self.operation.addFile(self.file)
    self.operation.addFile(self.file2)

    proxyInfo = getProxyInfo()['Value']
    self.request = Request()
    self.request.RequestName = "RequestManagerHandlerTests"
    self.request.OwnerDN = proxyInfo['identity']
    self.request.OwnerGroup = proxyInfo['group']
    self.request.JobID = 123
    self.request.addOperation(self.operation)

    # # JSON representation of a whole request
    self.jsonStr = self.request.toJSON()['Value']
    # # request client
    self.requestClient = ReqClient()

    self.stressRequests = 1000
    self.bulkRequest = 1000

  def tearDown(self):
    """ clean up """
    del self.request
    del self.operation
    del self.file
    del self.jsonStr

  def test03sql( self ):
    """ sql insert or update """
    operation = Operation()
    operation.Type = "ReplicateAndRegister"

    request = Request()
    request.RequestName = "testRequest"
    request.RequestID = 1

    # # no parent request set
    try:
      operation.toSQL()
    except Exception, error:
      self.assertEqual( isinstance( error, AttributeError ), True, "wrong exc raised" )
      self.assertEqual( str( error ), "RequestID not set", "wrong exc reason" )

def _sendToFailover( rpcStub ):
  """ Create a ForwardDISET operation for failover
  """
  try:
    request = Request()
    request.RequestName = "Accounting.DataStore.%s.%s" % ( time.time(), random.random() )
    forwardDISETOp = Operation()
    forwardDISETOp.Type = "ForwardDISET"
    forwardDISETOp.Arguments = DEncode.encode( rpcStub )
    request.addOperation( forwardDISETOp )

    return ReqClient().putRequest( request )

  # We catch all the exceptions, because it should never crash
  except Exception as e:  # pylint: disable=broad-except
    return S_ERROR( ERMSUKN, "Exception sending accounting failover request: %s" % repr( e ) )