Python KafkaCodec.encode_fetch_request方法代码示例

# 需要导入模块: from afkak.kafkacodec import KafkaCodec [as 别名]
# 或者: from afkak.kafkacodec.KafkaCodec import encode_fetch_request [as 别名]
    def test_makeRequest(self):
        id1 = 54321
        id2 = 76543
        reactor = MemoryReactorClock()
        c = KafkaBrokerClient('testmakeRequest', reactor=reactor)
        request = KafkaCodec.encode_fetch_request('testmakeRequest', id1)
        d = c.makeRequest(id1, request)
        eb1 = Mock()
        self.assertIsInstance(d, Deferred)
        d.addErrback(eb1)
        # Make sure the request shows unsent
        self.assertFalse(c.requests[id1].sent)
        # Make sure a connection was attempted
        self.assertTrue(c.connector)
        c.connector.factory = c  # MemoryReactor doesn't make this connection.
        # Bring up the "connection"...
        c.buildProtocol(None)
        # Replace the created proto with a mock
        c.proto = Mock()
        # Advance the clock so sendQueued() will be called
        reactor.advance(1.0)
        # The proto should have be asked to sendString the request
        c.proto.sendString.assert_called_once_with(request)

        # now call with 'expectReply=False'
        c.proto = Mock()
        request = KafkaCodec.encode_fetch_request('testmakeRequest2', id2)
        d2 = c.makeRequest(id2, request, expectResponse=False)
        self.assertIsInstance(d2, Deferred)
        c.proto.sendString.assert_called_once_with(request)

        # Now close the KafkaBrokerClient
        c.close()
        fail1 = eb1.call_args[0][0]  # The actual failure sent to errback
        self.assertTrue(fail1.check(CancelledError))

# 需要导入模块: from afkak.kafkacodec import KafkaCodec [as 别名]
# 或者: from afkak.kafkacodec.KafkaCodec import encode_fetch_request [as 别名]
 def test_makeUnconnectedRequest(self):
     """ test_makeUnconnectedRequest
     Ensure that sending a request when not connected will attempt to bring
     up a connection if one isn't already in the process of being brought up
     """
     id1 = 65432
     reactor = MemoryReactorClock()
     c = KafkaBrokerClient('testmakeUnconnectedRequest',
                           reactor=reactor)
     request = KafkaCodec.encode_fetch_request(
         'testmakeUnconnectedRequest', id1)
     d = c.makeRequest(id1, request)
     self.assertIsInstance(d, Deferred)
     # Make sure the request shows unsent
     self.assertFalse(c.requests[id1].sent)
     # Make sure a connection was attempted
     self.assertTrue(c.connector)
     c.connector.factory = c  # MemoryReactor doesn't make this connection.
     # Bring up the "connection"...
     c.buildProtocol(None)
     # Replace the created proto with a mock
     c.proto = Mock()
     reactor.advance(1.0)
     # Now, we should have seen the 'sendString' called
     c.proto.sendString.assert_called_once_with(request)

# 需要导入模块: from afkak.kafkacodec import KafkaCodec [as 别名]
# 或者: from afkak.kafkacodec.KafkaCodec import encode_fetch_request [as 别名]
 def test_cancelRequestNoReply(self):
     id2 = 87654
     reactor = MemoryReactorClock()
     c = KafkaBrokerClient('test_connect', reactor=reactor)
     c._connect()  # Force a connection attempt
     c.connector.factory = c  # MemoryReactor doesn't make this connection.
     # Fake a protocol
     c.proto = Mock()
     # now call with 'expectReply=False'
     c.proto = Mock()
     request = KafkaCodec.encode_fetch_request('testcancelRequest2', id2)
     d2 = c.makeRequest(id2, request, expectResponse=False)
     self.assertIsInstance(d2, Deferred)
     c.proto.sendString.assert_called_once_with(request)
     # This one we cancel by ID. It should fail due to the
     # expectResponse=False, since we don't keep the requestID after send
     self.assertRaises(KeyError, c.cancelRequest, id2)

# 需要导入模块: from afkak.kafkacodec import KafkaCodec [as 别名]
# 或者: from afkak.kafkacodec.KafkaCodec import encode_fetch_request [as 别名]
    def test_encode_fetch_request(self):
        requests = [
            FetchRequest("topic1", 0, 10, 1024),
            FetchRequest(u"topic2", 1, 20, 100),
        ]

        header = b"".join([
            struct.pack('>h', 1),       # Msg Header, Message type = Fetch
            struct.pack('>h', 0),               # Msg Header, API version
            struct.pack('>i', 3),               # Msg Header, Correlation ID
            struct.pack('>h7s', 7, b"client1"),  # Msg Header, The client ID
            struct.pack('>i', -1),              # Replica Id
            struct.pack('>i', 2),               # Max wait time
            struct.pack('>i', 100),             # Min bytes
            struct.pack('>i', 2),               # Num requests
        ])

        topic1 = b"".join([
            struct.pack('>h6s', 6, b'topic1'),  # Topic
            struct.pack('>i', 1),              # Num Payloads
            struct.pack('>i', 0),              # Partition 0
            struct.pack('>q', 10),             # Offset
            struct.pack('>i', 1024),           # Max Bytes
        ])

        topic2 = b"".join([
            struct.pack('>h6s', 6, b'topic2'),  # Topic
            struct.pack('>i', 1),              # Num Payloads
            struct.pack('>i', 1),              # Partition 0
            struct.pack('>q', 20),             # Offset
            struct.pack('>i', 100),            # Max Bytes
        ])

        expected1 = b"".join([header, topic1, topic2])
        expected2 = b"".join([header, topic2, topic1])

        encoded = KafkaCodec.encode_fetch_request(
            b"client1", 3, requests, 2, 100)
        self.assertIn(encoded, [expected1, expected2])

# 需要导入模块: from afkak.kafkacodec import KafkaCodec [as 别名]
# 或者: from afkak.kafkacodec.KafkaCodec import encode_fetch_request [as 别名]
 def test_requestsRetried(self):
     id1 = 65432
     reactor = MemoryReactorClock()
     c = KafkaBrokerClient('testrequestsRetried',
                           reactor=reactor)
     request = KafkaCodec.encode_fetch_request(
         'testrequestsRetried', id1)
     c.makeRequest(id1, request)
     # Make sure the request shows unsent
     self.assertFalse(c.requests[id1].sent)
     c.connector.factory = c  # MemoryReactor doesn't make this connection.
     # Bring up the "connection"...
     c.buildProtocol(None)
     # Replace the created proto with a mock
     c.proto = Mock()
     reactor.advance(0.1)
     # Now, we should have seen the 'sendString' called
     c.proto.sendString.assert_called_once_with(request)
     # And the request should be 'sent'
     self.assertTrue(c.requests[id1].sent)
     # Before the reply 'comes back' drop the connection
     from twisted.internet.main import CONNECTION_LOST
     c.clientConnectionLost(c.connector, Failure(CONNECTION_LOST))
     # Make sure the proto was reset
     self.assertIs(c.proto, None)
     # Advance the clock again
     reactor.advance(0.1)
     # Make sure the request shows unsent
     self.assertFalse(c.requests[id1].sent)
     # Bring up the "connection"...
     c.buildProtocol(None)
     # Replace the created proto with a mock
     c.proto = Mock()
     reactor.advance(0.1)
     # Now, we should have seen the 'sendString' called
     c.proto.sendString.assert_called_once_with(request)
     # And the request should be 'sent'
     self.assertTrue(c.requests[id1].sent)

# 需要导入模块: from afkak.kafkacodec import KafkaCodec [as 别名]
# 或者: from afkak.kafkacodec.KafkaCodec import encode_fetch_request [as 别名]
    def test_cancelRequest(self):
        errBackCalled = [False]

        def _handleCancelErrback(reason):
            log.debug("_handleCancelErrback: %r", reason)
            reason.trap(CancelledError)
            errBackCalled[0] = True

        id1 = 65432
        reactor = MemoryReactorClock()
        c = KafkaBrokerClient('test_connect', reactor=reactor)
        c._connect()  # Force a connection attempt
        c.connector.factory = c  # MemoryReactor doesn't make this connection.
        # Fake a protocol
        c.proto = Mock()
        request = KafkaCodec.encode_fetch_request('testcancelRequest', id1)
        d = c.makeRequest(id1, request)
        self.assertIsInstance(d, Deferred)
        d.addErrback(_handleCancelErrback)
        c.proto.sendString.assert_called_once_with(request)
        # Now try to cancel the request
        d.cancel()
        self.assertTrue(errBackCalled[0])

# 需要导入模块: from afkak.kafkacodec import KafkaCodec [as 别名]
# 或者: from afkak.kafkacodec.KafkaCodec import encode_fetch_request [as 别名]
    def test_makeRequest_fails(self):
        id1 = 15432
        reactor = MemoryReactorClock()
        c = KafkaBrokerClient('testmakeRequest', reactor=reactor)
        request = KafkaCodec.encode_fetch_request('testmakeRequest', id1)
        d = c.makeRequest(id1, request)
        eb1 = Mock()
        self.assertIsInstance(d, Deferred)
        d.addErrback(eb1)
        c.connector.factory = c  # MemoryReactor doesn't make this connection.
        # Bring up the "connection"...
        c.buildProtocol(None)
        # Replace the created proto with a mock
        c.proto = Mock()
        c.proto.sendString.side_effect = StringTooLongError(
            "Tried to send too many bytes")
        # Advance the clock so sendQueued() will be called
        reactor.advance(1.0)
        # The proto should have be asked to sendString the request
        c.proto.sendString.assert_called_once_with(request)

        # Now close the KafkaBrokerClient
        c.close()

# 需要导入模块: from afkak.kafkacodec import KafkaCodec [as 别名]
# 或者: from afkak.kafkacodec.KafkaCodec import encode_fetch_request [as 别名]
    def test_handleResponse(self):
        def make_fetch_response(id):
            t1 = "topic1"
            t2 = "topic2"
            msgs = map(
                create_message, ["message1", "hi", "boo", "foo", "so fun!"])
            ms1 = KafkaCodec._encode_message_set([msgs[0], msgs[1]])
            ms2 = KafkaCodec._encode_message_set([msgs[2]])
            ms3 = KafkaCodec._encode_message_set([msgs[3], msgs[4]])

            packFmt = '>iih{}siihqi{}sihqi{}sh{}siihqi{}s'.format(
                len(t1), len(ms1), len(ms2), len(t2), len(ms3))
            return struct.pack(
                packFmt, id, 2, len(t1), t1, 2, 0, 0, 10, len(ms1), ms1, 1,
                1, 20, len(ms2), ms2, len(t2), t2, 1, 0, 0, 30, len(ms3), ms3)

        c = KafkaBrokerClient('testhandleResponse')
        logsave = brokerclient.log
        try:
            brokerclient.log = Mock()
            badId = 98765
            response = make_fetch_response(badId)
            # First test that a response without first sending a request
            # generates a log message
            c.handleResponse(response)
            brokerclient.log.warning.assert_called_once_with(
                'Unexpected response:%r, %r', badId, response)

            # Now try a request/response pair and ensure the deferred is called
            goodId = 12345
            c.proto = Mock()
            request = KafkaCodec.encode_fetch_request(
                'testhandleResponse2', goodId)
            d = c.makeRequest(goodId, request)
            self.assertIsInstance(d, Deferred)
            c.proto.sendString.assert_called_once_with(request)
            response = make_fetch_response(goodId)
            self.assertFalse(d.called)
            # Ensure trying to make another request with the same ID before
            # we've got a response raises an exception
            c.proto.sendString = Mock(
                side_effect=Exception(
                    'brokerclient sending duplicate request'))
            self.assertRaises(
                DuplicateRequestError, c.makeRequest, goodId, request)
            c.handleResponse(response)
            self.assertTrue(d.called)

            # Now that we've got the response, try again with the goodId
            c.proto = Mock()  # Need a new mock...
            d = c.makeRequest(goodId, request)
            self.assertIsInstance(d, Deferred)
            c.proto.sendString.assert_called_once_with(request)
            response = make_fetch_response(goodId)
            self.assertFalse(d.called)
            c.handleResponse(response)
            self.assertTrue(d.called)

            # Now try with a real request, but with expectResponse=False
            # We still get a deferred back, because the request can be
            # cancelled before it's sent, and in that case, we errback()
            # the deferred
            d2 = c.makeRequest(goodId, request, expectResponse=False)
            self.assertIsInstance(d2, Deferred)
            # Send the (unexpected) response, and check for the log message
            c.handleResponse(response)
            brokerclient.log.warning.assert_called_with(
                'Unexpected response:%r, %r', goodId, response)
        finally:
            brokerclient.log = logsave