本文整理汇总了Python中afkak.kafkacodec.KafkaCodec类的典型用法代码示例。如果您正苦于以下问题:Python KafkaCodec类的具体用法?Python KafkaCodec怎么用?Python KafkaCodec使用的例子?那么, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了KafkaCodec类的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: test_makeRequest
def test_makeRequest(self):
id1 = 54321
id2 = 76543
reactor = MemoryReactorClock()
c = KafkaBrokerClient('testmakeRequest', reactor=reactor)
request = KafkaCodec.encode_fetch_request('testmakeRequest', id1)
d = c.makeRequest(id1, request)
eb1 = Mock()
self.assertIsInstance(d, Deferred)
d.addErrback(eb1)
# Make sure the request shows unsent
self.assertFalse(c.requests[id1].sent)
# Make sure a connection was attempted
self.assertTrue(c.connector)
c.connector.factory = c # MemoryReactor doesn't make this connection.
# Bring up the "connection"...
c.buildProtocol(None)
# Replace the created proto with a mock
c.proto = Mock()
# Advance the clock so sendQueued() will be called
reactor.advance(1.0)
# The proto should have be asked to sendString the request
c.proto.sendString.assert_called_once_with(request)
# now call with 'expectReply=False'
c.proto = Mock()
request = KafkaCodec.encode_fetch_request('testmakeRequest2', id2)
d2 = c.makeRequest(id2, request, expectResponse=False)
self.assertIsInstance(d2, Deferred)
c.proto.sendString.assert_called_once_with(request)
# Now close the KafkaBrokerClient
c.close()
fail1 = eb1.call_args[0][0] # The actual failure sent to errback
self.assertTrue(fail1.check(CancelledError))
示例2: test_decode_fetch_response
def test_decode_fetch_response(self):
t1 = "topic1"
t2 = u"topic2"
msgs = [create_message(m) for m in [b"message1", b"hi", b"boo", b"foo", b"so fun!"]]
ms1 = KafkaCodec._encode_message_set([msgs[0], msgs[1]])
ms2 = KafkaCodec._encode_message_set([msgs[2]])
ms3 = KafkaCodec._encode_message_set([msgs[3], msgs[4]])
encoded = struct.pack('>iih%dsiihqi%dsihqi%dsh%dsiihqi%ds' %
(len(t1), len(ms1), len(ms2), len(t2), len(ms3)),
4, 2, len(t1), t1.encode(), 2, 0, 0, 10, len(ms1), ms1, 1,
1, 20, len(ms2), ms2, len(t2), t2.encode(), 1, 0, 0, 30,
len(ms3), ms3)
responses = list(KafkaCodec.decode_fetch_response(encoded))
def expand_messages(response):
return FetchResponse(response.topic, response.partition,
response.error, response.highwaterMark,
list(response.messages))
expanded_responses = [expand_messages(r) for r in responses]
expect = [FetchResponse(t1, 0, 0, 10, [OffsetAndMessage(0, msgs[0]),
OffsetAndMessage(0, msgs[1])]),
FetchResponse(t1, 1, 1, 20, [OffsetAndMessage(0, msgs[2])]),
FetchResponse(t2, 0, 0, 30, [OffsetAndMessage(0, msgs[3]),
OffsetAndMessage(0, msgs[4])])]
self.assertEqual(expanded_responses, expect)
示例3: test_encode_produce_request
def test_encode_produce_request(self):
requests = [
ProduceRequest("topic1", 0, [
create_message(b"a"),
create_message(b"b"),
]),
ProduceRequest(u"topic2", 1, [
create_message(b"c"),
]),
]
msg_a_binary = KafkaCodec._encode_message(create_message(b"a"))
msg_b_binary = KafkaCodec._encode_message(create_message(b"b"))
msg_c_binary = KafkaCodec._encode_message(create_message(b"c"))
header = b"".join([
struct.pack('>h', 0), # Msg Header, Message type = Produce
struct.pack('>h', 0), # Msg Header, API version
struct.pack('>i', 2), # Msg Header, Correlation ID
struct.pack('>h7s', 7, b"client1"), # Msg Header, The client ID
struct.pack('>h', 2), # Num acks required
struct.pack('>i', 100), # Request Timeout
struct.pack('>i', 2), # The number of requests
])
total_len = len(msg_a_binary) + len(msg_b_binary)
topic1 = b"".join([
struct.pack('>h6s', 6, b'topic1'), # The topic1
struct.pack('>i', 1), # One message set
struct.pack('>i', 0), # Partition 0
struct.pack('>i', total_len + 24), # Size of message set
struct.pack('>q', 0), # No offset specified
struct.pack('>i', len(msg_a_binary)), # Length of message
msg_a_binary, # Actual message
struct.pack('>q', 0), # No offset specified
struct.pack('>i', len(msg_b_binary)), # Length of message
msg_b_binary, # Actual message
])
topic2 = b"".join([
struct.pack('>h6s', 6, b'topic2'), # The topic1
struct.pack('>i', 1), # One message set
struct.pack('>i', 1), # Partition 1
struct.pack('>i', len(msg_c_binary) + 12), # Size of message set
struct.pack('>q', 0), # No offset specified
struct.pack('>i', len(msg_c_binary)), # Length of message
msg_c_binary, # Actual message
])
expected1 = b"".join([header, topic1, topic2])
expected2 = b"".join([header, topic2, topic1])
encoded = KafkaCodec.encode_produce_request(
b"client1", 2, requests, 2, 100)
self.assertIn(encoded, [expected1, expected2])
示例4: make_fetch_response
def make_fetch_response(id):
t1 = "topic1"
t2 = "topic2"
msgs = map(
create_message, ["message1", "hi", "boo", "foo", "so fun!"])
ms1 = KafkaCodec._encode_message_set([msgs[0], msgs[1]])
ms2 = KafkaCodec._encode_message_set([msgs[2]])
ms3 = KafkaCodec._encode_message_set([msgs[3], msgs[4]])
packFmt = '>iih{}siihqi{}sihqi{}sh{}siihqi{}s'.format(
len(t1), len(ms1), len(ms2), len(t2), len(ms3))
return struct.pack(
packFmt, id, 2, len(t1), t1, 2, 0, 0, 10, len(ms1), ms1, 1,
1, 20, len(ms2), ms2, len(t2), t2, 1, 0, 0, 30, len(ms3), ms3)
示例5: test_decode_message_set_stop_iteration
def test_decode_message_set_stop_iteration(self):
encoded = b"".join([
struct.pack(">q", 0), # MsgSet Offset
struct.pack(">i", 18), # Msg Size
struct.pack(">i", 1474775406), # CRC
struct.pack(">bb", 0, 0), # Magic, flags
struct.pack(">i", 2), # Length of key
b"k1", # Key
struct.pack(">i", 2), # Length of value
b"v1", # Value
struct.pack(">q", 1), # MsgSet Offset
struct.pack(">i", 18), # Msg Size
struct.pack(">i", -16383415), # CRC
struct.pack(">bb", 0, 0), # Magic, flags
struct.pack(">i", 2), # Length of key
b"k2", # Key
struct.pack(">i", 2), # Length of value
b"v2", # Value
b"@1$%(Y!", # Random padding
])
msgs = list(KafkaCodec._decode_message_set_iter(encoded))
self.assertEqual(len(msgs), 2)
msg1, msg2 = msgs
returned_offset1, decoded_message1 = msg1
returned_offset2, decoded_message2 = msg2
self.assertEqual(returned_offset1, 0)
self.assertEqual(decoded_message1, create_message(b"v1", b"k1"))
self.assertEqual(returned_offset2, 1)
self.assertEqual(decoded_message2, create_message(b"v2", b"k2"))
示例6: test_encode_message_set
def test_encode_message_set(self):
message_set = [
create_message(b"v1", b"k1"),
create_message(b"v2", b"k2"),
]
encoded = KafkaCodec._encode_message_set(message_set)
expect = b"".join([
struct.pack(">q", 0), # MsgSet Offset
struct.pack(">i", 18), # Msg Size
struct.pack(">i", 1474775406), # CRC
struct.pack(">bb", 0, 0), # Magic, flags
struct.pack(">i", 2), # Length of key
b"k1", # Key
struct.pack(">i", 2), # Length of value
b"v1", # Value
struct.pack(">q", 0), # MsgSet Offset
struct.pack(">i", 18), # Msg Size
struct.pack(">i", -16383415), # CRC
struct.pack(">bb", 0, 0), # Magic, flags
struct.pack(">i", 2), # Length of key
b"k2", # Key
struct.pack(">i", 2), # Length of value
b"v2", # Value
])
self.assertEqual(encoded, expect)
示例7: test_makeUnconnectedRequest
def test_makeUnconnectedRequest(self):
""" test_makeUnconnectedRequest
Ensure that sending a request when not connected will attempt to bring
up a connection if one isn't already in the process of being brought up
"""
id1 = 65432
reactor = MemoryReactorClock()
c = KafkaBrokerClient('testmakeUnconnectedRequest',
reactor=reactor)
request = KafkaCodec.encode_fetch_request(
'testmakeUnconnectedRequest', id1)
d = c.makeRequest(id1, request)
self.assertIsInstance(d, Deferred)
# Make sure the request shows unsent
self.assertFalse(c.requests[id1].sent)
# Make sure a connection was attempted
self.assertTrue(c.connector)
c.connector.factory = c # MemoryReactor doesn't make this connection.
# Bring up the "connection"...
c.buildProtocol(None)
# Replace the created proto with a mock
c.proto = Mock()
reactor.advance(1.0)
# Now, we should have seen the 'sendString' called
c.proto.sendString.assert_called_once_with(request)
示例8: test_encode_offset_fetch_request
def test_encode_offset_fetch_request(self):
header = b"".join([
struct.pack('>h', 9), # Message type = offset fetch
struct.pack('>h', 1), # API version
struct.pack('>i', 42), # Correlation ID
struct.pack('>h9s', 9, b"client_id"), # The client ID
struct.pack('>h8s', 8, b"group_id"), # The group to commit for
struct.pack('>i', 2), # Num topics
])
topic1 = b"".join([
struct.pack(">h6s", 6, b"topic1"), # Topic for the request
struct.pack(">i", 2), # Two partitions
struct.pack(">i", 0), # Partition 0
struct.pack(">i", 1), # Partition 1
])
topic2 = b"".join([
struct.pack(">h6s", 6, b"topic2"), # Topic for the request
struct.pack(">i", 1), # One partitions
struct.pack(">i", 2), # Partition 2
])
expected1 = b"".join([header, topic1, topic2])
expected2 = b"".join([header, topic2, topic1])
encoded = KafkaCodec.encode_offset_fetch_request(
b"client_id", 42, u"group_id", [
OffsetFetchRequest("topic1", 0),
OffsetFetchRequest(u"topic1", 1),
OffsetFetchRequest("topic2", 2),
])
self.assertIn(encoded, [expected1, expected2])
示例9: test_encode_offset_request__no_payload
def test_encode_offset_request__no_payload(self):
expected = b"".join([
struct.pack('>h', 2), # Message type = offset fetch
struct.pack('>h', 0), # API version
struct.pack('>i', 4), # Correlation ID
struct.pack('>h3s', 3, b"cid"), # The client ID
struct.pack('>i', -1), # Replica Id
struct.pack('>i', 1), # Num topics
struct.pack(">h6s", 6, b"topic1"), # Topic for the request
struct.pack(">i", 2), # Two partitions
struct.pack(">i", 3), # Partition 3
struct.pack(">q", -1), # No time offset
struct.pack(">i", 1), # One offset requested
struct.pack(">i", 4), # Partition 3
struct.pack(">q", -1), # No time offset
struct.pack(">i", 1), # One offset requested
])
encoded = KafkaCodec.encode_offset_request(b"cid", 4, [
OffsetRequest('topic1', 3, -1, 1),
OffsetRequest(u'topic1', 4, -1, 1),
])
self.assertEqual(encoded, expected)
示例10: test_decode_offset_fetch_response
def test_decode_offset_fetch_response(self):
encoded = b"".join([
struct.pack(">i", 42), # Correlation ID
struct.pack(">i", 1), # One topics
struct.pack(">h6s", 6, b"topic1"), # First topic
struct.pack(">i", 2), # Two partitions
struct.pack(">i", 2), # Partition 2
struct.pack(">q", 4), # Offset 4
struct.pack(">h4s", 4, b"meta"), # Metadata
struct.pack(">h", 0), # No error
struct.pack(">i", 4), # Partition 4
struct.pack(">q", 8), # Offset 8
struct.pack(">h4s", 4, b"meta"), # Metadata
struct.pack(">h", 0), # No error
])
results = KafkaCodec.decode_offset_fetch_response(encoded)
self.assertEqual(set(results), set([
OffsetFetchResponse(topic=u'topic1', partition=2, offset=4,
error=0, metadata=b"meta"),
OffsetFetchResponse(topic='topic1', partition=4, offset=8,
error=0, metadata=b"meta"),
]))
示例11: test_get_response_correlation_id
def test_get_response_correlation_id(self):
t1 = b"topic1"
t2 = b"topic2"
corrID = 30372
encoded = struct.pack('>iih%dsiihqihqh%dsiihq' % (len(t1), len(t2)),
corrID, 2, len(t1), t1, 2, 0, 0, 10, 1, 1, 20,
len(t2), t2, 1, 0, 0, 30)
self.assertEqual(
corrID, KafkaCodec.get_response_correlation_id(encoded))
示例12: test_encode_message_header
def test_encode_message_header(self):
expect = b"".join([
struct.pack(">h", 10), # API Key
struct.pack(">h", 0), # API Version
struct.pack(">i", 4), # Correlation Id
struct.pack(">h", len("client3")), # Length of clientId
b"client3", # ClientId
])
encoded = KafkaCodec._encode_message_header(b"client3", 4, 10)
self.assertEqual(encoded, expect)
示例13: test_decode_produce_response
def test_decode_produce_response(self):
t1 = "topic1"
t2 = u"topic2"
encoded = struct.pack('>iih%dsiihqihqh%dsiihq' % (len(t1), len(t2)),
2, 2, len(t1), t1.encode(), 2, 0, 0, 10, 1, 1, 20,
len(t2), t2.encode(), 1, 0, 0, 30)
responses = list(KafkaCodec.decode_produce_response(encoded))
self.assertEqual(responses,
[ProduceResponse(t1, 0, 0, 10),
ProduceResponse(t1, 1, 1, 20),
ProduceResponse(t2, 0, 0, 30)])
示例14: test_decode_consumermetadata_response
def test_decode_consumermetadata_response(self):
expected = ConsumerMetadataResponse(0, 5, "theHost", 4242)
encoded = b"".join([
struct.pack('>i', 9), # Correlation ID
struct.pack('>h', 0), # Error Code
struct.pack('>i', 5), # Coordinator ID
struct.pack('>h', len(b"theHost")), b"theHost", # Coordinator Host
struct.pack('>i', 4242), # Coordinator port
])
decoded = KafkaCodec.decode_consumermetadata_response(encoded)
self.assertEqual(decoded, expected)
示例15: test_encode_metadata_request_no_topics
def test_encode_metadata_request_no_topics(self):
expected = b"".join([
struct.pack('>h', 3), # API key metadata fetch
struct.pack('>h', 0), # API version
struct.pack('>i', 4), # Correlation ID
struct.pack('>h3s', 3, b"cid"), # The client ID
struct.pack('>i', 0), # No topics, give all the data!
])
encoded = KafkaCodec.encode_metadata_request(b"cid", 4)
self.assertEqual(encoded, expected)