本文整理汇总了Python中kafka.KeyedProducer类的典型用法代码示例。如果您正苦于以下问题：Python KeyedProducer类的具体用法？Python KeyedProducer怎么用？Python KeyedProducer使用的例子？那么恭喜您, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了KeyedProducer类的15个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: __init__
def __init__(self, config):
self.brokers = config['brokers']
self.topic = config['topic']
self.kafka = KafkaClient(self.brokers)
if config['partitioner'] is None:
self.producer = KeyedProducer(self.kafka, partitioner=RoundRobinPartitioner)
else:
self.producer = KeyedProducer(self.kafka, partitioner=config['partitioner'])示例2: sendMsg
def sendMsg(topic, lines):
if lines.__len__() > 0:
brokers = '10.117.181.44:9092,10.117.108.143:9092,10.117.21.79:9092'
kafka = KafkaClient(brokers)
producer = KeyedProducer(kafka)
for line in lines:
ran = "_" + str(random.randint(0, 10))
producer.send_messages(topic, topic + ran, line)
producer.stop()示例3: genData
def genData(topic):
producer = KeyedProducer(kafka)
while True:
with open(source_file) as f:
for line in f:
key = line.split(" ")[0]
producer.send(topic, key, line.rstrip())
time.sleep(0.1) # Creating some delay to allow proper rendering of the cab locations on the map
source_file.close()示例4: test_async_keyed_producer
def test_async_keyed_producer(self):
start_offset0 = self.current_offset(self.topic, 0)
producer = KeyedProducer(self.client, partitioner = RoundRobinPartitioner, async=True)
resp = producer.send(self.topic, self.key("key1"), self.msg("one"))
self.assertEqual(len(resp), 0)
self.assert_fetch_offset(0, start_offset0, [ self.msg("one") ])
producer.stop()示例5: keyedProduce
def keyedProduce(self,topic, key, value):
kafka=KafkaClient(self.configs["broker_list"].split(","))
keyedProducer=KeyedProducer(kafka,async=True)
undone=True
while(undone):
try:
keyedProducer.send_messages(topic, key, value)
undone=False
except LeaderNotAvailableError:
sleep(10)
print("LeaderNotAvailableError")
pass示例6: NautilusDive
class NautilusDive(object):
def __init__(self, config):
self.brokers = config['brokers']
self.topic = config['topic']
self.kafka = KafkaClient(self.brokers)
if config['partitioner'] is None:
self.producer = KeyedProducer(self.kafka, partitioner=RoundRobinPartitioner)
else:
self.producer = KeyedProducer(self.kafka, partitioner=config['partitioner'])
def send(self, key, message):
self.producer.send(self.topic, key, message)示例7: test_async_keyed_producer
def test_async_keyed_producer(self):
partition = self.client.get_partition_ids_for_topic(self.topic)[0]
start_offset = self.current_offset(self.topic, partition)
producer = KeyedProducer(self.client, partitioner = RoundRobinPartitioner, async=True)
resp = producer.send_messages(self.topic, self.key("key1"), self.msg("one"))
self.assertEqual(len(resp), 0)
# wait for the server to report a new highwatermark
while self.current_offset(self.topic, partition) == start_offset:
time.sleep(0.1)
self.assert_fetch_offset(partition, start_offset, [ self.msg("one") ])
producer.stop()示例8: test_keyedproducer_message_types
def test_keyedproducer_message_types(self):
client = MagicMock()
client.get_partition_ids_for_topic.return_value = [0, 1]
producer = KeyedProducer(client)
topic = b"test-topic"
key = b"testkey"
bad_data_types = (u"你怎么样?", 12, ["a", "list"], ("a", "tuple"), {"a": "dict"})
for m in bad_data_types:
with self.assertRaises(TypeError):
logging.debug("attempting to send message of type %s", type(m))
producer.send_messages(topic, key, m)
good_data_types = (b"a string!", None)
for m in good_data_types:
# This should not raise an exception
producer.send_messages(topic, key, m)示例9: keyedProducerTest2
def keyedProducerTest2():
'''test KeyedProducer
@topic：单replica情况(JOB_TEST_1)
@function:测试KeyedProducer，向指定的broker发布消息，
并验证develops-dev1:9193关闭之后的异常报错
'''
import pdb
pdb.set_trace()
kafkaClient = KafkaClient('devops-dev1:9193')
producer = KeyedProducer(kafkaClient)
message = "This is a test-"
index = 0
while True:
tmpmsg = message + str(index)
producer.send_messages(b'JOB_TEST_1', 'keys', tmpmsg)
index += 1
time.sleep(1)示例10: kafkaTasks
def kafkaTasks(self, addr, topic,tasks):
try :
from kafka import SimpleProducer, KafkaClient, KeyedProducer
except:
logger.error("kafka-python is not installed")
raise Exception("kafka-python is not installed")
kafka_client = None
try :
kafka_client = KafkaClient(addr)
producer = KeyedProducer(kafka_client)
for task in tasks:
#self.producer.send_messages(self.warehouse,task.id, json.dumps(task,default=object2dict))
producer.send_messages(topic, self.manager.name, cPickle.dumps(task))
finally:
if kafka_client:
kafka_client.close()示例11: KeyedProducer
class KeyedProducer(BaseStreamProducer):
def __init__(self, connection, topic_done, partitioner_cls, codec):
self._prod = None
self._conn = connection
self._topic_done = topic_done
self._partitioner_cls = partitioner_cls
self._codec = codec
def _connect_producer(self):
if self._prod is None:
try:
self._prod = KafkaKeyedProducer(self._conn, partitioner=self._partitioner_cls, codec=self._codec)
except BrokerResponseError:
self._prod = None
logger.warning("Could not connect producer to Kafka server")
return False
return True
def send(self, key, *messages):
success = False
max_tries = 5
if self._connect_producer():
n_tries = 0
while not success and n_tries < max_tries:
try:
self._prod.send_messages(self._topic_done, key, *messages)
success = True
except MessageSizeTooLargeError as e:
logger.error(str(e))
break
except BrokerResponseError:
n_tries += 1
logger.warning(
"Could not send message. Try {0}/{1}".format(
n_tries, max_tries)
)
sleep(1.0)
return success
def flush(self):
if self._prod is not None:
self._prod.stop()
def get_offset(self, partition_id):
# Kafka has it's own offset management
raise KeyError示例12: keyed_messages
def keyed_messages():
'''Keyed messages'''
from kafka import (KafkaClient, KeyedProducer,
Murmur2Partitioner, RoundRobinPartitioner)
kafka = KafkaClient(KAFKA_SERVER)
# HashedPartitioner is default (currently uses python hash())
producer = KeyedProducer(kafka)
producer.send_messages(b'topic1', b'key1', b'some message')
producer.send_messages(b'topic1', b'key2', b'this methode')
# Murmur2Partitioner attempts to mirror the java client hashing
producer = KeyedProducer(kafka, partitioner=Murmur2Partitioner)
# Or just produce round-robin (or just use SimpleProducer)
producer = KeyedProducer(kafka, partitioner=RoundRobinPartitioner)示例13: _connect_producer
def _connect_producer(self):
if self._prod is None:
try:
self._prod = KafkaKeyedProducer(self._conn, partitioner=self._partitioner_cls, codec=CODEC_SNAPPY)
except BrokerResponseError:
self._prod = None
logger.warning("Could not connect producer to Kafka server")
return False
return True示例14: test_keyedproducer_message_types
def test_keyedproducer_message_types(self):
client = MagicMock()
client.get_partition_ids_for_topic.return_value = [0, 1]
producer = KeyedProducer(client)
topic = b"test-topic"
key = b"testkey"
bad_data_types = (u'你怎么样?', 12, ['a', 'list'],
('a', 'tuple'), {'a': 'dict'},)
for m in bad_data_types:
with self.assertRaises(TypeError):
logging.debug("attempting to send message of type %s", type(m))
producer.send_messages(topic, key, m)
good_data_types = (b'a string!', None,)
for m in good_data_types:
# This should not raise an exception
producer.send_messages(topic, key, m)示例15: KafkaBolt
class KafkaBolt(Bolt):
def initialize(self, stormconf, ctx):
self.kafka_client = KafkaClient(config['kafka']['hosts'])
self.keyed_producer = KeyedProducer(self.kafka_client)
self.simple_producer = SimpleProducer(self.kafka_client)
def process(self, tup):
report_id, record_type, report_data = tup.values
self.log('Processing: %s' % report_id)
json_data = str(report_data)
report_id = str(report_id)
topic = str("sanitised")
if record_type == "entry":
payload = str("e" + json_data)
elif record_type == "header":
payload = str("h" + json_data)
elif record_type == "footer":
payload = str("f" + json_data)
self.keyed_producer.send(topic, report_id, payload)