本文整理汇总了Python中kafka.KafkaConsumer._use_consumer_group方法的典型用法代码示例。如果您正苦于以下问题：Python KafkaConsumer._use_consumer_group方法的具体用法？Python KafkaConsumer._use_consumer_group怎么用？Python KafkaConsumer._use_consumer_group使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类kafka.KafkaConsumer的用法示例。
在下文中一共展示了KafkaConsumer._use_consumer_group方法的1个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: Consumer
# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import _use_consumer_group [as 别名]
class Consumer(BaseStreamConsumer):
"""
Used in DB and SW worker. SW consumes per partition.
"""
def __init__(self, location, topic, group, partition_id):
self._location = location
self._group = group
self._topic = topic
self._consumer = KafkaConsumer(
bootstrap_servers=self._location,
group_id=self._group,
max_partition_fetch_bytes=10485760,
consumer_timeout_ms=100,
client_id="%s-%s" % (self._topic, str(partition_id) if partition_id is not None else "all"),
request_timeout_ms=120 * 1000,
)
if partition_id is not None:
self._partition_ids = [TopicPartition(self._topic, partition_id)]
self._consumer.assign(self._partition_ids)
else:
self._partition_ids = [TopicPartition(self._topic, pid) for pid in self._consumer.partitions_for_topic(self._topic)]
self._consumer.subscribe(topics=[self._topic])
if self._consumer._use_consumer_group():
self._consumer._coordinator.ensure_coordinator_known()
self._consumer._coordinator.ensure_active_group()
self._consumer._update_fetch_positions(self._partition_ids)
self._start_looping_call()
def _start_looping_call(self, interval=60):
def errback(failure):
logger.exception(failure.value)
if failure.frames:
logger.critical(str("").join(format_tb(failure.getTracebackObject())))
self._poll_task.start(interval).addErrback(errback)
self._poll_task = LoopingCall(self._poll_client)
self._poll_task.start(interval).addErrback(errback)
def _poll_client(self):
self._consumer._client.poll()
def get_messages(self, timeout=0.1, count=1):
result = []
while count > 0:
try:
m = next(self._consumer)
result.append(m.value)
count -= 1
except StopIteration:
break
return result
def get_offset(self, partition_id):
for tp in self._partition_ids:
if tp.partition == partition_id:
return self._consumer.position(tp)
raise KeyError("Can't find partition %d", partition_id)
def close(self):
self._poll_task.stop()
self._consumer.commit()
# getting kafka client event loop running some more and execute commit
tries = 3
while tries:
self.get_messages()
sleep(2.0)
tries -= 1
self._consumer.close()