本文整理汇总了Python中kafka.KafkaConsumer.poll方法的典型用法代码示例。如果您正苦于以下问题:Python KafkaConsumer.poll方法的具体用法?Python KafkaConsumer.poll怎么用?Python KafkaConsumer.poll使用的例子?那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类kafka.KafkaConsumer
的用法示例。
在下文中一共展示了KafkaConsumer.poll方法的12个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: start
# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import poll [as 别名]
def start(self):
kafka_brokers = '{0}:{1}'.format(self._server,self._port)
consumer = KC(bootstrap_servers=[kafka_brokers],group_id=self._topic)
partition = [TopicPartition(self._topic,int(self._id))]
consumer.assign(partitions=partition)
consumer.poll()
return consumer
示例2: __consume_topic_with_func
# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import poll [as 别名]
def __consume_topic_with_func(self, topic, func):
consumer = KafkaConsumer(topic,
client_id='fooltrader',
group_id=self.trader_id,
value_deserializer=lambda m: json.loads(m.decode('utf8')),
bootstrap_servers=[KAFKA_HOST])
topic_partition = TopicPartition(topic=topic, partition=0)
start_timestamp = int(self.start_date.timestamp())
partition_map_offset_and_timestamp = consumer.offsets_for_times({topic_partition: start_timestamp})
if partition_map_offset_and_timestamp:
offset_and_timestamp = partition_map_offset_and_timestamp[topic_partition]
if offset_and_timestamp:
# partition assigned after poll, and we could seek
consumer.poll(5, 1)
consumer.seek(topic_partition, offset_and_timestamp.offset)
end_offset = consumer.end_offsets([topic_partition])[topic_partition]
consuming_time = self.current_time
for message in consumer:
message_time = pd.Timestamp(message.value['timestamp'])
# 设定了结束日期的话,时间到了或者kafka没数据了就结束
if self.end_date and (message_time > self.end_date or message.offset + 1 == end_offset):
consumer.close()
break
# 收到的时间戳与消费了的时间戳比较
time_delta = message_time.date() - consuming_time.date()
# 为了准确计算当天收盘账户,必须等各级别都把当天的行情撸完了
if time_delta.days >= 1:
self.barrier.wait()
self.account_service.save_account(self.current_time, trading_close=True)
getattr(self, func)(message.value)
consuming_time = message_time
# 时间以最小级别为准
if self.level_step.get(func) == self.step:
self.current_time = message_time
else:
consumer.poll(5, 1)
consumer.seek(topic_partition, consumer.end_offsets([topic_partition])[topic_partition] - 1)
message = consumer.poll(5000, 1)
kafka_start_date = datetime.fromtimestamp(message[topic_partition][0].timestamp).strftime(
TIME_FORMAT_DAY)
self.logger.warn("start:{} is after the last record:{}".format(self.start_date, kafka_start_date))
示例3: run
# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import poll [as 别名]
def run(self):
consumer = KafkaConsumer(bootstrap_servers='localhost:9092',
auto_offset_reset='latest')
consumer.subscribe(['parking-test-summary'])
while True:
msg = consumer.poll(10)
time.sleep(5)
print msg
示例4: from_kafka
# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import poll [as 别名]
def from_kafka(args, mgr):
consumer = KafkaConsumer(kafka_param["topic"],
group_id=kafka_param["group_id"],
bootstrap_servers=kafka_param["bootstrap.servers"],
auto_offset_reset="earliest",
enable_auto_commit=False
)
max_records = args["max_records"]
no_message_count = 0
no_message_time = 5
try:
stop_count = 0
fail_msg_count = 0
while True:
messages = consumer.poll(timeout_ms=1000, max_records=max_records)
queue = mgr.get_queue("input")
group_msgs_count = 0
group_msgs = []
for tp, records in messages.items():
for record in records:
try:
with io.BytesIO(record.value) as f:
msg_value = pickle.load(f)
if msg_value == "_stop_":
stop_count += 1
else:
group_msgs.append(msg_value)
group_msgs_count += 1
except:
fail_msg_count += 0
print("unpickle from kafka fail")
sys.stdout.flush()
pass
if len(group_msgs) > 0:
no_message_count = 0
queue.put(group_msgs, block=True)
if len(group_msgs) == 0 and no_message_count < 10:
time.sleep(no_message_time)
no_message_count += 1
if (stop_count >= internal_system_param["stopFlagNum"] and group_msgs_count == 0) or (
no_message_count >= 10 and group_msgs_count == 0):
queue.put(["_stop_"], block=True)
print(
"no message from kafka, send _stop_ message. no_message_count={},stop_count={},stopFlagNum={}".format(
no_message_count, stop_count, internal_system_param["stopFlagNum"]))
sys.stdout.flush()
break
finally:
consumer.close()
示例5: poll
# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import poll [as 别名]
def poll(topic, offset=0, hostname=None, port_num=None, max_timeout=100):
hostname, port_num = insure_host_port(hostname, port_num)
server = hostname+':'+str(port_num)
topic_partition = TopicPartition(topic, partition)
consumer = KafkaConsumer(bootstrap_servers=server, group_id=None)
consumer.assign([topic_partition])
consumer.seek(topic_partition, offset)
msgs = consumer.poll(max_timeout).values()
consumer.close()
if len(msgs) > 0:
return msgs[0]
else:
return {}
示例6: KafkaAgent
# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import poll [as 别名]
#.........这里部分代码省略.........
Returns: None
Note:
Callback method for subscribing.
Subscribe message topic: 'command-to-cloud' send command to cloud,
producer(KafkaAgent)-> kafka broker(Cloud) -> consumer(Cloud)
Created: SungonLee, 2017-10-20
Deleted: .
'''
try:
msg = {
'message_sender': sender,
'kafka_message_sender': 'KafkaAgent',
'description': 'message from VOLTTRON to KafkaBroker',
'message': message
}
# Send command to Consumer(in Cloud)
self.producer.send(self.kafka_consumer_topic, msg)
except Exception as e:
_log.error('Send_to_broker: {}'.format(e))
@Core.receiver("onstart")
def on_message_topic(self, sender, **kwargs):
'''
Function: Resister callback method for sending data to Kafka Broker.
Args: .
Returns: None
Note:
This method is executed after '__init__' method.
Subscribes to the platform message bus on the actuator, record, datalogger, and device topics.
Created: SungonLee, 2017-10-20
Deleted: .
'''
_log.debug("sender {}, Kwargs {}".format(sender, kwargs))
# Define method for resistering callback method
def subscriber(subscription, callback_method):
'''
Args:
subscription: topic (e.g. "devices/fake-campus/fake-building/fake-device/PowerState")
callback_method: method resistered
Note:
callback_mothod: 'post_data', 'send_to_broker'
'''
_log.debug("Subscribing to topic : {}".format(subscription))
self.vip.pubsub.subscribe(peer='pubsub',
prefix=subscription,
callback=callback_method)
# Resister callback method with 'subscriber'
for topic_subscriptions in self.services_topic_list:
subscriber(topic_subscriptions, self.send_to_broker)
@Core.periodic(1)
def receive_from_broker(self):
'''
Function: Receive message from Kafka broker and Publish message to MessageBus.
Args: None
Returns: None
Note: None
Created: SungonLee, 2017-10-20
Deleted: .
'''
# partition type : nametuple
# if timeout_ms is 0, check that is there any message in broker imm
partition = self.consumer.poll(timeout_ms=0, max_records=None)
try:
if len(partition) > 0:
for p in partition:
for response in partition[p]:
# _log.info('Receive_from_broker: {}'.format(response))
# convert string to dictionary
response_dict = ast.literal_eval(response.value)
# _log.info('Receive_from_broker: Receive message from kafka broker message: {}'.format(response_dict))
topic = response.topic
# sender = response_dict['sender']
headers = {
'date': str(datetime.datetime.now())
}
message = {
# 'kafka_message_sender': sender,
# 'kafka_message_receiver':'KafkaAgent',
'message': response_dict
}
self.vip.pubsub.publish('pubsub', topic, headers, message)
else:
pass
# _log.info('Receive_from_broker: No receive message from kafka broker')
except Exception as e:
_log.error('Receive_from_broker: {}'.format(e))
示例7: CheckKafka
# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import poll [as 别名]
#.........这里部分代码省略.........
try:
if list_partitions:
if self.topic:
self.print_topic_partitions(self.topic)
else:
for topic in self.get_topics():
self.print_topic_partitions(topic)
sys.exit(ERRORS['UNKNOWN'])
except KafkaError:
raise CriticalError(self.exception_msg())
self.partition = self.get_opt('partition')
# technically optional, will hash to a random partition, but need to know which partition to get offset
# if self.partition is not None:
validate_int(self.partition, "partition", 0, 10000)
self.topic_partition = TopicPartition(self.topic, self.partition)
self.acks = self.get_opt('acks')
try:
self.acks = int(self.acks)
except ValueError:
pass
log_option('acks', self.acks)
self.validate_thresholds()
def subscribe(self):
self.consumer = KafkaConsumer(
#self.topic,
bootstrap_servers=self.brokers,
# client_id=self.client_id,
# group_id=self.group_id,
request_timeout_ms=self.timeout_ms
)
#key_serializer
#value_serializer
# this is only a guess as Kafka doesn't expose it's API version
#log.debug('kafka api version: %s', self.consumer.config['api_version'])
log.debug('partition assignments: {0}'.format(self.consumer.assignment()))
# log.debug('subscribing to topic \'{0}\' parition \'{1}\''.format(self.topic, self.partition))
# self.consumer.subscribe(TopicPartition(self.topic, self.partition))
# log.debug('partition assignments: {0}'.format(self.consumer.assignment()))
log.debug('assigning partition {0} to consumer'.format(self.partition))
# self.consumer.assign([self.partition])
self.consumer.assign([self.topic_partition])
log.debug('partition assignments: {0}'.format(self.consumer.assignment()))
log.debug('getting current offset')
# see also highwater, committed, seek_to_end
self.start_offset = self.consumer.position(self.topic_partition)
if self.start_offset is None:
# don't do this, I've seen scenario where None is returned and all messages are read again, better to fail
# log.warn('consumer position returned None, resetting to zero')
# self.start_offset = 0
raise UnknownError('Kafka Consumer reported current starting offset = {0}'.format(self.start_offset))
log.debug('recorded starting offset \'{0}\''.format(self.start_offset))
# self.consumer.pause()
def publish(self):
log.debug('creating producer')
self.producer = KafkaProducer(
bootstrap_servers=self.brokers,
client_id=self.client_id,
acks=self.acks,
batch_size=0,
max_block_ms=self.timeout_ms,
request_timeout_ms=self.timeout_ms
)
#key_serializer
#value_serializer
log.debug('producer.send()')
self.producer.send(
self.topic,
key=self.key,
partition=self.partition,
value=self.publish_message
)
log.debug('producer.flush()')
self.producer.flush()
def consume(self):
self.consumer.assign([self.topic_partition])
log.debug('consumer.seek({0})'.format(self.start_offset))
self.consumer.seek(self.topic_partition, self.start_offset)
# self.consumer.resume()
log.debug('consumer.poll(timeout_ms={0})'.format(self.timeout_ms))
obj = self.consumer.poll(timeout_ms=self.timeout_ms)
log.debug('msg object returned: %s', obj)
msg = None
try:
for consumer_record in obj[self.topic_partition]:
if consumer_record.key == self.key:
msg = consumer_record.value
break
except KeyError:
raise UnknownError('TopicPartition key was not found in response')
if msg is None:
raise UnknownError("failed to find matching consumer record with key '{0}'".format(self.key))
return msg
示例8: KafkaConsumer
# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import poll [as 别名]
import sys
import argparse
from kafka import KafkaConsumer
parser = argparse.ArgumentParser(description='Zoe Kafka log viewer')
parser.add_argument('kafka_address', help='Address of the Kafka broker')
parser.add_argument('--list-logs', action='store_true', help='List all the available service logs')
parser.add_argument('--topic', help='Service name to fetch and monitor for activity')
args = parser.parse_args()
consumer = KafkaConsumer(bootstrap_servers=args.kafka_address)
if args.list_logs:
for topic in consumer.topics():
if topic[0] != '_':
print(topic)
sys.exit(0)
consumer.subscribe(pattern=args.topic)
consumer.poll(1)
consumer.seek_to_beginning()
try:
for msg in consumer:
print(msg.value.decode('utf-8'))
except KeyboardInterrupt:
print('showlog exiting...')
示例9: MessagehubStreamingAdapter
# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import poll [as 别名]
class MessagehubStreamingAdapter(StreamingDataAdapter):
def __init__(self, topic, username, password, prod=True):
# Create a new context using system defaults, disable all but TLS1.2
context = ssl.create_default_context()
context.options &= ssl.OP_NO_TLSv1
context.options &= ssl.OP_NO_TLSv1_1
conf = {
'client_id': 'pixieapp.client.id',
'group_id': 'pixieapp.group',
'sasl_mechanism': 'PLAIN',
'security_protocol': 'SASL_SSL',
'ssl_context': context,
"bootstrap_servers": [ "kafka0{}-{}.messagehub.services.us-south.bluemix.net:9093".format(i, "prod01" if prod else "stage1") for i in range(1,6)],
"sasl_plain_username": username,
"sasl_plain_password": password,
"auto_offset_reset":"latest"
}
self.consumer = KafkaConsumer(**conf)
self.consumer.subscribe([topic])
self.schema = {}
self.sampleDocCount = 0
def close(self):
self.consumer.unsubscribe()
self.consumer.close()
def tryCast(self, value, t):
def _innerTryCast(value, t):
try:
return t(value)
except:
return None
if isinstance(t, tuple):
for a in t:
ret = _innerTryCast(value, a)
if ret is not None:
return ret
return None
return _innerTryCast(value, t)
def inferType(self, value):
if isinstance(value, string_types):
value = self.tryCast(value, integer_types) or self.tryCast(value, float) or value
return "integer" if value.__class__==int else "float" if value.__class__ == float else "string"
def inferSchema(self, eventJSON):
if self.sampleDocCount > 20:
return
for key,value in iteritems(eventJSON):
if not key in self.schema:
self.schema[key] = self.inferType(value)
self.sampleDocCount = self.sampleDocCount + 1
def doGetNextData(self):
msgs = []
msg = self.consumer.poll(1000, max_records=10)
if msg is not None:
for topicPartition,records in iteritems(msg):
for record in records:
if record.value is not None:
jsonValue = json.loads(record.value.decode('utf-8'))
self.inferSchema(jsonValue)
msgs.append(jsonValue)
return msgs
def close(self):
self.consumer.close()
示例10: main
# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import poll [as 别名]
def main():
parser = argparse.ArgumentParser()
parser.add_argument("topic")
parser.add_argument("-H", "--host", type=str, help="Kafka server and port.",
default="localhost:9092")
parser.add_argument("-r", "--replay", action="store_true",
help="Display all available log entries.",
default=False)
parser.add_argument("-m", "--match", type=str, help="Initial match pattern.",
default=None)
args = parser.parse_args()
pattern = args.match
if args.replay:
auto_offset_reset = 'earliest'
else:
auto_offset_reset = 'latest'
if args.topic[-5:] == '.json':
value_deserializer = json_value_deserializer
else:
value_deserializer = None
consumer = KafkaConsumer(args.topic,
group_id=None,
bootstrap_servers=args.host,
value_deserializer=value_deserializer,
auto_offset_reset=auto_offset_reset)
while True:
messages = consumer.poll(250)
for tp in six.itervalues(messages):
for message in tp:
if isinstance(message.value, dict):
if message.value['klog_level'] in colors:
c = colors[message.value['klog_level']]
else:
c = attr(0)
params = {'topic': message.topic,
'offset': message.offset,
'level': message.value['klog_level'].upper()}
params['time'] = str(datetime.datetime.fromtimestamp(float(message.value['klog_time'])))
params['msg'] = message.value['klog_message']
if pattern and re.search(pattern, params['msg']) is not None:
c += match_color
msg = msg_format.format(**params)
else:
c = attr(0)
msg = message.value
print(c+msg+attr(0))
po = select.poll()
po.register(sys.stdin, select.POLLIN)
if po.poll(0):
ch = sys.stdin.read(1)
if ch == 'm':
pattern = sys.stdin.readline().rstrip('\n').encode('utf-8')
pattern = pattern.rstrip('\n').encode('utf-8')
elif ch == 'r':
offset = sys.stdin.readline().rstrip('\n').encode('utf-8')
offset = int(offset)
for tp in consumer.assignment():
position = consumer.position(tp)
consumer.seek(tp, max(0, position-offset))
elif ch == 'R':
for tp in consumer.assignment():
consumer.seek_to_beginning(tp)
elif ch == 'p':
for tp in consumer.assignment():
consumer.pause(tp)
elif ch == 'P':
for tp in consumer.assignment():
consumer.resume(tp)
elif ch == 'q':
# FIXME: kafka currently (1.0.1) raises an exception on close
#consumer.close()
exit()
示例11: _run
# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import poll [as 别名]
def _run(self):
pcount = 0
pause = False
self.part_prev_time = 0
while True:
try:
if pause:
gevent.sleep(5)
pause = False
self._logger.error("Newer KafkaClient %s" % self._topic)
self._failed = False
try:
consumer = KafkaConsumer(self._topic,
bootstrap_servers=self._brokers.split(','),
group_id=None)
except Exception as ex:
self.part_cur_time = time.time()
if slef.part_prev_time == 0 or self.part_cur_time - self.part_prev_time > 60:
self.part_prev_time = self.part_cur_time
template = "Consumer Failure {0} occured. Arguments:\n{1!r}"
messag = template.format(type(ex).__name__, ex.args)
self._logger.error("Error: %s trace %s" % \
(messag, traceback.format_exc()))
else:
template = "Consumer Failure {0} occured. Arguments:\n{1!r}"
messag = template.format(type(ex).__name__, ex.args)
self._logger.error("Error: %s" % (messag))
self._failed = True
raise RuntimeError(messag)
self._logger.error("Starting %s" % self._topic)
if self._limit:
raise gevent.GreenletExit
while True:
try:
mdict = consumer.poll()
self.resource_check()
if len(mdict):
counts = {}
for tp,tv in mdict.iteritems():
if tp not in counts:
counts[tp] = 0
counts[tp] += len(tv)
if not self.msg_handler(tv):
raise gevent.GreenletExit
pcount += len(tv)
self._logger.debug("poll for topic %s : %s" % (self._topic, str(counts)))
else:
gevent.sleep(0.5)
except TypeError as ex:
self._logger.error("Type Error: %s trace %s" % \
(str(ex.args), traceback.format_exc()))
gevent.sleep(0.1)
except common.FailedPayloadsError as ex:
self._logger.error("Payload Error: %s" % str(ex.args))
gevent.sleep(0.1)
except gevent.GreenletExit:
break
except AssertionError as ex:
self._partoffset = ex
break
except Exception as ex:
self.part_cur_time = time.time()
if self.part_prev_time == 0 or self.part_cur_time - self.part_prev_time > 60:
self.part_prev_time = self.part_cur_time
template = "An exception of type {0} occured. Arguments:\n{1!r}"
messag = template.format(type(ex).__name__, ex.args)
self._logger.error("%s %s : traceback %s" % \
(self._topic, messag, traceback.format_exc()))
else:
template = "An exception of type {0} occured. Arguments:\n{1!r}"
messag = template.format(type(ex).__name__, ex.args)
self._logger.error("%s %s" % (self._topic, messag))
self.stop_partition()
self._failed = True
pause = True
if hasattr(ex,'errno'):
# This is an unrecoverable error
if ex.errno == errno.EMFILE:
raise SystemExit(1)
self._logger.error("Stopping %s pcount %d" % (self._topic, pcount))
partdb = self.stop_partition()
return self._partoffset, partdb
示例12: KafkaConsumer
# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import poll [as 别名]
from kafka import KafkaConsumer
import json
import ast
import time
# value_deserializer=lambda m: json.loads(m).decode('utf-8')
consumer = KafkaConsumer(bootstrap_servers=['localhost:9092'],
# group_id='my-group-2',
)
consumer.subscribe(['msg-from-volttron'])
cnt = 0
while True:
partition = consumer.poll(timeout_ms=1000, max_records=None)
# obj : type dict
if len(partition) > 0:
print('poll - receive')
for p in partition:
for response in partition[p]:
print('poll topic: {}'.format(response.topic))
# print('poll value: {}'.format(response.value))
# string to dict
# dic_value = ast.literal_eval(response.value)
print('poll value: {}'.format(response.value))
# print('poll value: {}, new_value: {}'.format(dic_value, dic_value['new_value']))
# print('poll value type: {}'.format(type(dic_value)))
else:
print('poll - no receive yet {}'.format(cnt))
cnt += 1