Python KafkaConsumer.poll方法代码示例

# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import poll [as 别名]
 def start(self):
     
     kafka_brokers = '{0}:{1}'.format(self._server,self._port)
     consumer =  KC(bootstrap_servers=[kafka_brokers],group_id=self._topic)
     partition = [TopicPartition(self._topic,int(self._id))]
     consumer.assign(partitions=partition)
     consumer.poll()
     return consumer

# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import poll [as 别名]
    def __consume_topic_with_func(self, topic, func):
        consumer = KafkaConsumer(topic,
                                 client_id='fooltrader',
                                 group_id=self.trader_id,
                                 value_deserializer=lambda m: json.loads(m.decode('utf8')),
                                 bootstrap_servers=[KAFKA_HOST])
        topic_partition = TopicPartition(topic=topic, partition=0)
        start_timestamp = int(self.start_date.timestamp())

        partition_map_offset_and_timestamp = consumer.offsets_for_times({topic_partition: start_timestamp})

        if partition_map_offset_and_timestamp:
            offset_and_timestamp = partition_map_offset_and_timestamp[topic_partition]

            if offset_and_timestamp:
                # partition  assigned after poll, and we could seek
                consumer.poll(5, 1)
                consumer.seek(topic_partition, offset_and_timestamp.offset)
                end_offset = consumer.end_offsets([topic_partition])[topic_partition]
                consuming_time = self.current_time
                for message in consumer:
                    message_time = pd.Timestamp(message.value['timestamp'])
                    # 设定了结束日期的话,时间到了或者kafka没数据了就结束
                    if self.end_date and (message_time > self.end_date or message.offset + 1 == end_offset):
                        consumer.close()
                        break

                    # 收到的时间戳与消费了的时间戳比较
                    time_delta = message_time.date() - consuming_time.date()

                    # 为了准确计算当天收盘账户,必须等各级别都把当天的行情撸完了
                    if time_delta.days >= 1:
                        self.barrier.wait()
                        self.account_service.save_account(self.current_time, trading_close=True)

                    getattr(self, func)(message.value)

                    consuming_time = message_time
                    # 时间以最小级别为准
                    if self.level_step.get(func) == self.step:
                        self.current_time = message_time

            else:
                consumer.poll(5, 1)
                consumer.seek(topic_partition, consumer.end_offsets([topic_partition])[topic_partition] - 1)
                message = consumer.poll(5000, 1)
                kafka_start_date = datetime.fromtimestamp(message[topic_partition][0].timestamp).strftime(
                    TIME_FORMAT_DAY)
                self.logger.warn("start:{} is after the last record:{}".format(self.start_date, kafka_start_date))

# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import poll [as 别名]
 def run(self):
     consumer = KafkaConsumer(bootstrap_servers='localhost:9092',
                              auto_offset_reset='latest')
     consumer.subscribe(['parking-test-summary'])
     while True:
         msg = consumer.poll(10)
         time.sleep(5)
         print msg

# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import poll [as 别名]
    def from_kafka(args, mgr):
        consumer = KafkaConsumer(kafka_param["topic"],
                                 group_id=kafka_param["group_id"],
                                 bootstrap_servers=kafka_param["bootstrap.servers"],
                                 auto_offset_reset="earliest",
                                 enable_auto_commit=False
                                 )

        max_records = args["max_records"]
        no_message_count = 0
        no_message_time = 5
        try:
            stop_count = 0
            fail_msg_count = 0
            while True:
                messages = consumer.poll(timeout_ms=1000, max_records=max_records)
                queue = mgr.get_queue("input")
                group_msgs_count = 0
                group_msgs = []
                for tp, records in messages.items():
                    for record in records:
                        try:
                            with io.BytesIO(record.value) as f:
                                msg_value = pickle.load(f)
                            if msg_value == "_stop_":
                                stop_count += 1
                            else:
                                group_msgs.append(msg_value)
                                group_msgs_count += 1
                        except:
                            fail_msg_count += 0
                            print("unpickle from kafka fail")
                            sys.stdout.flush()
                            pass
                if len(group_msgs) > 0:
                    no_message_count = 0
                    queue.put(group_msgs, block=True)

                if len(group_msgs) == 0 and no_message_count < 10:
                    time.sleep(no_message_time)
                    no_message_count += 1

                if (stop_count >= internal_system_param["stopFlagNum"] and group_msgs_count == 0) or (
                                no_message_count >= 10 and group_msgs_count == 0):
                    queue.put(["_stop_"], block=True)
                    print(
                        "no message from kafka, send _stop_ message. no_message_count={},stop_count={},stopFlagNum={}".format(
                            no_message_count, stop_count, internal_system_param["stopFlagNum"]))
                    sys.stdout.flush()
                    break
        finally:
            consumer.close()

# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import poll [as 别名]
def poll(topic, offset=0, hostname=None, port_num=None, max_timeout=100):
    hostname, port_num = insure_host_port(hostname, port_num)
    server = hostname+':'+str(port_num)
    topic_partition = TopicPartition(topic, partition)

    consumer = KafkaConsumer(bootstrap_servers=server, group_id=None)
    consumer.assign([topic_partition])
    consumer.seek(topic_partition, offset)
    msgs = consumer.poll(max_timeout).values()
    consumer.close()
    if len(msgs) > 0:
        return msgs[0]
    else:
        return {}

# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import poll [as 别名]

#.........这里部分代码省略.........
            Returns: None

            Note:
                Callback method for subscribing.
                Subscribe message topic: 'command-to-cloud' send command to cloud,
                                         producer(KafkaAgent)-> kafka broker(Cloud) -> consumer(Cloud)

            Created: SungonLee, 2017-10-20
            Deleted: .
        '''
        try:
            msg = {
                'message_sender': sender,
                'kafka_message_sender': 'KafkaAgent',
                'description': 'message from VOLTTRON to KafkaBroker',
                'message': message
                }
            # Send command to Consumer(in Cloud)
            self.producer.send(self.kafka_consumer_topic, msg)

        except Exception as e:
            _log.error('Send_to_broker: {}'.format(e))

    @Core.receiver("onstart")
    def on_message_topic(self, sender, **kwargs):
        '''
            Function: Resister callback method for sending data to Kafka Broker.

            Args: .

            Returns: None

            Note:
                This method is executed after '__init__' method.
                Subscribes to the platform message bus on the actuator, record, datalogger, and device topics.

            Created: SungonLee, 2017-10-20
            Deleted: .
        '''
        _log.debug("sender {}, Kwargs {}".format(sender, kwargs))

        # Define method for resistering callback method
        def subscriber(subscription, callback_method):
            '''
                Args:
                    subscription: topic (e.g. "devices/fake-campus/fake-building/fake-device/PowerState")
                    callback_method: method resistered

                Note:
                    callback_mothod: 'post_data', 'send_to_broker'
            '''
            _log.debug("Subscribing to topic : {}".format(subscription))
            self.vip.pubsub.subscribe(peer='pubsub',
                                      prefix=subscription,
                                      callback=callback_method)

        # Resister callback method with 'subscriber'
        for topic_subscriptions in self.services_topic_list:
            subscriber(topic_subscriptions, self.send_to_broker)

    @Core.periodic(1)
    def receive_from_broker(self):
        '''
            Function: Receive message from Kafka broker and Publish message to MessageBus.
            Args: None
            Returns: None
            Note: None
            Created: SungonLee, 2017-10-20
            Deleted: .
        '''
        # partition type : nametuple
        # if timeout_ms is 0, check that is there any message in broker imm
        partition = self.consumer.poll(timeout_ms=0, max_records=None)

        try:
            if len(partition) > 0:
                for p in partition:
                    for response in partition[p]:
                        # _log.info('Receive_from_broker: {}'.format(response))
                        # convert string to dictionary
                        response_dict = ast.literal_eval(response.value)
                        # _log.info('Receive_from_broker: Receive message from kafka broker message: {}'.format(response_dict))
                        topic = response.topic
                        # sender = response_dict['sender']
                        headers = {
                            'date': str(datetime.datetime.now())
                            }
                        message = {
                            # 'kafka_message_sender': sender,
                            # 'kafka_message_receiver':'KafkaAgent',
                            'message': response_dict
                            }

                        self.vip.pubsub.publish('pubsub', topic, headers, message)
            else:
                pass
                # _log.info('Receive_from_broker: No receive message from kafka broker')

        except Exception as e:
            _log.error('Receive_from_broker: {}'.format(e))

# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import poll [as 别名]

#.........这里部分代码省略.........

        try:
            if list_partitions:
                if self.topic:
                    self.print_topic_partitions(self.topic)
                else:
                    for topic in self.get_topics():
                        self.print_topic_partitions(topic)
                sys.exit(ERRORS['UNKNOWN'])
        except KafkaError:
            raise CriticalError(self.exception_msg())

        self.partition = self.get_opt('partition')
        # technically optional, will hash to a random partition, but need to know which partition to get offset
        # if self.partition is not None:
        validate_int(self.partition, "partition", 0, 10000)
        self.topic_partition = TopicPartition(self.topic, self.partition)
        self.acks = self.get_opt('acks')
        try:
            self.acks = int(self.acks)
        except ValueError:
            pass
        log_option('acks', self.acks)
        self.validate_thresholds()

    def subscribe(self):
        self.consumer = KafkaConsumer(
            #self.topic,
            bootstrap_servers=self.brokers,
            # client_id=self.client_id,
            # group_id=self.group_id,
            request_timeout_ms=self.timeout_ms
            )
            #key_serializer
            #value_serializer
        # this is only a guess as Kafka doesn't expose it's API version
        #log.debug('kafka api version: %s', self.consumer.config['api_version'])
        log.debug('partition assignments: {0}'.format(self.consumer.assignment()))

        # log.debug('subscribing to topic \'{0}\' parition \'{1}\''.format(self.topic, self.partition))
        # self.consumer.subscribe(TopicPartition(self.topic, self.partition))
        # log.debug('partition assignments: {0}'.format(self.consumer.assignment()))

        log.debug('assigning partition {0} to consumer'.format(self.partition))
        # self.consumer.assign([self.partition])
        self.consumer.assign([self.topic_partition])
        log.debug('partition assignments: {0}'.format(self.consumer.assignment()))

        log.debug('getting current offset')
        # see also highwater, committed, seek_to_end
        self.start_offset = self.consumer.position(self.topic_partition)
        if self.start_offset is None:
            # don't do this, I've seen scenario where None is returned and all messages are read again, better to fail
            # log.warn('consumer position returned None, resetting to zero')
            # self.start_offset = 0
            raise UnknownError('Kafka Consumer reported current starting offset = {0}'.format(self.start_offset))
        log.debug('recorded starting offset \'{0}\''.format(self.start_offset))
        # self.consumer.pause()

    def publish(self):
        log.debug('creating producer')
        self.producer = KafkaProducer(
            bootstrap_servers=self.brokers,
            client_id=self.client_id,
            acks=self.acks,
            batch_size=0,
            max_block_ms=self.timeout_ms,
            request_timeout_ms=self.timeout_ms
            )
            #key_serializer
            #value_serializer
        log.debug('producer.send()')
        self.producer.send(
            self.topic,
            key=self.key,
            partition=self.partition,
            value=self.publish_message
            )
        log.debug('producer.flush()')
        self.producer.flush()

    def consume(self):
        self.consumer.assign([self.topic_partition])
        log.debug('consumer.seek({0})'.format(self.start_offset))
        self.consumer.seek(self.topic_partition, self.start_offset)
        # self.consumer.resume()
        log.debug('consumer.poll(timeout_ms={0})'.format(self.timeout_ms))
        obj = self.consumer.poll(timeout_ms=self.timeout_ms)
        log.debug('msg object returned: %s', obj)
        msg = None
        try:
            for consumer_record in obj[self.topic_partition]:
                if consumer_record.key == self.key:
                    msg = consumer_record.value
                    break
        except KeyError:
            raise UnknownError('TopicPartition key was not found in response')
        if msg is None:
            raise UnknownError("failed to find matching consumer record with key '{0}'".format(self.key))
        return msg

# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import poll [as 别名]
import sys
import argparse

from kafka import KafkaConsumer

parser = argparse.ArgumentParser(description='Zoe Kafka log viewer')
parser.add_argument('kafka_address', help='Address of the Kafka broker')
parser.add_argument('--list-logs', action='store_true', help='List all the available service logs')
parser.add_argument('--topic', help='Service name to fetch and monitor for activity')

args = parser.parse_args()

consumer = KafkaConsumer(bootstrap_servers=args.kafka_address)

if args.list_logs:
    for topic in consumer.topics():
        if topic[0] != '_':
            print(topic)
    sys.exit(0)

consumer.subscribe(pattern=args.topic)
consumer.poll(1)
consumer.seek_to_beginning()
try:
    for msg in consumer:
        print(msg.value.decode('utf-8'))
except KeyboardInterrupt:
    print('showlog exiting...')

# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import poll [as 别名]
class MessagehubStreamingAdapter(StreamingDataAdapter):
    def __init__(self, topic, username, password, prod=True):
        # Create a new context using system defaults, disable all but TLS1.2
        context = ssl.create_default_context()
        context.options &= ssl.OP_NO_TLSv1
        context.options &= ssl.OP_NO_TLSv1_1
        conf = {
            'client_id': 'pixieapp.client.id',
            'group_id': 'pixieapp.group',
            'sasl_mechanism': 'PLAIN',
            'security_protocol': 'SASL_SSL',
            'ssl_context': context,
            "bootstrap_servers": [ "kafka0{}-{}.messagehub.services.us-south.bluemix.net:9093".format(i, "prod01" if prod else "stage1") for i in range(1,6)],
            "sasl_plain_username": username,
            "sasl_plain_password": password,
            "auto_offset_reset":"latest"
        }
        self.consumer = KafkaConsumer(**conf)
        self.consumer.subscribe([topic])
        self.schema = {}
        self.sampleDocCount = 0
        
    def close(self):
        self.consumer.unsubscribe()
        self.consumer.close() 
        
    def tryCast(self, value, t):
        def _innerTryCast(value, t):
            try:
                return t(value)
            except:
                return None

        if isinstance(t, tuple):
            for a in t:
                ret = _innerTryCast(value, a)
                if ret is not None:
                    return ret
            return None
        
        return _innerTryCast(value, t)
        
    def inferType(self, value):
        if isinstance(value, string_types):
            value = self.tryCast(value, integer_types) or self.tryCast(value, float) or value
        return "integer" if value.__class__==int else "float" if value.__class__ == float else "string"
        
    def inferSchema(self, eventJSON):
        if self.sampleDocCount > 20:
            return
        for key,value in iteritems(eventJSON):
            if not key in self.schema:
                self.schema[key] = self.inferType(value)
        self.sampleDocCount = self.sampleDocCount + 1 
    
    def doGetNextData(self):
        msgs = []
        msg = self.consumer.poll(1000, max_records=10)
        if msg is not None:
            for topicPartition,records in iteritems(msg):
                for record in records:
                    if record.value is not None:                    
                        jsonValue = json.loads(record.value.decode('utf-8'))
                        self.inferSchema(jsonValue)
                        msgs.append(jsonValue)
        return msgs
    
    def close(self):
        self.consumer.close()

# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import poll [as 别名]
def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("topic")
    parser.add_argument("-H", "--host", type=str, help="Kafka server and port.",
                        default="localhost:9092")
    parser.add_argument("-r", "--replay", action="store_true",
                        help="Display all available log entries.",
                        default=False)
    parser.add_argument("-m", "--match", type=str, help="Initial match pattern.",
                        default=None)
    args = parser.parse_args()

    pattern = args.match

    if args.replay:
        auto_offset_reset = 'earliest'
    else:
        auto_offset_reset = 'latest'

    if args.topic[-5:] == '.json':
        value_deserializer = json_value_deserializer
    else:
        value_deserializer = None

    consumer = KafkaConsumer(args.topic,
                             group_id=None,
                             bootstrap_servers=args.host,
                             value_deserializer=value_deserializer,
                             auto_offset_reset=auto_offset_reset)

    while True:
        messages = consumer.poll(250)
        for tp in six.itervalues(messages):
            for message in tp:
                if isinstance(message.value, dict):
                    if message.value['klog_level'] in colors:
                        c = colors[message.value['klog_level']]
                    else:
                        c = attr(0)

                    params = {'topic': message.topic,
                              'offset': message.offset,
                              'level': message.value['klog_level'].upper()}
                    params['time'] = str(datetime.datetime.fromtimestamp(float(message.value['klog_time'])))

                    params['msg'] = message.value['klog_message']

                    if pattern and re.search(pattern, params['msg']) is not None:
                        c += match_color

                    msg = msg_format.format(**params)
                else:
                    c = attr(0)
                    msg = message.value

                print(c+msg+attr(0))

        po = select.poll()
        po.register(sys.stdin, select.POLLIN)
        if po.poll(0):
            ch = sys.stdin.read(1)
            if ch == 'm':
                pattern = sys.stdin.readline().rstrip('\n').encode('utf-8')
                pattern = pattern.rstrip('\n').encode('utf-8')
            elif ch == 'r':
                offset = sys.stdin.readline().rstrip('\n').encode('utf-8')
                offset = int(offset)
                for tp in consumer.assignment():
                    position = consumer.position(tp)
                    consumer.seek(tp, max(0, position-offset))
            elif ch == 'R':
                for tp in consumer.assignment():
                    consumer.seek_to_beginning(tp)
            elif ch == 'p':
                for tp in consumer.assignment():
                    consumer.pause(tp)
            elif ch == 'P':
                for tp in consumer.assignment():
                    consumer.resume(tp)
            elif ch == 'q':
                # FIXME: kafka currently (1.0.1) raises an exception on close
                #consumer.close()
                exit()

# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import poll [as 别名]
    def _run(self):
	pcount = 0
        pause = False
        self.part_prev_time = 0
        while True:
            try:
                if pause:
                    gevent.sleep(5)
                    pause = False
                self._logger.error("Newer KafkaClient %s" % self._topic)
                self._failed = False
                try:
                    consumer = KafkaConsumer(self._topic,
                        bootstrap_servers=self._brokers.split(','),
                        group_id=None)
                except Exception as ex:
                    self.part_cur_time = time.time()
                    if slef.part_prev_time == 0 or self.part_cur_time - self.part_prev_time > 60:
                        self.part_prev_time = self.part_cur_time
                        template = "Consumer Failure {0} occured. Arguments:\n{1!r}"
                        messag = template.format(type(ex).__name__, ex.args)
                        self._logger.error("Error: %s trace %s" % \
                            (messag, traceback.format_exc()))
                    else:
                        template = "Consumer Failure {0} occured. Arguments:\n{1!r}"
                        messag = template.format(type(ex).__name__, ex.args)
                        self._logger.error("Error: %s" % (messag))
		    self._failed = True
                    raise RuntimeError(messag)

                self._logger.error("Starting %s" % self._topic)

                if self._limit:
                    raise gevent.GreenletExit

                while True:
                    try:
                        mdict = consumer.poll()
                        self.resource_check()
                        if len(mdict):
                            counts = {}
                            for tp,tv in mdict.iteritems():
                                if tp not in counts:
                                    counts[tp] = 0
                                counts[tp] += len(tv)
                                if not self.msg_handler(tv):
                                    raise gevent.GreenletExit
                                pcount += len(tv)
                            self._logger.debug("poll for topic %s : %s" % (self._topic, str(counts)))
                        else:
                            gevent.sleep(0.5)

                    except TypeError as ex:
                        self._logger.error("Type Error: %s trace %s" % \
                                (str(ex.args), traceback.format_exc()))
                        gevent.sleep(0.1)
                    except common.FailedPayloadsError as ex:
                        self._logger.error("Payload Error: %s" %  str(ex.args))
                        gevent.sleep(0.1)
            except gevent.GreenletExit:
                break
            except AssertionError as ex:
                self._partoffset = ex
                break
            except Exception as ex:
                self.part_cur_time = time.time()
                if self.part_prev_time == 0 or self.part_cur_time - self.part_prev_time > 60:
                    self.part_prev_time = self.part_cur_time
                    template = "An exception of type {0} occured. Arguments:\n{1!r}"
                    messag = template.format(type(ex).__name__, ex.args)
                    self._logger.error("%s %s : traceback %s" % \
                                      (self._topic, messag, traceback.format_exc()))
                else:
                    template = "An exception of type {0} occured. Arguments:\n{1!r}"
                    messag = template.format(type(ex).__name__, ex.args)
                    self._logger.error("%s %s" % (self._topic, messag))
                self.stop_partition()
		self._failed = True
                pause = True
                if hasattr(ex,'errno'):
                    # This is an unrecoverable error
                    if ex.errno == errno.EMFILE:
                       raise SystemExit(1)

        self._logger.error("Stopping %s pcount %d" % (self._topic, pcount))
        partdb = self.stop_partition()
        return self._partoffset, partdb

# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import poll [as 别名]
from kafka import KafkaConsumer
import json
import ast
import time
# value_deserializer=lambda m: json.loads(m).decode('utf-8')
consumer = KafkaConsumer(bootstrap_servers=['localhost:9092'],
                        #  group_id='my-group-2',
                         )
consumer.subscribe(['msg-from-volttron'])
cnt = 0

while True:
    partition = consumer.poll(timeout_ms=1000, max_records=None)
    # obj : type dict
    if len(partition) > 0:
        print('poll - receive')
        for p in partition:
            for response in partition[p]:
                print('poll topic: {}'.format(response.topic))
                # print('poll value: {}'.format(response.value))
                # string to dict
                # dic_value = ast.literal_eval(response.value)
                print('poll value: {}'.format(response.value))
                # print('poll value: {}, new_value: {}'.format(dic_value, dic_value['new_value']))
                # print('poll value type: {}'.format(type(dic_value)))
    else:
        print('poll - no receive yet {}'.format(cnt))
        cnt += 1