Python KafkaConsumer.commit方法代码示例

# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import commit [as 别名]
def commit_offsets_in_kafka(broker, group_name, group_dict):
    cons = KafkaConsumer(bootstrap_servers=broker, group_id=group_name)
    for topic_name, topic_dict in group_dict.iteritems():
        for partition, offset in topic_dict.iteritems():
            logging.info(
                "Commiting {} {} to topic {} and partition number {}".format(
                    group_name, offset, topic_name, partition))
            tp = TopicPartition(topic_name, int(partition))
            cons.assign([tp])
            cons.seek(tp, int(offset))
            # commit it
            cons.commit()
            time.sleep(8)
    cons.close()
    time.sleep(1)

# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import commit [as 别名]
def advanceConsumer():
    '''
    multipic topics and auto commit offset
    '''
    consumer = KafkaConsumer('bamboo1', 'bamboo2', 
            bootstrap_servers=['10.1.200.63:9092'], 
            group_id = '8_consumer_group',
            auto_commit_enable=True,
            auto_commit_interval_ms=30 * 1000, 
            auto_offset_reset='smallest')

    # initialize iteration
    for message in consumer:
        print("TOPIC:%s Partition:%d offset%d key=%s value=%s" % (\
                message.topic, message.partition,
                message.offset, message.key,
                message.value)) 
        consumer.task_done(message)

    consumer.commit()

    # Batch process interface
    while True:
        for m in consumer.fetch_messages():
            print("===Topic:%s Partition:%d offset%d key=%s value=%s" % (\
                    message.topic, message.partition,
                    message.offset, message.key,
                    message.value)) 
            consumer.task_done(m)

# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import commit [as 别名]
    def consume(self):
        consumer = None
        try:
            print "\nCONSUMER TOPICS = " + str(self.listener_topics)
            consumer = KafkaConsumer(*self.listener_topics,
                                     client_id=self.name,
                                     group_id='kafka',
                                     bootstrap_servers=self.connection_string,
                                     auto_offset_reset='smallest')
            self._set_alive(True)
        except Exception as e:
            print "A consumer couldn't be created."
            print e
        while is_running(self.name):
            for message in consumer.fetch_messages():
                asset_success = True
                message_success = True
                if not is_running(self.name):
                    break
                try:
                    try:
                        key = Key.objects.get(listener=Listener.objects.get(listener_topic=message.topic),
                                              listener_key=message.key)
                        feature_data = json.loads(message.value)
                        for asset_type in ['photos', 'videos', 'sounds']:
                            if feature_data.get('properties').get(asset_type):
                                import urllib2
                                urls = []
                                for index, value in enumerate(feature_data.get('properties').get(asset_type)):
                                    asset, created = write_asset(key, value, asset_type, feature_data.get('properties').get('{}_url'.format(asset_type))[index-1])
                                    if not asset:
                                        asset_success = False
                                    else:
                                        print "Asset {} was written.".format(value)
                                    urls += [asset.asset_data.url]
                                feature_data['properties']['{}_url'.format(asset_type)] = urls
                                print "URLS:" + str(urls)
                        if not write_message(key, json.dumps(feature_data)):
                            message_success = False
                        else:
                            print "Message {} was written.".format(feature_data.get('properties').get('city'))
                    except Exception as e:
                        if 'DoesNotExist' in e:
                            continue
                        else:
                            print e
                            message_success = False

                except KeyboardInterrupt:
                    break
                if message_success and asset_success:
                    consumer.task_done(message)
                    consumer.commit()
        consumer.close()
        self._set_alive(False)

# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import commit [as 别名]
def create_consumer_group(topic, group_name, num_messages=1):
    consumer = KafkaConsumer(
        topic,
        group_id=group_name,
        auto_commit_enable=False,
        bootstrap_servers=[KAFKA_URL],
        auto_offset_reset='smallest')
    for i in xrange(num_messages):
        message = consumer.next()
        consumer.task_done(message)
    consumer.commit()
    return consumer

# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import commit [as 别名]
class Consumer(BaseStreamConsumer):
    """
    Used in DB and SW worker. SW consumes per partition.
    """
    def __init__(self, location, enable_ssl, cert_path, topic, group, partition_id):
        self._location = location
        self._group = group
        self._topic = topic
        kwargs = _prepare_kafka_ssl_kwargs(cert_path) if enable_ssl else {}
        self._consumer = KafkaConsumer(
            bootstrap_servers=self._location,
            group_id=self._group,
            max_partition_fetch_bytes=10485760,
            consumer_timeout_ms=100,
            client_id="%s-%s" % (self._topic, str(partition_id) if partition_id is not None else "all"),
            request_timeout_ms=120 * 1000,
            heartbeat_interval_ms=10000,
            **kwargs
        )

        # explicitly causing consumer to bootstrap the cluster metadata
        self._consumer.topics()

        if partition_id is not None:
            self._partitions = [TopicPartition(self._topic, partition_id)]
            self._consumer.assign(self._partitions)
        else:
            self._partitions = [TopicPartition(self._topic, pid) for pid in self._consumer.partitions_for_topic(self._topic)]
            self._consumer.subscribe(topics=[self._topic])

    def get_messages(self, timeout=0.1, count=1):
        result = []
        while count > 0:
            try:
                m = next(self._consumer)
                result.append(m.value)
                count -= 1
            except StopIteration:
                break
        return result

    def get_offset(self, partition_id):
        for tp in self._partitions:
            if tp.partition == partition_id:
                return self._consumer.position(tp)
        raise KeyError("Can't find partition %d", partition_id)

    def close(self):
        self._consumer.commit()
        self._consumer.close()

# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import commit [as 别名]
class KafkaEventsConsumer:
    '''A generic kafka consumer of events'''
    
    def __init__(self, topic, group_id, host, port, strategy):
        '''Create a new KafkaRealTimeEventsProducer instance
        
        topic   the kafka topic to of interest
        group   the consumer group for this consumer
        host	the ipaddress or host name for the kafka server as a string
        port	the port name the kafka server is listening on as a string
        '''
        
        if (isinstance(host, str) == False or 
            isinstance(port, str) == False or 
            isinstance(topic, str) == False):
            raise Exception("unexepcted kafka server parameters provided")
        
	self.consumer = KafkaConsumer(topic,
                                      group_id=group_id,
                                      bootstrap_servers=["%s:%s" % (host, port)],
                                      consumer_timeout_ms=-1)
        
	print("In KafkaEventsConsumer.__init__  Created KafkaConsumer instance: {0}".format(self.consumer))
        self.topic = topic
	self.strategy = strategy
        self.worker = threading.Thread(target=lambda : self.collect_messages())
	print("In KafkaEventsConsumer.__init__  Created worker thread: {0}".format(self.worker))
        self.worker.start()
        
    def collect_messages(self):
        '''Collects raw messages from the consumer for storage in an instance queue'''
        
        try:
	    print ("In KafkaEventsConsumer waiting for events delivery")
            for msg in self.consumer:
		print("KafkaConsumer collected 1 msg from topic: {0} in collect_messages".format(self.topic))
                self.strategy.handle_message(msg)

                # Mark this message as fully consumed
                # so it can be included in the next commit
                self.consumer.task_done(msg)

                # Commit the message that was just consumed
                self.consumer.commit()        
	except Exception as err:
                print("Exception: {0}".format(err))
                pass

# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import commit [as 别名]
def main():
    # more advanced consumer -- multiple topics w/ auto commit offset
    # management
    consumer = KafkaConsumer('email',  bootstrap_servers=['localhost:9092'], group_id='None', auto_commit_enable=True, auto_commit_interval_ms=30 * 1000, auto_offset_reset='smallest')

    # Infinite iteration
    for m in consumer:
        # do_some_work(m)

        # Mark this message as fully consumed
        # so it can be included in the next commit
        #
        # **messages that are not marked w/ task_done currently do not commit!
        #print 'm is ',m
        #post data example is ,{"body":{"aa":["1","2",3]}}
        try:
            msg = eval(m.value)
            print ("email eval msg is %s")  % (str(msg.keys()))
        except:
            print ("email error msg is %s")  % (m)
        """
        import json
        msg = json.loads(m.value)
        print('json msg is %s' % (str(msg)))
        print
        """
        #format mail info entry
        mailParam = formatEmailEntry(msg)
        #send mail entry info
        sendMail(mailParam)
        #end send mail
        #end format
        consumer.task_done(m)
        # If auto_commit_enable is False, remember to commit() periodically
        consumer.commit()

    # Batch process interface
    while True:
        for m in kafka.fetch_messages():
            # process_message(m)
            consumer.task_done(m)
        time.sleep(1)

# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import commit [as 别名]
def testRestart():
    '''test restart
    测试某一个group在消费了kafka中的消息并commit之后，是否能重新读取？
            ->事实是不行
    '''
    topicsList = ['JOB_NGINX', 'JOB_BASIC', 'JOB_JMON', 'JOB_TOMCAT']
    brokerList = ['10.10.90.171:9092', '10.10.82.114:9092', '10.10.94.15:9092']
    groupId = '8_consumer_group'
    kc = KafkaConsumer(
            *topicsList,
            fetch_min_bytes = 1024, 
            group_id = groupId,
            bootstrap_servers = brokerList,
            consumer_timeout_ms = 10*1000,
            auto_offset_reset='smallest'
            )
    print kc.offsets()
    for partition in [0, 1, 2]:
        consumerMsg = KafkaMessage('JOB_BASIC', \
                partition, 2000, 'shit', 'shit')
        kc.task_done(consumerMsg)
    print kc.offsets()
    kc.set_topic_partitions(
            ('JOB_BASIC', 0, 2000), 
            ('JOB_BASIC', 1, 2000),
            ('JOB_BASIC', 2, 1000))
    import pdb
    pdb.set_trace()
    while 1:
        try:
            for consumer in kc:
                print consumer
                print type(consumer)
                kc.task_done(consumer)
        except ConsumerTimeout:
            kc.commit()
            print 'xxxxxxxxxxxxxxxxx'
            continue

# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import commit [as 别名]
class Consumer(object):
	def __init__(self, group, topic):
		"""Initialize Consumer with kafka broker IP, and topic."""

		# for Kafka
		self.topic = topic
		self.group = group
		self.redisClient = redis.StrictRedis(host='localhost', port=6379, db=0)
		self.consumer = KafkaConsumer(self.topic,
                         bootstrap_servers=['localhost:9092'])

	def consume_topic(self):
		"""Consumes a stream of messages from the "auto_trnx" topic.
		"""
		while True:
			try:
				for message in self.consumer:
					msg_info = message.value.decode()
					print msg_info
					self.redisClient.lpush("auto_trnx", msg_info)
				self.consumer.commit()
			except:
				pass

# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import commit [as 别名]
    auto_commit_enable=True,
    auto_commit_interval_ms=30 * 1000,
    auto_offset_reset="smallest",
)

# Infinite iteration
for message in consumer:
    # do_some_work(m)
    print ("%s:%d:%d: key=%s value=%s" % (message.topic, message.partition, message.offset, message.key, message.value))
    fd = log.getLogFile()
    msg = generateLog(message.value)
    if msg:
        fd.write(msg)
        fd.write("\n")
        fd.flush()
    # Mark this message as fully consumed
    # so it can be included in the next commit
    #
    # **messages that are not marked w/ task_done currently do not commit!
    consumer.task_done(message)

    # If auto_commit_enable is False, remember to commit() periodically
    consumer.commit()

# Batch process interface
while True:
    for m in kafka.fetch_messages():
        process_message(m)
        consumer.task_done(m)
#'''

# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import commit [as 别名]
def cylc_kafka_consumer(kafka_server, kafka_topic, group_id, message, debug):
    r"""Look for a matching message in a Kafka topic.

    ARGUMENTS:
     * kafka_server - Kafka server URL, e.g. "localhost:9092".
     * kafka_topic - the Kafka topic to check, e.g. "data-avail".
     * group_id - determines Kafka offset ownership (see below).
     * message - string-ified dict with optional pattern elements (see below).
     * debug - boolean; set by daemon debug mode; prints to suite err log.

    The topic is first consumed from the beginning, then from the previous
    committed offset. If the message is not found by end of topic, commit the
    offset and return (to will try again later). If found, return the result.

    Kafka commits offsets per "consumer group" so the group_id argument
    must be unique per distinct trigger in the suite - this allows each trigger
    to separately consume the topic from the beginning, looking for its own
    messages (otherwise, with shared offsets, one trigger could move the offset
    beyond the messages of another trigger). This goes for successive instances
    of an external-triggered cycling task too, because out-of-order triggering
    could be required sometimes. So this argument should typically be, e.g.:

        group_id=x%(id)s  # id ID of the dependent task

    where "x" is an arbitrary string you can use to change the group name if
    you need to re-run the suite, and the messages, from the start again,
    without re-running the producer suite. Note this also serves to make the
    function signature cycle-point-specific for Cylc even if the message does
    not contain the cycle point (although it probably should).

    The "message" argument is a stringified dict, e.g.:
        {'system': 'prod', 'point': '2025', 'data': '<nwp.*\.nc>'}
    should be represented as:
        "system:prod point:2025 data:<nwp.*\.nc>"

    A match occurs Kafka if all message dict items match, and the result
    returned is the sub-dict of the actual values of items containing
    angle-bracket-delineated regex patterns. E.g. above {'data': 'nwp-2025.nc'}

    """

    consumer = KafkaConsumer(kafka_topic, bootstrap_servers=[kafka_server],
                             value_deserializer=json.loads,
                             consumer_timeout_ms=CONSUMER_TIMEOUT_MS,
                             auto_offset_reset='earliest',
                             group_id=group_id)

    # Construct a dict from the message argument "key1=val1 key2=val2 ...".
    cylc_msg = dict(m.split(':') for m in message.split())

    result = (False, {})
    n_cons = 0
    for kafka_msg in consumer:
        n_cons += 1
        m = _match_msg(cylc_msg, kafka_msg)
        if m:
            result = (True, m)
            break
        # (else consume and compare next message)
    consumer.commit()
    # Unsubscribe before exit, otherwise next call will be slow while
    # Kafka times out waiting for this original consumer connection.
    consumer.unsubscribe()
    if debug:
        if result[0]:
            res = "\n  MATCHED: %s" % result[1]
        else:
            res = "no match."
        LOG.debug('Kafka: "%s" (consumed %d) ... %s', message, n_cons, res)
    return result

# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import commit [as 别名]
class KafkaMessageSensor(Sensor):
    """
    Read multiple topics from Apache Kafka cluster and auto-commit offset (mark tasks as finished).
    If responded topic message is JSON - try to convert it to object for reuse inside st2.
    """
    TRIGGER = 'kafka.new_message'
    DEFAULT_GROUP_ID = 'st2-sensor-group'
    DEFAULT_CLIENT_ID = 'st2-kafka-consumer'

    def __init__(self, sensor_service, config=None):
        """
        Parse config variables, set defaults.
        """
        super(KafkaMessageSensor, self).__init__(sensor_service=sensor_service, config=config)
        self._logger = self._sensor_service.get_logger(__name__)

        message_sensor = self._config.get('message_sensor')
        if not message_sensor:
            raise ValueError('[KafkaMessageSensor]: "message_sensor" config value is required!')

        self._hosts = message_sensor.get('hosts')
        if not self._hosts:
            raise ValueError('[KafkaMessageSensor]: "message_sensor.hosts" config value is required!')

        self._topics = set(message_sensor.get('topics', []))
        if not self._topics:
            raise ValueError('[KafkaMessageSensor]: "message_sensor.topics" should list at least one topic!')

        # set defaults for empty values
        self._group_id = message_sensor.get('group_id') or self.DEFAULT_GROUP_ID
        self._client_id = message_sensor.get('client_id') or self.DEFAULT_CLIENT_ID
        self._consumer = None

    def setup(self):
        """
        Create connection and initialize Kafka Consumer.
        """
        self._logger.debug('[KafkaMessageSensor]: Initializing consumer ...')
        self._consumer = KafkaConsumer(*self._topics,
                                       client_id=self._client_id,
                                       group_id=self._group_id,
                                       bootstrap_servers=self._hosts,
                                       deserializer_class=self._try_deserialize)
        self._ensure_topics_existence()

    def _ensure_topics_existence(self):
        """
        Ensure that topics we're listening to exist.

        Fetching metadata for a non-existent topic will automatically try to create it
        with the default replication factor and number of partitions (default server config).
        Otherwise Kafka server is not configured to auto-create topics and partitions.
        """
        map(self._consumer._client.ensure_topic_exists, self._topics)
        self._consumer.set_topic_partitions(*self._topics)

    def run(self):
        """
        Run infinite loop, continuously reading for Kafka message bus,
        dispatch trigger with payload data if message received.
        """
        self._logger.debug('[KafkaMessageSensor]: Entering into listen mode ...')

        for message in self._consumer:
            self._logger.debug(
                "[KafkaMessageSensor]: Received %s:%d:%d: key=%s message=%s" %
                (message.topic, message.partition,
                 message.offset, message.key, message.value)
            )
            payload = {
                'topic': message.topic,
                'partition': message.partition,
                'offset': message.offset,
                'key': message.key,
                'message': message.value,
            }
            self._sensor_service.dispatch(trigger=self.TRIGGER, payload=payload)
            # Mark this message as fully consumed
            self._consumer.task_done(message)
            self._consumer.commit()

    def cleanup(self):
        """
        Close connection, just to be sure.
        """
        self._consumer._client.close()

    def add_trigger(self, trigger):
        pass

    def update_trigger(self, trigger):
        pass

    def remove_trigger(self, trigger):
        pass

    @staticmethod
    def _try_deserialize(body):
        """
        Try to deserialize received message body.
#.........这里部分代码省略.........

# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import commit [as 别名]
class GreenFeedConsumer(threading.Thread):
    """
    Greenlet based feed consumer
    All callbacks are spawned as greenlets on a background thread


    keyword arguments :

    broker (list): List of initial broker nodes the consumer should contact to
    bootstrap initial cluster metadata.  This does not have to be the full node list.
    It just needs to have at least one broker

    group (str): the name of the consumer group to join, Offsets are fetched /
    committed to this group name.

    offset='smallest' : read all msgs from beginning of time;  default read fresh

    commit_every_t_ms:  How much time (in milliseconds) to before commit to zookeeper
    kill_signal: What is the kill signal to handle exit gracefully.
    wait_time_before_exit: How much time to wait before exiting green threads

    """
    daemon = True
    
    def __init__(self, broker, group, offset='largest', commit_every_t_ms=1000,
                 parts=None, kill_signal=signal.SIGTERM, wait_time_before_exit=10):
        self.brokerurl = broker
        self.kill_signal = kill_signal
        self.exit_consumer = False
        self.wait_time_before_exit = wait_time_before_exit
        self.create_kill_signal_handler()
        try:
            self.cons = KafkaConsumer(bootstrap_servers=broker,
                                      auto_offset_reset=offset,
                                      auto_commit_enable=True,
                                      auto_commit_interval_ms=commit_every_t_ms,
                                      group_id=group
                                      )
        except KafkaUnavailableError:
            log.critical( "\nCluster Unavailable %s : Check broker string\n", broker)
            raise
        except:
            raise

        self.topics = []
        self.callbacks = {}
        super(GreenFeedConsumer, self).__init__()

    def add_topic(self, topic, todo , parts=None):
        """
        Set the topic/partitions to consume

        todo (callable) : callback for the topic
        NOTE: Callback is for entire topic, if you call this for multiple
        partitions for same topic with diff callbacks, only the last callback
        is retained

        topic : topic to listen to

        parts (list) : tuple of the partitions to listen to

        """
        self.callbacks[topic] = todo

        if parts is None:
            log.info(" GreenConsumer : adding topic %s ", topic)
            self.topics.append(topic)
        else:
              for part in parts:
                  log.info(" GreenConsumer : adding topic %s %s", topic , part)
                  self.topics.append((topic,part))

        self.cons._client.ensure_topic_exists(topic)
        self.cons.set_topic_partitions(*self.topics)

    def remove_topic(self, topic,  parts=None):
        try:

            if parts is None:
                self.topics.remove(topic)
            else:
                for part in parts:
                    self.topics.remove((topic,part))
        except:
            log.critical("GreenConsumer : no such topic %s", topic)
            return
        log.info(" GreenConsumer : removed topic %s", topic)
        self.cons.set_topic_partitions(*self.topics)

    def create_kill_signal_handler(self):

        def set_stop_signal(signal, frame):
            self.exit_consumer = True

        signal.signal(self.kill_signal, set_stop_signal)

    def wrap(self, callback, mesg):
        callback(mesg.key, mesg.value)
        self.cons.task_done(mesg)

#.........这里部分代码省略.........

# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import commit [as 别名]
class Consumer(BaseStreamConsumer):
    """
    Used in DB and SW worker. SW consumes per partition.
    """
    def __init__(self, location, topic, group, partition_id):
        self._location = location
        self._group = group
        self._topic = topic
        self._consumer = KafkaConsumer(
            bootstrap_servers=self._location,
            group_id=self._group,
            max_partition_fetch_bytes=10485760,
            consumer_timeout_ms=100,
            client_id="%s-%s" % (self._topic, str(partition_id) if partition_id is not None else "all"),
            request_timeout_ms=120 * 1000,
        )

        if partition_id is not None:
            self._partition_ids = [TopicPartition(self._topic, partition_id)]
            self._consumer.assign(self._partition_ids)
        else:
            self._partition_ids = [TopicPartition(self._topic, pid) for pid in self._consumer.partitions_for_topic(self._topic)]
            self._consumer.subscribe(topics=[self._topic])
            if self._consumer._use_consumer_group():
                self._consumer._coordinator.ensure_coordinator_known()
                self._consumer._coordinator.ensure_active_group()

        self._consumer._update_fetch_positions(self._partition_ids)
        self._start_looping_call()

    def _start_looping_call(self, interval=60):
        def errback(failure):
            logger.exception(failure.value)
            if failure.frames:
                logger.critical(str("").join(format_tb(failure.getTracebackObject())))
            self._poll_task.start(interval).addErrback(errback)

        self._poll_task = LoopingCall(self._poll_client)
        self._poll_task.start(interval).addErrback(errback)

    def _poll_client(self):
        self._consumer._client.poll()

    def get_messages(self, timeout=0.1, count=1):
        result = []
        while count > 0:
            try:
                m = next(self._consumer)
                result.append(m.value)
                count -= 1
            except StopIteration:
                break
        return result

    def get_offset(self, partition_id):
        for tp in self._partition_ids:
            if tp.partition == partition_id:
                return self._consumer.position(tp)
        raise KeyError("Can't find partition %d", partition_id)

    def close(self):
        self._poll_task.stop()
        self._consumer.commit()
        # getting kafka client event loop running some more and execute commit
        tries = 3
        while tries:
            self.get_messages()
            sleep(2.0)
            tries -= 1
        self._consumer.close()

# 需要导入模块: from kafka import KafkaConsumer [as 别名]
# 或者: from kafka.KafkaConsumer import commit [as 别名]
class FeedConsumer(threading.Thread):
    """Threaded gomsg feed consumer
    callbacks are called on a separate thread in the same method
    
    keyword arguments :

    broker (list): List of initial broker nodes the consumer should contact to
    bootstrap initial cluster metadata.  This does not have to be the full node list.
    It just needs to have at least one broker

    group (str): the name of the consumer group to join, Offsets are fetched /
    committed to this group name.

    offset='earliest' : read all msgs from beginning of time;  default read fresh

    async=True : In case of Zk commit offset everytime

    commit_every_t_ms:  How much time (in milliseconds) to before commit to zookeeper

    kwargs : anything else you want to pass to kafka-python

    """
    daemon = True

    def __init__(self,
                 broker,
                 group,
                 offset='latest',
                 commit_every_t_ms=1000,
                 parts=None,
                 kill_signal=signal.SIGTERM,
                 wait_time_before_exit=1,
                 use_zk=False,
                 async_commit=True,
                 handler_timeout_ms=30000,
                 **kwargs):
        self.brokerurl = broker
        self.kill_signal = kill_signal
        self.exit_consumer = False
        self.async_commit = async_commit
        try:
            self.create_kill_signal_handler()
        except Exception as e:
            log.error(
                "[feedconsumer log] exception %s. Skipping signal handler install. ",
                str(e))
            pass

        self.wait_time_before_exit = wait_time_before_exit

        if use_zk:
            kwargs['api_version'] = kwargs.get('api_version', '0.8.1')
            # ZK autocommit does not seem to work reliably
            # TODO
            self.async_commit = False

        else:
            kwargs['api_version'] = kwargs.get('api_version',
                                               CURRENT_PROD_BROKER_VERSION)

        # curb over-optimism
        handler_timeout_ms = min(handler_timeout_ms, 60000)

        try:
            self.cons = KafkaConsumer(
                bootstrap_servers=broker,
                auto_offset_reset=offset,
                enable_auto_commit=self.async_commit,
                auto_commit_interval_ms=commit_every_t_ms,
                group_id=group,
                session_timeout_ms=handler_timeout_ms,
                **kwargs)

        except KafkaTimeoutError as e:
            log.error(
                "[feedconsumer log] INIT KafkaTimeoutError  %s. Please check broker string %s \n",
                str(e), broker)
            raise e
        except Exception as e1:
            log.error("[feedconsumer log] INIT err %s \n", str(e1))
            raise e1

        self.callbacks = {}
        super(FeedConsumer, self).__init__()

    def add_topic(self, topic, todo, parts=None):
        """
        Set the topic/partitions to consume

        todo (callable) : callback for the topic
        NOTE: Callback is for entire topic, if you call this for multiple
        partitions for same topic with diff callbacks, only the last callback
        is retained

        topic : topic to listen to

        parts (list) : tuple of the partitions to listen to

        """
        try:
#.........这里部分代码省略.........