Python KafkaClient.in_flight_request_count方法代码示例

# 需要导入模块: from kafka.client_async import KafkaClient [as 别名]
# 或者: from kafka.client_async.KafkaClient import in_flight_request_count [as 别名]

#.........这里部分代码省略.........
        self._fetcher.update_fetch_positions(partitions)

    def _message_generator(self):
        assert self.assignment() or self.subscription() is not None, 'No topic subscription or manual partition assignment'
        while time.time() < self._consumer_timeout:

            if self._use_consumer_group():
                self._coordinator.ensure_coordinator_known()
                self._coordinator.ensure_active_group()

            # 0.8.2 brokers support kafka-backed offset storage via group coordinator
            elif self.config['group_id'] is not None and self.config['api_version'] >= (0, 8, 2):
                self._coordinator.ensure_coordinator_known()

            # fetch offsets for any subscribed partitions that we arent tracking yet
            if not self._subscription.has_all_fetch_positions():
                partitions = self._subscription.missing_fetch_positions()
                self._update_fetch_positions(partitions)

            poll_ms = 1000 * (self._consumer_timeout - time.time())
            if not self._fetcher.in_flight_fetches():
                poll_ms = 0
            self._client.poll(timeout_ms=poll_ms, sleep=True)

            # We need to make sure we at least keep up with scheduled tasks,
            # like heartbeats, auto-commits, and metadata refreshes
            timeout_at = self._next_timeout()

            # Because the consumer client poll does not sleep unless blocking on
            # network IO, we need to explicitly sleep when we know we are idle
            # because we haven't been assigned any partitions to fetch / consume
            if self._use_consumer_group() and not self.assignment():
                sleep_time = max(timeout_at - time.time(), 0)
                if sleep_time > 0 and not self._client.in_flight_request_count():
                    log.debug('No partitions assigned; sleeping for %s', sleep_time)
                    time.sleep(sleep_time)
                    continue

            # Short-circuit the fetch iterator if we are already timed out
            # to avoid any unintentional interaction with fetcher setup
            if time.time() > timeout_at:
                continue

            for msg in self._fetcher:
                yield msg
                if time.time() > timeout_at:
                    log.debug("internal iterator timeout - breaking for poll")
                    break

            # an else block on a for loop only executes if there was no break
            # so this should only be called on a StopIteration from the fetcher
            # and we assume that it is safe to init_fetches when fetcher is done
            # i.e., there are no more records stored internally
            else:
                self._fetcher.send_fetches()

    def _next_timeout(self):
        timeout = min(self._consumer_timeout,
                      self._client._delayed_tasks.next_at() + time.time(),
                      self._client.cluster.ttl() / 1000.0 + time.time())

        # Although the delayed_tasks timeout above should cover processing
        # HeartbeatRequests, it is still possible that HeartbeatResponses
        # are left unprocessed during a long _fetcher iteration without
        # an intermediate poll(). And because tasks are responsible for
        # rescheduling themselves, an unprocessed response will prevent