Python common.KafkaUnavailableError方法代码示例

# 需要导入模块: from kafka import common [as 别名]
# 或者: from kafka.common import KafkaUnavailableError [as 别名]
def get_watermark_for_topic(
    kafka_client,
    topic,
):
    """This method:
        * refreshes metadata for the kafka client
        * fetches watermarks

    :param kafka_client: KafkaToolClient instance
    :param topic: the topic
    :returns: dict <topic>: [ConsumerPartitionOffsets]
    """
    # Refresh client metadata. We do not use the topic list, because we
    # don't want to accidentally create the topic if it does not exist.
    # If Kafka is unavailable, let's retry loading client metadata
    try:
        kafka_client.load_metadata_for_topics()
    except KafkaUnavailableError:
        kafka_client.load_metadata_for_topics()

    watermarks = get_topics_watermarks(
        kafka_client, [topic]
    )
    return watermarks 

# 需要导入模块: from kafka import common [as 别名]
# 或者: from kafka.common import KafkaUnavailableError [as 别名]
def get_kafka_connection(cls, timeout_seconds=15):
        """Returns a kafka connection, waiting timeout_seconds for the container
        to come up.

        Args:
            timeout_seconds: Retry time (seconds) to get a kafka connection
        """
        end_time = time.time() + timeout_seconds
        logger.info("Getting connection to Kafka container on yocalhost")
        while end_time > time.time():
            try:
                return KafkaClient(get_config().cluster_config.broker_list)
            except KafkaUnavailableError:
                logger.info("Kafka not yet available, waiting...")
                time.sleep(0.1)
        raise KafkaUnavailableError() 

# 需要导入模块: from kafka import common [as 别名]
# 或者: from kafka.common import KafkaUnavailableError [as 别名]
def get_watermark_for_regex(
    kafka_client,
    topic_regex,
):
    """This method:
        * refreshes metadata for the kafka client
        * fetches watermarks

    :param kafka_client: KafkaToolClient instance
    :param topic: the topic regex
    :returns: dict <topic>: [ConsumerPartitionOffsets]
    """
    # Refresh client metadata. We do not use the topic list, because we
    # don't want to accidentally create the topic if it does not exist.
    # If Kafka is unavailable, let's retry loading client metadata
    try:
        kafka_client.load_metadata_for_topics()
    except KafkaUnavailableError:
        kafka_client.load_metadata_for_topics()

    topics_to_be_considered = []

    for topic in kafka_client.topic_partitions:
        if re.search(topic_regex, topic):
            topics_to_be_considered.append(topic)

    watermarks = get_topics_watermarks(
        kafka_client, topics_to_be_considered
    )
    return watermarks 

# 需要导入模块: from kafka import common [as 别名]
# 或者: from kafka.common import KafkaUnavailableError [as 别名]
def test_get_metadata_kafka_error(self, kafka_client_mock):
        with mock.patch.object(
            MyKafkaToolClient,
            'load_metadata_for_topics',
            side_effect=KafkaUnavailableError("Boom!"),
            autospec=True
        ) as mock_func:
            with pytest.raises(KafkaUnavailableError):
                get_consumer_offsets_metadata(
                    kafka_client_mock,
                    self.group,
                    {'topic1': [99]},
                )
            assert mock_func.call_count == 2 

# 需要导入模块: from kafka import common [as 别名]
# 或者: from kafka.common import KafkaUnavailableError [as 别名]
def test_get_metadata_kafka_error(self, kafka_client_mock):
        with mock.patch.object(
            MyKafkaClient,
            'load_metadata_for_topics',
            side_effect=KafkaUnavailableError("Boom!"),
            autospec=True
        ) as mock_func:
            with pytest.raises(KafkaUnavailableError):
                get_consumer_offsets_metadata(
                    kafka_client_mock,
                    self.group,
                    {'topic1': [99]},
                )
            assert mock_func.call_count == 2 

# 需要导入模块: from kafka import common [as 别名]
# 或者: from kafka.common import KafkaUnavailableError [as 别名]
def test__release_retry(self, cluster):
        config = KafkaConsumerConfig(
            self.group,
            cluster,
            auto_commit_enable=True
        )
        consumer = KafkaConsumerGroup([], config)

        mock_consumer = mock.Mock()
        mock_consumer.set_topic_partitions.side_effect = KafkaUnavailableError
        consumer.consumer = mock_consumer

        with pytest.raises(KafkaUnavailableError):
            consumer._release({})
        assert mock_consumer.set_topic_partitions.call_count == 2 

# 需要导入模块: from kafka import common [as 别名]
# 或者: from kafka.common import KafkaUnavailableError [as 别名]
def test_get_kafka_topics_recover_from_error():
    expected = {
        'topic1': [0, 1, 2, 3],
        'topic2': [0, 1]
    }
    mock_client = mock.Mock()
    mock_client.topic_partitions = expected
    mock_client.load_metadata_for_topics.side_effect = [KafkaUnavailableError(), None]
    actual = utils.get_kafka_topics(mock_client)
    assert expected == actual 

# 需要导入模块: from kafka import common [as 别名]
# 或者: from kafka.common import KafkaUnavailableError [as 别名]
def test_get_kafka_topics_error():
    expected = {
        'topic1': [0, 1, 2, 3],
        'topic2': [0, 1]
    }
    mock_client = mock.Mock()
    mock_client.topic_partitions = expected
    mock_client.load_metadata_for_topics.side_effect = KafkaUnavailableError('Boom!')
    with pytest.raises(KafkaUnavailableError):
        utils.get_kafka_topics(mock_client) 

# 需要导入模块: from kafka import common [as 别名]
# 或者: from kafka.common import KafkaUnavailableError [as 别名]
def get_kafka_topics(kafkaclient):
    """Connect to kafka and fetch all the topics/partitions."""
    try:
        kafkaclient.load_metadata_for_topics()
    except KafkaUnavailableError:
        # Sometimes the kafka server closes the connection for inactivity
        # in this case the second call should succeed otherwise the kafka
        # server is down and we should fail
        log.debug("First call to kafka for loading metadata failed."
                  " Trying again.")
        kafkaclient.load_metadata_for_topics()
    return kafkaclient.topic_partitions 

# 需要导入模块: from kafka import common [as 别名]
# 或者: from kafka.common import KafkaUnavailableError [as 别名]
def retry_if_kafka_unavailable_error(exception):
    """Returns true if the exception is of type KafkaUnavailableError

    :param: exception: the exception to be checked
    :returns boolean
    """
    return isinstance(exception, KafkaUnavailableError) 

# 需要导入模块: from kafka import common [as 别名]
# 或者: from kafka.common import KafkaUnavailableError [as 别名]
def _connect(self):
        connection = "{0}:{1}".format(self.host, self.port)
        logging.info("Connecting to Kafka at %s...", connection)
        try:
            self.consumer = KafkaConsumer(self.topic,
                                          group_id=self.group,
                                          bootstrap_servers=[connection]
                                          )
        except KafkaUnavailableError as e:
            raise EncoderError(e) 

# 需要导入模块: from kafka import common [as 别名]
# 或者: from kafka.common import KafkaUnavailableError [as 别名]
def run(cls, args, cluster_config):
        # Setup the Kafka client
        client = KafkaToolClient(cluster_config.broker_list)
        client.load_metadata_for_topics()

        topics_dict = cls.preprocess_args(
            groupid=args.groupid,
            topic=args.topic,
            partitions=args.partitions,
            cluster_config=cluster_config,
            client=client,
            use_admin_client=args.use_admin_client,
        )
        try:
            consumer_offsets_metadata = get_consumer_offsets_metadata(
                client,
                args.groupid,
                topics_dict,
            )
        except KafkaUnavailableError:
            print(
                "Error: Encountered error with Kafka, please try again later.",
                file=sys.stderr,
            )
            raise

        # Warn the user if a topic being subscribed to does not exist in Kafka.
        for topic in topics_dict:
            if topic not in consumer_offsets_metadata:
                print(
                    "Warning: Topic {topic} does not exist in Kafka"
                    .format(topic=topic),
                    file=sys.stderr,
                )

        cls.save_offsets(
            consumer_offsets_metadata,
            topics_dict,
            args.json_file,
            args.groupid,
        )
        client.close() 

# 需要导入模块: from kafka import common [as 别名]
# 或者: from kafka.common import KafkaUnavailableError [as 别名]
def get_consumer_offsets_metadata(
    kafka_client,
    group,
    topics,
    raise_on_error=True,
):
    """This method:
        * refreshes metadata for the kafka client
        * fetches group offsets
        * fetches watermarks

    :param kafka_client: KafkaToolClient instance
    :param group: group id
    :param topics: list of topics
    :param raise_on_error: if False the method ignores missing topics and
      missing partitions. It still may fail on the request send.
    :returns: dict <topic>: [ConsumerPartitionOffsets]
    """
    # Refresh client metadata. We do not use the topic list, because we
    # don't want to accidentally create the topic if it does not exist.
    # If Kafka is unavailable, let's retry loading client metadata
    try:
        kafka_client.load_metadata_for_topics()
    except KafkaUnavailableError:
        kafka_client.load_metadata_for_topics()

    group_offsets = get_current_consumer_offsets(
        kafka_client, group, topics, raise_on_error
    )

    watermarks = get_topics_watermarks(
        kafka_client, topics, raise_on_error
    )

    result = {}
    for topic, partitions in six.iteritems(group_offsets):
        result[topic] = [
            ConsumerPartitionOffsets(
                topic=topic,
                partition=partition,
                current=group_offsets[topic][partition],
                highmark=watermarks[topic][partition].highmark,
                lowmark=watermarks[topic][partition].lowmark,
            ) for partition in partitions
        ]
    return result 

# 需要导入模块: from kafka import common [as 别名]
# 或者: from kafka.common import KafkaUnavailableError [as 别名]
def get_consumer_offsets_metadata(
    kafka_client,
    group,
    topics,
    raise_on_error=True,
    offset_storage='zookeeper',
):
    """This method:
        * refreshes metadata for the kafka client
        * fetches group offsets
        * fetches watermarks

    :param kafka_client: KafkaClient instance
    :param group: group id
    :param topics: list of topics
    :param raise_on_error: if False the method ignores missing topics and
      missing partitions. It still may fail on the request send.
    :param offset_storage: String, one of {zookeeper, kafka}.
    :returns: dict <topic>: [ConsumerPartitionOffsets]
    """

    # Refresh client metadata. We do now use the topic list, because we
    # don't want to accidentally create the topic if it does not exist.
    # If Kafka is unavailable, let's retry loading client metadata (YELPKAFKA-30)
    try:
        kafka_client.load_metadata_for_topics()
    except KafkaUnavailableError:
        kafka_client.load_metadata_for_topics()

    group_offsets = get_current_consumer_offsets(
        kafka_client, group, topics, raise_on_error, offset_storage
    )

    watermarks = get_topics_watermarks(
        kafka_client, topics, raise_on_error
    )

    result = {}
    for topic, partitions in six.iteritems(group_offsets):
        result[topic] = [
            ConsumerPartitionOffsets(
                topic=topic,
                partition=partition,
                current=group_offsets[topic][partition],
                highmark=watermarks[topic][partition].highmark,
                lowmark=watermarks[topic][partition].lowmark,
            ) for partition in partitions
        ]
    return result