本文整理汇总了Python中kafka.coordinator.consumer.ConsumerCoordinator类的典型用法代码示例。如果您正苦于以下问题：Python ConsumerCoordinator类的具体用法？Python ConsumerCoordinator怎么用？Python ConsumerCoordinator使用的例子？那么, 这里精选的类代码示例或许可以为您提供帮助。
在下文中一共展示了ConsumerCoordinator类的6个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: test_maybe_auto_commit_offsets_sync
def test_maybe_auto_commit_offsets_sync(mocker, api_version, group_id, enable,
error, has_auto_commit, commit_offsets,
warn, exc):
mock_warn = mocker.patch('kafka.coordinator.consumer.log.warning')
mock_exc = mocker.patch('kafka.coordinator.consumer.log.exception')
client = KafkaClient(api_version=api_version)
coordinator = ConsumerCoordinator(client, SubscriptionState(),
Metrics(), 'consumer',
api_version=api_version,
enable_auto_commit=enable,
group_id=group_id)
commit_sync = mocker.patch.object(coordinator, 'commit_offsets_sync',
side_effect=error)
if has_auto_commit:
assert coordinator._auto_commit_task is not None
else:
assert coordinator._auto_commit_task is None
assert coordinator._maybe_auto_commit_offsets_sync() is None
if has_auto_commit:
assert coordinator._auto_commit_task is not None
assert commit_sync.call_count == (1 if commit_offsets else 0)
assert mock_warn.call_count == (1 if warn else 0)
assert mock_exc.call_count == (1 if exc else 0)示例2: __init__
def __init__(self, *topics, **configs):
self.config = copy.copy(self.DEFAULT_CONFIG)
for key in self.config:
if key in configs:
self.config[key] = configs.pop(key)
# Only check for extra config keys in top-level class
assert not configs, 'Unrecognized configs: %s' % configs
deprecated = {'smallest': 'earliest', 'largest': 'latest'}
if self.config['auto_offset_reset'] in deprecated:
new_config = deprecated[self.config['auto_offset_reset']]
log.warning('use auto_offset_reset=%s (%s is deprecated)',
new_config, self.config['auto_offset_reset'])
self.config['auto_offset_reset'] = new_config
metrics_tags = {'client-id': self.config['client_id']}
metric_config = MetricConfig(samples=self.config['metrics_num_samples'],
time_window_ms=self.config['metrics_sample_window_ms'],
tags=metrics_tags)
reporters = [reporter() for reporter in self.config['metric_reporters']]
self._metrics = Metrics(metric_config, reporters)
metric_group_prefix = 'consumer'
# TODO _metrics likely needs to be passed to KafkaClient, etc.
# api_version was previously a str. accept old format for now
if isinstance(self.config['api_version'], str):
str_version = self.config['api_version']
if str_version == 'auto':
self.config['api_version'] = None
else:
self.config['api_version'] = tuple(map(int, str_version.split('.')))
log.warning('use api_version=%s (%s is deprecated)',
str(self.config['api_version']), str_version)
self._client = KafkaClient(**self.config)
# Get auto-discovered version from client if necessary
if self.config['api_version'] is None:
self.config['api_version'] = self._client.config['api_version']
self._subscription = SubscriptionState(self.config['auto_offset_reset'])
self._fetcher = Fetcher(
self._client, self._subscription, self._metrics, metric_group_prefix, **self.config)
self._coordinator = ConsumerCoordinator(
self._client, self._subscription, self._metrics, metric_group_prefix,
assignors=self.config['partition_assignment_strategy'],
**self.config)
self._closed = False
self._iterator = None
self._consumer_timeout = float('inf')
if topics:
self._subscription.subscribe(topics=topics)
self._client.set_topics(topics)示例3: __init__
def __init__(self, *topics, **configs):
self.config = copy.copy(self.DEFAULT_CONFIG)
for key in self.config:
if key in configs:
self.config[key] = configs.pop(key)
# Only check for extra config keys in top-level class
assert not configs, 'Unrecognized configs: %s' % configs
deprecated = {'smallest': 'earliest', 'largest': 'latest' }
if self.config['auto_offset_reset'] in deprecated:
new_config = deprecated[self.config['auto_offset_reset']]
log.warning('use auto_offset_reset=%s (%s is deprecated)',
new_config, self.config['auto_offset_reset'])
self.config['auto_offset_reset'] = new_config
metrics_tags = {'client-id': self.config['client_id']}
metric_config = MetricConfig(samples=self.config['metrics_num_samples'],
time_window_ms=self.config['metrics_sample_window_ms'],
tags=metrics_tags)
reporters = [reporter() for reporter in self.config['metric_reporters']]
reporters.append(DictReporter('kafka.consumer'))
self._metrics = Metrics(metric_config, reporters)
metric_group_prefix = 'consumer'
# TODO _metrics likely needs to be passed to KafkaClient, etc.
client = self.config.pop('client', None) or KafkaClient(**self.config)
self._client = client
# Check Broker Version if not set explicitly
if self.config['api_version'] == 'auto':
self.config['api_version'] = self._client.check_version()
assert self.config['api_version'] in ('0.10', '0.9', '0.8.2', '0.8.1', '0.8.0'), 'Unrecognized api version'
# Convert api_version config to tuple for easy comparisons
self.config['api_version'] = tuple(
map(int, self.config['api_version'].split('.')))
self._subscription = SubscriptionState(self.config['auto_offset_reset'])
self._fetcher = Fetcher(
self._client, self._subscription, self._metrics, metric_group_prefix, **self.config)
self._coordinator = ConsumerCoordinator(
self._client, self._subscription, self._metrics, metric_group_prefix,
assignors=self.config['partition_assignment_strategy'],
**self.config)
self._closed = False
self._iterator = None
self._consumer_timeout = float('inf')
if topics:
self._subscription.subscribe(topics=topics)
self._client.set_topics(topics)示例4: __init__
def __init__(self, *topics, **configs):
self.config = copy.copy(self.DEFAULT_CONFIG)
for key in self.config:
if key in configs:
self.config[key] = configs.pop(key)
# Only check for extra config keys in top-level class
assert not configs, 'Unrecognized configs: %s' % configs
deprecated = {'smallest': 'earliest', 'largest': 'latest' }
if self.config['auto_offset_reset'] in deprecated:
new_config = deprecated[self.config['auto_offset_reset']]
log.warning('use auto_offset_reset=%s (%s is deprecated)',
new_config, self.config['auto_offset_reset'])
self.config['auto_offset_reset'] = new_config
self._client = KafkaClient(**self.config)
# Check Broker Version if not set explicitly
if self.config['api_version'] == 'auto':
self.config['api_version'] = self._client.check_version()
assert self.config['api_version'] in ('0.9', '0.8.2', '0.8.1', '0.8.0'), 'Unrecognized api version'
# Convert api_version config to tuple for easy comparisons
self.config['api_version'] = tuple(
map(int, self.config['api_version'].split('.')))
self._subscription = SubscriptionState(self.config['auto_offset_reset'])
self._fetcher = Fetcher(
self._client, self._subscription, **self.config)
self._coordinator = ConsumerCoordinator(
self._client, self._subscription,
assignors=self.config['partition_assignment_strategy'],
**self.config)
self._closed = False
self._iterator = None
self._consumer_timeout = float('inf')
#self.metrics = None
if topics:
self._subscription.subscribe(topics=topics)
self._client.set_topics(topics)示例5: __init__
def __init__(self, *topics, **configs):
self.config = copy.copy(self.DEFAULT_CONFIG)
for key in self.config:
if key in configs:
self.config[key] = configs.pop(key)
# Only check for extra config keys in top-level class
assert not configs, "Unrecognized configs: %s" % configs
deprecated = {"smallest": "earliest", "largest": "latest"}
if self.config["auto_offset_reset"] in deprecated:
new_config = deprecated[self.config["auto_offset_reset"]]
log.warning("use auto_offset_reset=%s (%s is deprecated)", new_config, self.config["auto_offset_reset"])
self.config["auto_offset_reset"] = new_config
self._client = KafkaClient(**self.config)
# Check Broker Version if not set explicitly
if self.config["api_version"] == "auto":
self.config["api_version"] = self._client.check_version()
assert self.config["api_version"] in ("0.9", "0.8.2", "0.8.1", "0.8.0")
# Convert api_version config to tuple for easy comparisons
self.config["api_version"] = tuple(map(int, self.config["api_version"].split(".")))
self._subscription = SubscriptionState(self.config["auto_offset_reset"])
self._fetcher = Fetcher(self._client, self._subscription, **self.config)
self._coordinator = ConsumerCoordinator(
self._client, self._subscription, assignors=self.config["partition_assignment_strategy"], **self.config
)
self._closed = False
self._iterator = None
self._consumer_timeout = float("inf")
# self.metrics = None
if topics:
self._subscription.subscribe(topics=topics)
self._client.set_topics(topics)示例6: KafkaConsumer
#.........这里部分代码省略.........
"connections_max_idle_ms": 9 * 60 * 1000, # not implemented yet
#'metric_reporters': None,
#'metrics_num_samples': 2,
#'metrics_sample_window_ms': 30000,
}
def __init__(self, *topics, **configs):
self.config = copy.copy(self.DEFAULT_CONFIG)
for key in self.config:
if key in configs:
self.config[key] = configs.pop(key)
# Only check for extra config keys in top-level class
assert not configs, "Unrecognized configs: %s" % configs
deprecated = {"smallest": "earliest", "largest": "latest"}
if self.config["auto_offset_reset"] in deprecated:
new_config = deprecated[self.config["auto_offset_reset"]]
log.warning("use auto_offset_reset=%s (%s is deprecated)", new_config, self.config["auto_offset_reset"])
self.config["auto_offset_reset"] = new_config
self._client = KafkaClient(**self.config)
# Check Broker Version if not set explicitly
if self.config["api_version"] == "auto":
self.config["api_version"] = self._client.check_version()
assert self.config["api_version"] in ("0.9", "0.8.2", "0.8.1", "0.8.0")
# Convert api_version config to tuple for easy comparisons
self.config["api_version"] = tuple(map(int, self.config["api_version"].split(".")))
self._subscription = SubscriptionState(self.config["auto_offset_reset"])
self._fetcher = Fetcher(self._client, self._subscription, **self.config)
self._coordinator = ConsumerCoordinator(
self._client, self._subscription, assignors=self.config["partition_assignment_strategy"], **self.config
)
self._closed = False
self._iterator = None
self._consumer_timeout = float("inf")
# self.metrics = None
if topics:
self._subscription.subscribe(topics=topics)
self._client.set_topics(topics)
def assign(self, partitions):
"""Manually assign a list of TopicPartitions to this consumer.
Arguments:
partitions (list of TopicPartition): assignment for this instance.
Raises:
IllegalStateError: if consumer has already called subscribe()
Warning:
It is not possible to use both manual partition assignment with
assign() and group assignment with subscribe().
Note:
This interface does not support incremental assignment and will
replace the previous assignment (if there was one).
Note:
Manual topic assignment through this method does not use the
consumer's group management functionality. As such, there will be
no rebalance operation triggered when group membership or cluster