本文整理匯總了Python中allennlp.data.iterators.BasicIterator._create_batches方法的典型用法代碼示例。如果您正苦於以下問題：Python BasicIterator._create_batches方法的具體用法？Python BasicIterator._create_batches怎麽用？Python BasicIterator._create_batches使用的例子？那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類allennlp.data.iterators.BasicIterator的用法示例。
在下文中一共展示了BasicIterator._create_batches方法的8個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Python代碼示例。
示例1: test_multiple_cursors
# 需要導入模塊: from allennlp.data.iterators import BasicIterator [as 別名]
# 或者: from allennlp.data.iterators.BasicIterator import _create_batches [as 別名]
def test_multiple_cursors(self):
# pylint: disable=protected-access
lazy_instances1 = _LazyInstances(lambda: (i for i in self.instances))
lazy_instances2 = _LazyInstances(lambda: (i for i in self.instances))
eager_instances1 = self.instances[:]
eager_instances2 = self.instances[:]
for instances1, instances2 in [(eager_instances1, eager_instances2),
(lazy_instances1, lazy_instances2)]:
iterator = BasicIterator(batch_size=1, instances_per_epoch=2)
iterator.index_with(self.vocab)
# First epoch through dataset1
batches = list(iterator._create_batches(instances1, shuffle=False))
grouped_instances = [batch.instances for batch in batches]
assert grouped_instances == [[self.instances[0]], [self.instances[1]]]
# First epoch through dataset2
batches = list(iterator._create_batches(instances2, shuffle=False))
grouped_instances = [batch.instances for batch in batches]
assert grouped_instances == [[self.instances[0]], [self.instances[1]]]
# Second epoch through dataset1
batches = list(iterator._create_batches(instances1, shuffle=False))
grouped_instances = [batch.instances for batch in batches]
assert grouped_instances == [[self.instances[2]], [self.instances[3]]]
# Second epoch through dataset2
batches = list(iterator._create_batches(instances2, shuffle=False))
grouped_instances = [batch.instances for batch in batches]
assert grouped_instances == [[self.instances[2]], [self.instances[3]]]示例2: test_shuffle
# 需要導入模塊: from allennlp.data.iterators import BasicIterator [as 別名]
# 或者: from allennlp.data.iterators.BasicIterator import _create_batches [as 別名]
def test_shuffle(self):
# pylint: disable=protected-access
for test_instances in (self.instances, self.lazy_instances):
iterator = BasicIterator(batch_size=2, instances_per_epoch=100)
in_order_batches = list(iterator._create_batches(test_instances, shuffle=False))
shuffled_batches = list(iterator._create_batches(test_instances, shuffle=True))
assert len(in_order_batches) == len(shuffled_batches)
# With 100 instances, shuffling better change the order.
assert in_order_batches != shuffled_batches
# But not the counts of the instances.
in_order_counts = Counter(id(instance) for batch in in_order_batches for instance in batch)
shuffled_counts = Counter(id(instance) for batch in shuffled_batches for instance in batch)
assert in_order_counts == shuffled_counts示例3: test_create_batches_groups_correctly
# 需要導入模塊: from allennlp.data.iterators import BasicIterator [as 別名]
# 或者: from allennlp.data.iterators.BasicIterator import _create_batches [as 別名]
def test_create_batches_groups_correctly(self):
# pylint: disable=protected-access
for test_instances in (self.instances, self.lazy_instances):
iterator = BasicIterator(batch_size=2)
batches = list(iterator._create_batches(test_instances, shuffle=False))
grouped_instances = [batch.instances for batch in batches]
assert grouped_instances == [[self.instances[0], self.instances[1]],
[self.instances[2], self.instances[3]],
[self.instances[4]]]示例4: test_max_instances_in_memory
# 需要導入模塊: from allennlp.data.iterators import BasicIterator [as 別名]
# 或者: from allennlp.data.iterators.BasicIterator import _create_batches [as 別名]
def test_max_instances_in_memory(self):
# pylint: disable=protected-access
for test_instances in (self.instances, self.lazy_instances):
iterator = BasicIterator(batch_size=2, max_instances_in_memory=3)
# One epoch: 5 instances -> [2, 1, 2]
batches = list(iterator._create_batches(test_instances, shuffle=False))
grouped_instances = [batch.instances for batch in batches]
assert grouped_instances == [[self.instances[0], self.instances[1]],
[self.instances[2]],
[self.instances[3], self.instances[4]]]示例5: test_many_instances_per_epoch
# 需要導入模塊: from allennlp.data.iterators import BasicIterator [as 別名]
# 或者: from allennlp.data.iterators.BasicIterator import _create_batches [as 別名]
def test_many_instances_per_epoch(self):
# pylint: disable=protected-access
for test_instances in (self.instances, self.lazy_instances):
iterator = BasicIterator(batch_size=2, instances_per_epoch=7)
# First epoch: 7 instances -> [2, 2, 2, 1]
batches = list(iterator._create_batches(test_instances, shuffle=False))
grouped_instances = [batch.instances for batch in batches]
assert grouped_instances == [[self.instances[0], self.instances[1]],
[self.instances[2], self.instances[3]],
[self.instances[4], self.instances[0]],
[self.instances[1]]]
# Second epoch: 7 instances -> [2, 2, 2, 1]
batches = list(iterator._create_batches(test_instances, shuffle=False))
grouped_instances = [batch.instances for batch in batches]
assert grouped_instances == [[self.instances[2], self.instances[3]],
[self.instances[4], self.instances[0]],
[self.instances[1], self.instances[2]],
[self.instances[3]]]示例6: test_maximum_samples_per_batch
# 需要導入模塊: from allennlp.data.iterators import BasicIterator [as 別名]
# 或者: from allennlp.data.iterators.BasicIterator import _create_batches [as 別名]
def test_maximum_samples_per_batch(self):
for test_instances in (self.instances, self.lazy_instances):
# pylint: disable=protected-access
iterator = BasicIterator(
batch_size=3, maximum_samples_per_batch=['num_tokens', 9]
)
iterator.index_with(self.vocab)
batches = list(iterator._create_batches(test_instances, shuffle=False))
stats = self.get_batches_stats(batches)
# ensure all instances are in a batch
assert stats['total_instances'] == len(self.instances)
# ensure correct batch sizes
assert stats['batch_lengths'] == [2, 1, 1, 1]
# ensure correct sample sizes (<= 9)
assert stats['sample_sizes'] == [8, 3, 9, 1]示例7: test_maximum_samples_per_batch_packs_tightly
# 需要導入模塊: from allennlp.data.iterators import BasicIterator [as 別名]
# 或者: from allennlp.data.iterators.BasicIterator import _create_batches [as 別名]
def test_maximum_samples_per_batch_packs_tightly(self):
# pylint: disable=protected-access
token_counts = [10, 4, 3]
test_instances = self.create_instances_from_token_counts(token_counts)
iterator = BasicIterator(
batch_size=3, maximum_samples_per_batch=['num_tokens', 11]
)
iterator.index_with(self.vocab)
batches = list(iterator._create_batches(test_instances, shuffle=False))
stats = self.get_batches_stats(batches)
# ensure all instances are in a batch
assert stats['total_instances'] == len(token_counts)
# ensure correct batch sizes
assert stats['batch_lengths'] == [1, 2]
# ensure correct sample sizes (<= 11)
assert stats['sample_sizes'] == [10, 8]示例8: test_maximum_samples_per_batch
# 需要導入模塊: from allennlp.data.iterators import BasicIterator [as 別名]
# 或者: from allennlp.data.iterators.BasicIterator import _create_batches [as 別名]
def test_maximum_samples_per_batch(self):
for test_instances in (self.instances, self.lazy_instances):
# pylint: disable=protected-access
iterator = BasicIterator(
batch_size=3, maximum_samples_per_batch=['num_tokens', 9]
)
batches = list(iterator._create_batches(test_instances, shuffle=False))
# ensure all instances are in a batch
grouped_instances = [batch.instances for batch in batches]
num_instances = sum(len(group) for group in grouped_instances)
assert num_instances == len(self.instances)
# ensure all batches are sufficiently small
for batch in batches:
batch_sequence_length = max(
[instance.get_padding_lengths()['text']['num_tokens']
for instance in batch.instances]
)
assert batch_sequence_length * len(batch.instances) <= 9