本文整理汇总了Python中torchtext.vocab.append方法的典型用法代码示例。如果您正苦于以下问题：Python vocab.append方法的具体用法？Python vocab.append怎么用？Python vocab.append使用的例子？那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类torchtext.vocab的用法示例。
在下文中一共展示了vocab.append方法的9个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: create_batches
# 需要导入模块: from torchtext import vocab [as 别名]
# 或者: from torchtext.vocab import append [as 别名]
def create_batches(self):
if self.train:
def pool(data, random_shuffler):
for p in torchtext.data.batch(data, self.batch_size * 100):
p_batch = torchtext.data.batch(
sorted(p, key=self.sort_key),
self.batch_size, self.batch_size_fn)
for b in random_shuffler(list(p_batch)):
yield b
self.batches = pool(self.data(), self.random_shuffler)
else:
self.batches = []
# print(self.data().src)
# print(self.data().node1)
for b in torchtext.data.batch(self.data(), self.batch_size,
self.batch_size_fn):
# print(b[0].src)
# print(b[0].node1)
self.batches.append(sorted(b, key=self.sort_key)) 示例2: build_vocab
# 需要导入模块: from torchtext import vocab [as 别名]
# 或者: from torchtext.vocab import append [as 别名]
def build_vocab(train, dev, test, opt):
fields = train.fields
merge_list = []
merge_name_list = ('src', 'tbl')
for split in (dev, test, train,):
for merge_name_it in merge_name_list:
fields[merge_name_it].build_vocab(
split, max_size=opt.src_vocab_size, min_freq=0)
merge_list.append(fields[merge_name_it].vocab)
# build vocabulary only based on the training set
fields["ent"].build_vocab(
train, max_size=opt.src_vocab_size, min_freq=0)
fields["lay"].build_vocab(
train, max_size=opt.src_vocab_size, min_freq=0)
fields["cond_op"].build_vocab(
train, max_size=opt.src_vocab_size, min_freq=0)
# need to know all the words to filter the pretrained word embeddings
merged_vocab = merge_vocabs(merge_list, vocab_size=opt.src_vocab_size)
for merge_name_it in merge_name_list:
fields[merge_name_it].vocab = merged_vocab 示例3: create_batches
# 需要导入模块: from torchtext import vocab [as 别名]
# 或者: from torchtext.vocab import append [as 别名]
def create_batches(self):
""" Create batches """
if self.train:
def _pool(data, random_shuffler):
for p in torchtext.data.batch(data, self.batch_size * 100):
p_batch = torchtext.data.batch(
sorted(p, key=self.sort_key),
self.batch_size, self.batch_size_fn)
for b in random_shuffler(list(p_batch)):
yield b
self.batches = _pool(self.data(), self.random_shuffler)
else:
self.batches = []
for b in torchtext.data.batch(self.data(), self.batch_size,
self.batch_size_fn):
self.batches.append(sorted(b, key=self.sort_key)) 示例4: save_fields_to_vocab
# 需要导入模块: from torchtext import vocab [as 别名]
# 或者: from torchtext.vocab import append [as 别名]
def save_fields_to_vocab(fields):
"""
Save Vocab objects in Field objects to `vocab.pt` file.
"""
vocab = []
for k, f in fields.items():
if f is not None and 'vocab' in f.__dict__:
f.vocab.stoi = dict(f.vocab.stoi)
vocab.append((k, f.vocab))
return vocab 示例5: collect_features
# 需要导入模块: from torchtext import vocab [as 别名]
# 或者: from torchtext.vocab import append [as 别名]
def collect_features(fields, side="src"):
"""
Collect features from Field object.
"""
assert side in ["src", "tgt"]
feats = []
for j in count():
key = side + "_feat_" + str(j)
if key not in fields:
break
feats.append(key)
return feats 示例6: collect_feature_vocabs
# 需要导入模块: from torchtext import vocab [as 别名]
# 或者: from torchtext.vocab import append [as 别名]
def collect_feature_vocabs(fields, side):
"""
Collect feature Vocab objects from Field object.
"""
assert side in ['src', 'tgt']
feature_vocabs = []
for j in count():
key = side + "_feat_" + str(j)
if key not in fields:
break
feature_vocabs.append(fields[key].vocab)
return feature_vocabs 示例7: create_batches
# 需要导入模块: from torchtext import vocab [as 别名]
# 或者: from torchtext.vocab import append [as 别名]
def create_batches(self):
if self.train:
def pool(data, random_shuffler):
for p in torchtext.data.batch(data, self.batch_size * 100):
p_batch = torchtext.data.batch(
sorted(p, key=self.sort_key),
self.batch_size, self.batch_size_fn)
for b in random_shuffler(list(p_batch)):
yield b
self.batches = pool(self.data(), self.random_shuffler)
else:
self.batches = []
for b in torchtext.data.batch(self.data(), self.batch_size,
self.batch_size_fn):
self.batches.append(sorted(b, key=self.sort_key)) 示例8: create_batches
# 需要导入模块: from torchtext import vocab [as 别名]
# 或者: from torchtext.vocab import append [as 别名]
def create_batches(self):
if self.train:
self.batches = torchtext.data.pool(
self.data(), self.batch_size,
self.sort_key, self.batch_size_fn,
random_shuffler=self.random_shuffler)
else:
self.batches = []
for b in torchtext.data.batch(self.data(), self.batch_size,
self.batch_size_fn):
self.batches.append(sorted(b, key=self.sort_key)) 示例9: save_vocab
# 需要导入模块: from torchtext import vocab [as 别名]
# 或者: from torchtext.vocab import append [as 别名]
def save_vocab(fields):
vocab = []
for k, f in fields.items():
if 'vocab' in f.__dict__:
f.vocab.stoi = dict(f.vocab.stoi)
vocab.append((k, f.vocab))
return vocab