本文整理汇总了Python中torchtext.vocab.pt方法的典型用法代码示例。如果您正苦于以下问题：Python vocab.pt方法的具体用法？Python vocab.pt怎么用？Python vocab.pt使用的例子？那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类torchtext.vocab的用法示例。
在下文中一共展示了vocab.pt方法的6个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: _load_fields
# 需要导入模块: from torchtext import vocab [as 别名]
# 或者: from torchtext.vocab import pt [as 别名]
def _load_fields(dataset, data_type, opt, checkpoint):
if checkpoint is not None:
logger.info('Loading vocab from checkpoint at %s.' % opt.train_from)
fields = load_fields_from_vocab(
checkpoint['vocab'], data_type)
else:
fields = load_fields_from_vocab(
torch.load(opt.data + '.vocab.pt'), data_type)
fields = dict([(k, f) for (k, f) in fields.items()
if k in dataset.examples[0].__dict__])
if data_type == 'text':
logger.info(' * vocabulary size. source = %d; target = %d; template = %d'%
(len(fields['src'].vocab), len(fields['tgt'].vocab),len(fields['template'].vocab)))
else:
logger.info(' * vocabulary size. target = %d' %
(len(fields['tgt'].vocab)))
return fields 示例2: load_fields_from_vocab
# 需要导入模块: from torchtext import vocab [as 别名]
# 或者: from torchtext.vocab import pt [as 别名]
def load_fields_from_vocab(vocab, data_type="text"):
"""
Load Field objects from `vocab.pt` file.
"""
vocab = dict(vocab)
n_src_features = len(collect_features(vocab, 'src'))
n_tgt_features = len(collect_features(vocab, 'tgt'))
fields = get_fields(data_type, n_src_features, n_tgt_features)
for k, v in vocab.items():
# Hack. Can't pickle defaultdict :(
v.stoi = defaultdict(lambda: 0, v.stoi)
fields[k].vocab = v
return fields 示例3: save_fields_to_vocab
# 需要导入模块: from torchtext import vocab [as 别名]
# 或者: from torchtext.vocab import pt [as 别名]
def save_fields_to_vocab(fields):
"""
Save Vocab objects in Field objects to `vocab.pt` file.
"""
vocab = []
for k, f in fields.items():
if f is not None and 'vocab' in f.__dict__:
f.vocab.stoi = dict(f.vocab.stoi)
vocab.append((k, f.vocab))
return vocab 示例4: save_fields_to_vocab
# 需要导入模块: from torchtext import vocab [as 别名]
# 或者: from torchtext.vocab import pt [as 别名]
def save_fields_to_vocab(fields):
"""
Save Vocab objects in Field objects to `vocab.pt` file.
"""
vocab = []
for k, f in fields.items():
if 'vocab' in f.__dict__:
f.vocab.stoi = dict(f.vocab.stoi)
vocab.append((k, f.vocab))
return vocab 示例5: save_fields_to_vocab
# 需要导入模块: from torchtext import vocab [as 别名]
# 或者: from torchtext.vocab import pt [as 别名]
def save_fields_to_vocab(fields):
"""
Save Vocab objects in Field objects to `vocab.pt` file.
"""
vocab = []
for k, f in fields.items():
if f is not None and 'vocab' in f.__dict__:
f.vocab.stoi = f.vocab.stoi
vocab.append((k, f.vocab))
return vocab 示例6: lazily_load_dataset
# 需要导入模块: from torchtext import vocab [as 别名]
# 或者: from torchtext.vocab import pt [as 别名]
def lazily_load_dataset(corpus_type, opt):
"""
Dataset generator. Don't do extra stuff here, like printing,
because they will be postponed to the first loading time.
Args:
corpus_type: 'train' or 'valid'
Returns:
A list of dataset, the dataset(s) are lazily loaded.
"""
assert corpus_type in ["train", "valid"]
def _lazy_dataset_loader(pt_file, corpus_type):
dataset = torch.load(pt_file)
logger.info('Loading %s dataset from %s, number of examples: %d' %
(corpus_type, pt_file, len(dataset)))
return dataset
# Sort the glob output by file name (by increasing indexes).
pts = sorted(glob.glob(opt.data + '.' + corpus_type + '.[0-9]*.pt'))
if pts:
for pt in pts:
yield _lazy_dataset_loader(pt, corpus_type)
else:
# Only one inputters.*Dataset, simple!
pt = opt.data + '.' + corpus_type + '.pt'
yield _lazy_dataset_loader(pt, corpus_type)