本文整理汇总了Python中fuel.transformers.Padding方法的典型用法代码示例。如果您正苦于以下问题：Python transformers.Padding方法的具体用法？Python transformers.Padding怎么用？Python transformers.Padding使用的例子？那么恭喜您, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类fuel.transformers的用法示例。
在下文中一共展示了transformers.Padding方法的8个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: setup_datastream
# 需要导入模块: from fuel import transformers [as 别名]
# 或者: from fuel.transformers import Padding [as 别名]
def setup_datastream(path, vocab_file, config):
ds = QADataset(path, vocab_file, config.n_entities, need_sep_token=config.concat_ctx_and_question)
it = QAIterator(path, shuffle=config.shuffle_questions)
stream = DataStream(ds, iteration_scheme=it)
if config.concat_ctx_and_question:
stream = ConcatCtxAndQuestion(stream, config.concat_question_before, ds.reverse_vocab['<SEP>'])
# Sort sets of multiple batches to make batches of similar sizes
stream = Batch(stream, iteration_scheme=ConstantScheme(config.batch_size * config.sort_batch_count))
comparison = _balanced_batch_helper(stream.sources.index('question' if config.concat_ctx_and_question else 'context'))
stream = Mapping(stream, SortMapping(comparison))
stream = Unpack(stream)
stream = Batch(stream, iteration_scheme=ConstantScheme(config.batch_size))
stream = Padding(stream, mask_sources=['context', 'question', 'candidates'], mask_dtype='int32')
return ds, stream 示例2: setup_datastream
# 需要导入模块: from fuel import transformers [as 别名]
# 或者: from fuel.transformers import Padding [as 别名]
def setup_datastream(path, batch_size, sort_batch_count, valid=False):
A = numpy.load(os.path.join(path, ('valid_x_raw.npy' if valid else 'train_x_raw.npy')))
B = numpy.load(os.path.join(path, ('valid_phn.npy' if valid else 'train_phn.npy')))
C = numpy.load(os.path.join(path, ('valid_seq_to_phn.npy' if valid else 'train_seq_to_phn.npy')))
D = [B[x[0]:x[1], 2] for x in C]
ds = IndexableDataset({'input': A, 'output': D})
stream = DataStream(ds, iteration_scheme=ShuffledExampleScheme(len(A)))
stream = Batch(stream, iteration_scheme=ConstantScheme(batch_size * sort_batch_count))
comparison = _balanced_batch_helper(stream.sources.index('input'))
stream = Mapping(stream, SortMapping(comparison))
stream = Unpack(stream)
stream = Batch(stream, iteration_scheme=ConstantScheme(batch_size, num_examples=len(A)))
stream = Padding(stream, mask_sources=['input', 'output'])
return ds, stream 示例3: obtain_stream
# 需要导入模块: from fuel import transformers [as 别名]
# 或者: from fuel.transformers import Padding [as 别名]
def obtain_stream(dataset, batch_size, size=1):
if size == 1:
data_stream = dataset.get_example_stream()
data_stream = transformers.Batch(data_stream, iteration_scheme=schemes.ConstantScheme(batch_size))
# add padding and masks to the dataset
data_stream = transformers.Padding(data_stream, mask_sources=('data'))
return data_stream
else:
data_streams = [dataset.get_example_stream() for _ in range(size)]
data_streams = [transformers.Batch(data_stream, iteration_scheme=schemes.ConstantScheme(batch_size))
for data_stream in data_streams]
data_streams = [transformers.Padding(data_stream, mask_sources=('data')) for data_stream in data_streams]
return data_streams 示例4: output_stream
# 需要导入模块: from fuel import transformers [as 别名]
# 或者: from fuel.transformers import Padding [as 别名]
def output_stream(dataset, batch_size, size=1):
data_stream = dataset.get_example_stream()
data_stream = transformers.Batch(data_stream,
iteration_scheme=schemes.ConstantScheme(batch_size))
# add padding and masks to the dataset
# Warning: in multiple output case, will raise ValueError: All dimensions except length must be equal, need padding manually
# data_stream = transformers.Padding(data_stream, mask_sources=('source', 'target', 'target_c'))
# data_stream = transformers.Padding(data_stream, mask_sources=('source', 'target'))
return data_stream 示例5: setup_datastream
# 需要导入模块: from fuel import transformers [as 别名]
# 或者: from fuel.transformers import Padding [as 别名]
def setup_datastream(batch_size, **kwargs):
ds = ToyDataset(**kwargs)
stream = DataStream(ds, iteration_scheme=SequentialExampleScheme(kwargs['nb_examples']))
stream = Batch(stream, iteration_scheme=ConstantScheme(batch_size))
stream = Padding(stream, mask_sources=['input', 'output'])
return ds, stream 示例6: output_stream
# 需要导入模块: from fuel import transformers [as 别名]
# 或者: from fuel.transformers import Padding [as 别名]
def output_stream(dataset, batch_size, size=1):
data_stream = dataset.get_example_stream()
data_stream = transformers.Batch(data_stream,
iteration_scheme=schemes.ConstantScheme(batch_size))
# add padding and masks to the dataset
data_stream = transformers.Padding(data_stream, mask_sources=('source', 'target'))
return data_stream 示例7: output_stream
# 需要导入模块: from fuel import transformers [as 别名]
# 或者: from fuel.transformers import Padding [as 别名]
def output_stream(dataset, batch_size, size=1):
data_stream = dataset.get_example_stream()
data_stream = transformers.Batch(data_stream,
iteration_scheme=schemes.ConstantScheme(batch_size))
# add padding and masks to the dataset
data_stream = transformers.Padding(data_stream, mask_sources=('source', 'target', 'target_c'))
return data_stream 示例8: obtain_stream
# 需要导入模块: from fuel import transformers [as 别名]
# 或者: from fuel.transformers import Padding [as 别名]
def obtain_stream(dataset, batch_size, size=1):
if size == 1:
data_stream = dataset.get_example_stream()
data_stream = transformers.Batch(data_stream, iteration_scheme=schemes.ConstantScheme(batch_size))
# add padding and masks to the dataset
data_stream = transformers.Padding(data_stream, mask_sources=('data'))
return data_stream
else:
data_streams = [dataset.get_example_stream() for _ in xrange(size)]
data_streams = [transformers.Batch(data_stream, iteration_scheme=schemes.ConstantScheme(batch_size))
for data_stream in data_streams]
data_streams = [transformers.Padding(data_stream, mask_sources=('data')) for data_stream in data_streams]
return data_streams