Python transformers.Batch方法代碼示例

本文整理匯總了Python中fuel.transformers.Batch方法的典型用法代碼示例。如果您正苦於以下問題：Python transformers.Batch方法的具體用法？Python transformers.Batch怎麽用？Python transformers.Batch使用的例子？那麽, 這裏精選的方法代碼示例或許可以為您提供幫助。您也可以進一步了解該方法所在類fuel.transformers的用法示例。

在下文中一共展示了transformers.Batch方法的9個代碼示例，這些例子默認根據受歡迎程度排序。您可以為喜歡或者感覺有用的代碼點讚，您的評價將有助於係統推薦出更棒的Python代碼示例。

示例1: setup_datastream

# 需要導入模塊: from fuel import transformers [as 別名]
# 或者: from fuel.transformers import Batch [as 別名]
def setup_datastream(path, vocab_file, config):
    ds = QADataset(path, vocab_file, config.n_entities, need_sep_token=config.concat_ctx_and_question)
    it = QAIterator(path, shuffle=config.shuffle_questions)

    stream = DataStream(ds, iteration_scheme=it)

    if config.concat_ctx_and_question:
        stream = ConcatCtxAndQuestion(stream, config.concat_question_before, ds.reverse_vocab['<SEP>'])

    # Sort sets of multiple batches to make batches of similar sizes
    stream = Batch(stream, iteration_scheme=ConstantScheme(config.batch_size * config.sort_batch_count))
    comparison = _balanced_batch_helper(stream.sources.index('question' if config.concat_ctx_and_question else 'context'))
    stream = Mapping(stream, SortMapping(comparison))
    stream = Unpack(stream)

    stream = Batch(stream, iteration_scheme=ConstantScheme(config.batch_size))
    stream = Padding(stream, mask_sources=['context', 'question', 'candidates'], mask_dtype='int32')

    return ds, stream

開發者ID:thomasmesnard，項目名稱:DeepMind-Teaching-Machines-to-Read-and-Comprehend，代碼行數:21，代碼來源:data.py

示例2: setup_datastream

# 需要導入模塊: from fuel import transformers [as 別名]
# 或者: from fuel.transformers import Batch [as 別名]
def setup_datastream(path, batch_size, sort_batch_count, valid=False):
    A = numpy.load(os.path.join(path, ('valid_x_raw.npy' if valid else 'train_x_raw.npy')))
    B = numpy.load(os.path.join(path, ('valid_phn.npy' if valid else 'train_phn.npy')))
    C = numpy.load(os.path.join(path, ('valid_seq_to_phn.npy' if valid else 'train_seq_to_phn.npy')))

    D = [B[x[0]:x[1], 2] for x in C]

    ds = IndexableDataset({'input': A, 'output': D})
    stream = DataStream(ds, iteration_scheme=ShuffledExampleScheme(len(A)))

    stream = Batch(stream, iteration_scheme=ConstantScheme(batch_size * sort_batch_count))
    comparison = _balanced_batch_helper(stream.sources.index('input'))
    stream = Mapping(stream, SortMapping(comparison))
    stream = Unpack(stream)

    stream = Batch(stream, iteration_scheme=ConstantScheme(batch_size, num_examples=len(A)))
    stream = Padding(stream, mask_sources=['input', 'output'])

    return ds, stream

開發者ID:thomasmesnard，項目名稱:CTC-LSTM，代碼行數:21，代碼來源:timit.py

示例3: obtain_stream

# 需要導入模塊: from fuel import transformers [as 別名]
# 或者: from fuel.transformers import Batch [as 別名]
def obtain_stream(dataset, batch_size, size=1):
    if size == 1:
        data_stream = dataset.get_example_stream()
        data_stream = transformers.Batch(data_stream, iteration_scheme=schemes.ConstantScheme(batch_size))

        # add padding and masks to the dataset
        data_stream = transformers.Padding(data_stream, mask_sources=('data'))
        return data_stream
    else:
        data_streams = [dataset.get_example_stream() for _ in range(size)]
        data_streams = [transformers.Batch(data_stream, iteration_scheme=schemes.ConstantScheme(batch_size))
                        for data_stream in data_streams]
        data_streams = [transformers.Padding(data_stream, mask_sources=('data')) for data_stream in data_streams]
        return data_streams

開發者ID:memray，項目名稱:seq2seq-keyphrase，代碼行數:16，代碼來源:build_dataset.py

示例4: output_stream

# 需要導入模塊: from fuel import transformers [as 別名]
# 或者: from fuel.transformers import Batch [as 別名]
def output_stream(dataset, batch_size, size=1):
    data_stream = dataset.get_example_stream()
    data_stream = transformers.Batch(data_stream,
                                     iteration_scheme=schemes.ConstantScheme(batch_size))

    # add padding and masks to the dataset
    # Warning: in multiple output case, will raise ValueError: All dimensions except length must be equal, need padding manually
    # data_stream = transformers.Padding(data_stream, mask_sources=('source', 'target', 'target_c'))
    # data_stream = transformers.Padding(data_stream, mask_sources=('source', 'target'))
    return data_stream

開發者ID:memray，項目名稱:seq2seq-keyphrase，代碼行數:12，代碼來源:keyphrase_copynet.py

示例5: setup_datastream

# 需要導入模塊: from fuel import transformers [as 別名]
# 或者: from fuel.transformers import Batch [as 別名]
def setup_datastream(batch_size, **kwargs):
    ds = ToyDataset(**kwargs)
    stream = DataStream(ds, iteration_scheme=SequentialExampleScheme(kwargs['nb_examples']))

    stream = Batch(stream, iteration_scheme=ConstantScheme(batch_size))
    stream = Padding(stream, mask_sources=['input', 'output'])

    return ds, stream

開發者ID:thomasmesnard，項目名稱:CTC-LSTM，代碼行數:10，代碼來源:toy_dataset.py

示例6: get_data_stream

# 需要導入模塊: from fuel import transformers [as 別名]
# 或者: from fuel.transformers import Batch [as 別名]
def get_data_stream(iterable):
    """Returns a 'fuel.Batch' datastream of
    [x~input~numbers, y~targets~roots], with each iteration returning a
    batch of 20 training examples
    """
    numbers = numpy.asarray(iterable, dtype=floatX)
    dataset = IterableDataset(
        {'numbers': numbers, 'roots': numpy.sqrt(numbers)})
    return Batch(dataset.get_example_stream(), ConstantScheme(20))

開發者ID:mila-iqia，項目名稱:blocks-examples，代碼行數:11，代碼來源:__init__.py

示例7: output_stream

# 需要導入模塊: from fuel import transformers [as 別名]
# 或者: from fuel.transformers import Batch [as 別名]
def output_stream(dataset, batch_size, size=1):
        data_stream = dataset.get_example_stream()
        data_stream = transformers.Batch(data_stream,
                                         iteration_scheme=schemes.ConstantScheme(batch_size))

        # add padding and masks to the dataset
        data_stream = transformers.Padding(data_stream, mask_sources=('source', 'target'))
        return data_stream

開發者ID:MultiPath，項目名稱:CopyNet，代碼行數:10，代碼來源:weibo_vest.py

示例8: output_stream

# 需要導入模塊: from fuel import transformers [as 別名]
# 或者: from fuel.transformers import Batch [as 別名]
def output_stream(dataset, batch_size, size=1):
        data_stream = dataset.get_example_stream()
        data_stream = transformers.Batch(data_stream,
                                         iteration_scheme=schemes.ConstantScheme(batch_size))

        # add padding and masks to the dataset
        data_stream = transformers.Padding(data_stream, mask_sources=('source', 'target', 'target_c'))
        return data_stream

開發者ID:MultiPath，項目名稱:CopyNet，代碼行數:10，代碼來源:lcsts_test.py

示例9: obtain_stream

# 需要導入模塊: from fuel import transformers [as 別名]
# 或者: from fuel.transformers import Batch [as 別名]
def obtain_stream(dataset, batch_size, size=1):
    if size == 1:
        data_stream = dataset.get_example_stream()
        data_stream = transformers.Batch(data_stream, iteration_scheme=schemes.ConstantScheme(batch_size))

        # add padding and masks to the dataset
        data_stream = transformers.Padding(data_stream, mask_sources=('data'))
        return data_stream
    else:
        data_streams = [dataset.get_example_stream() for _ in xrange(size)]
        data_streams = [transformers.Batch(data_stream, iteration_scheme=schemes.ConstantScheme(batch_size))
                        for data_stream in data_streams]
        data_streams = [transformers.Padding(data_stream, mask_sources=('data')) for data_stream in data_streams]
        return data_streams

開發者ID:MultiPath，項目名稱:CopyNet，代碼行數:16，代碼來源:build_dataset.py

注：本文中的fuel.transformers.Batch方法示例由純淨天空整理自Github/MSDocs等開源代碼及文檔管理平台，相關代碼片段篩選自各路編程大神貢獻的開源項目，源碼版權歸原作者所有，傳播和使用請參考對應項目的License；未經允許，請勿轉載。