Python datasets.IndexableDataset方法代码示例

# 需要导入模块: from fuel import datasets [as 别名]
# 或者: from fuel.datasets import IndexableDataset [as 别名]
def setup_datastream(path, batch_size, sort_batch_count, valid=False):
    A = numpy.load(os.path.join(path, ('valid_x_raw.npy' if valid else 'train_x_raw.npy')))
    B = numpy.load(os.path.join(path, ('valid_phn.npy' if valid else 'train_phn.npy')))
    C = numpy.load(os.path.join(path, ('valid_seq_to_phn.npy' if valid else 'train_seq_to_phn.npy')))

    D = [B[x[0]:x[1], 2] for x in C]

    ds = IndexableDataset({'input': A, 'output': D})
    stream = DataStream(ds, iteration_scheme=ShuffledExampleScheme(len(A)))

    stream = Batch(stream, iteration_scheme=ConstantScheme(batch_size * sort_batch_count))
    comparison = _balanced_batch_helper(stream.sources.index('input'))
    stream = Mapping(stream, SortMapping(comparison))
    stream = Unpack(stream)

    stream = Batch(stream, iteration_scheme=ConstantScheme(batch_size, num_examples=len(A)))
    stream = Padding(stream, mask_sources=['input', 'output'])

    return ds, stream 

# 需要导入模块: from fuel import datasets [as 别名]
# 或者: from fuel.datasets import IndexableDataset [as 别名]
def build_fuel(data):
    # create fuel dataset.
    dataset     = datasets.IndexableDataset(indexables=OrderedDict([('data', data)]))
    dataset.example_iteration_scheme \
                = schemes.ShuffledExampleScheme(dataset.num_examples)
    return dataset, len(data) 

# 需要导入模块: from fuel import datasets [as 别名]
# 或者: from fuel.datasets import IndexableDataset [as 别名]
def build_data(data):
    # create fuel dataset.
    dataset = datasets.IndexableDataset(indexables=OrderedDict([('source', data['source']),
                                                                ('target', data['target']),
                                                                # ('target_c', data['target_c']),
                                                                ]))
    dataset.example_iteration_scheme \
        = schemes.ShuffledExampleScheme(dataset.num_examples)
    return dataset 

# 需要导入模块: from fuel import datasets [as 别名]
# 或者: from fuel.datasets import IndexableDataset [as 别名]
def test_mean_aggregator():
    num_examples = 4
    batch_size = 2

    features = numpy.array([[0, 3],
                           [2, 9],
                           [2, 4],
                           [5, 1]], dtype=theano.config.floatX)

    dataset = IndexableDataset(OrderedDict([('features', features)]))

    data_stream = DataStream(dataset,
                             iteration_scheme=SequentialScheme(num_examples,
                                                               batch_size))

    x = tensor.matrix('features')
    y = (x**2).mean(axis=0)
    y.name = 'y'
    z = y.sum()
    z.name = 'z'

    y.tag.aggregation_scheme = Mean(y, 1.)
    z.tag.aggregation_scheme = Mean(z, 1.)

    assert_allclose(DatasetEvaluator([y]).evaluate(data_stream)['y'],
                    numpy.array([8.25, 26.75], dtype=theano.config.floatX))
    assert_allclose(DatasetEvaluator([z]).evaluate(data_stream)['z'],
                    numpy.array([35], dtype=theano.config.floatX)) 

# 需要导入模块: from fuel import datasets [as 别名]
# 或者: from fuel.datasets import IndexableDataset [as 别名]
def test_getattr(self):
        assert_equal(getattr(IndexableDataset({'a': (1, 2)}), 'a'), (1, 2)) 

# 需要导入模块: from fuel import datasets [as 别名]
# 或者: from fuel.datasets import IndexableDataset [as 别名]
def test_value_error_get_data_state(self):
        assert_raises(
            ValueError, IndexableDataset([1, 2, 3]).get_data, True, [1, 2]) 

# 需要导入模块: from fuel import datasets [as 别名]
# 或者: from fuel.datasets import IndexableDataset [as 别名]
def test_value_error_get_data_none_request(self):
        assert_raises(
            ValueError, IndexableDataset([1, 2, 3]).get_data, None, None) 

# 需要导入模块: from fuel import datasets [as 别名]
# 或者: from fuel.datasets import IndexableDataset [as 别名]
def test_pickling(self):
        cPickle.loads(cPickle.dumps(IndexableDataset({'a': (1, 2)}))) 

# 需要导入模块: from fuel import datasets [as 别名]
# 或者: from fuel.datasets import IndexableDataset [as 别名]
def test_num_examples():
    assert_raises(ValueError, IterableDataset,
                  {'features': range(10), 'targets': range(7)})
    dataset = IterableDataset({'features': range(7),
                               'targets': range(7)})
    assert dataset.num_examples == 7
    dataset = IterableDataset(repeat(1))
    assert numpy.isnan(dataset.num_examples)
    x = numpy.random.rand(5, 3)
    y = numpy.random.rand(5, 4)
    dataset = IndexableDataset({'features': x, 'targets': y})
    assert dataset.num_examples == 5
    assert_raises(ValueError, IndexableDataset,
                  {'features': x, 'targets': y[:4]}) 

# 需要导入模块: from fuel import datasets [as 别名]
# 或者: from fuel.datasets import IndexableDataset [as 别名]
def test_axis_labels_on_produces_batches(self):
        dataset = IndexableDataset(numpy.eye(2))
        axis_labels = {'data': ('batch', 'features')}
        dataset.axis_labels = axis_labels
        stream = DataStream(dataset, iteration_scheme=SequentialScheme(2, 2))
        assert_equal(stream.axis_labels, axis_labels) 

# 需要导入模块: from fuel import datasets [as 别名]
# 或者: from fuel.datasets import IndexableDataset [as 别名]
def test_flatten_batches(self):
        wrapper = Flatten(
            DataStream(IndexableDataset(self.data),
                       iteration_scheme=SequentialScheme(4, 2)),
            which_sources=('features',))
        assert_equal(
            list(wrapper.get_epoch_iterator()),
            [(numpy.ones((2, 4)), numpy.array([[0], [1]])),
             (numpy.ones((2, 4)), numpy.array([[0], [1]]))]) 

# 需要导入模块: from fuel import datasets [as 别名]
# 或者: from fuel.datasets import IndexableDataset [as 别名]
def test_axis_labels_on_flatten_batches(self):
        wrapper = Flatten(
            DataStream(IndexableDataset(self.data),
                       iteration_scheme=SequentialScheme(4, 2),
                       axis_labels={'features': ('batch', 'width', 'height'),
                                    'targets': ('batch', 'index')}),
            which_sources=('features',))
        assert_equal(wrapper.axis_labels, {'features': ('batch', 'feature'),
                                           'targets': ('batch', 'index')}) 

# 需要导入模块: from fuel import datasets [as 别名]
# 或者: from fuel.datasets import IndexableDataset [as 别名]
def test_axis_labels_on_flatten_batches_with_none(self):
        wrapper = Flatten(
            DataStream(IndexableDataset(self.data),
                       iteration_scheme=SequentialScheme(4, 2),
                       axis_labels={'features': None,
                                    'targets': ('batch', 'index')}),
            which_sources=('features',))
        assert_equal(wrapper.axis_labels, {'features': None,
                                           'targets': ('batch', 'index')}) 

# 需要导入模块: from fuel import datasets [as 别名]
# 或者: from fuel.datasets import IndexableDataset [as 别名]
def test_axis_labels_on_flatten_examples(self):
        wrapper = Flatten(
            DataStream(IndexableDataset(self.data),
                       iteration_scheme=SequentialExampleScheme(4),
                       axis_labels={'features': ('batch', 'width', 'height'),
                                    'targets': ('batch', 'index')}),
            which_sources=('features',))
        assert_equal(wrapper.axis_labels, {'features': ('feature',),
                                           'targets': ('index',)}) 

# 需要导入模块: from fuel import datasets [as 别名]
# 或者: from fuel.datasets import IndexableDataset [as 别名]
def test_filter_batches(self):
        data = [1, 2, 3, 4]
        data_filtered = [([3, 4],)]
        stream = DataStream(IndexableDataset(data),
                            iteration_scheme=SequentialScheme(4, 2))
        wrapper = Filter(stream, lambda d: d[0][0] % 3 == 0)
        assert_equal(list(wrapper.get_epoch_iterator()), data_filtered)