本文整理汇总了Python中pylearn2.utils.safe_izip函数的典型用法代码示例。如果您正苦于以下问题:Python safe_izip函数的具体用法?Python safe_izip怎么用?Python safe_izip使用的例子?那么恭喜您, 这里精选的函数代码示例或许可以为您提供帮助。
在下文中一共展示了safe_izip函数的15个代码示例,这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞,您的评价将有助于系统推荐出更棒的Python代码示例。
示例1: getFeatures
def getFeatures(indexes):
"""
.. todo::
Write me
"""
if self._load_to_memory:
sequences = [self.samples_sequences[i] for i in indexes]
else:
sequences = [self.node[i] for i in indexes]
# Get random source word index for "ngram"
source_i = [numpy.random.randint(self.frame_length/2 +1, len(s)-self.frame_length/2, 1)[0]
for s in sequences]
target_i = [min(abs(int(numpy.random.normal(s_i, self.frame_length/3.0))), len(s)-1)
for s_i, s in safe_izip(source_i, sequences)]
# Words mapped to integers greater than input max are set to 1 (unknown)
X = [numpy.asarray([s[i]]) for i, s in safe_izip(source_i, sequences)]
X[X>=self.X_labels] = numpy.asarray([1])
X = numpy.asarray(X)
y = [numpy.asarray([s[i]]) for i, s in safe_izip(target_i, sequences)]
y[y>=self.X_labels] = numpy.asarray([1])
y = numpy.asarray(y)
# Store the targets generated by these indices.
self.lastY = (y, indexes)
#print X
#print y
return X
示例2: _fallback_next
def _fallback_next(self, next_index):
# TODO: handle fancy-index copies by allocating a buffer and
# using np.take()
return tuple(
fn(data[next_index]) if fn else data[next_index]
for data, fn in safe_izip(self._raw_data, self._convert)
)
示例3: iterator
def iterator(self, mode=None, batch_size=None, num_batches=None,
rng=None, data_specs=None, return_tuple=False):
allowed_modes = ('sequential', 'random_slice', 'even_sequential',
'batchwise_shuffled_sequential',
'even_batchwise_shuffled_sequential')
if mode is not None and mode not in allowed_modes:
raise ValueError("Due to HDF5 limitations on advanced indexing, " +
"the '" + mode + "' iteration mode is not " +
"supported")
if data_specs is None:
data_specs = self._iter_data_specs
space, source = data_specs
sub_spaces, sub_sources = (
(space.components, source) if isinstance(space, CompositeSpace)
else ((space,), (source,)))
convert = [None for sp, src in safe_izip(sub_spaces, sub_sources)]
mode = (self._iter_subset_class if mode is None
else resolve_iterator_class(mode))
if batch_size is None:
batch_size = getattr(self, '_iter_batch_size', None)
if num_batches is None:
num_batches = getattr(self, '_iter_num_batches', None)
if rng is None and mode.stochastic:
rng = self.rng
return VariableImageDatasetIterator(
dataset=self,
subset_iterator=mode(
self.num_examples, batch_size, num_batches, rng),
data_specs=data_specs,
return_tuple=return_tuple,
convert=convert)
示例4: next
def next(self):
next_index = self._subset_iterator.next()
rvals = []
if hasattr(self._dataset, 'get'):
raw_data = self._next(next_index)
else:
raw_data = self._fallback_next(next_index)
for (space, source, data, fn) in safe_izip(self._space, self._source,
#self._raw_data,
raw_data,
self._convert):
rval = data
if isinstance(space, SequenceDataSpace):
# Add padding
max_sequence_length = max(len(sample) for sample
in data)
batch = np.zeros((len(rval), max_sequence_length,
space.dim), dtype=space.dtype)
for i, sample in enumerate(rval):
batch[i, :len(sample)] = sample
rvals.append(np.transpose(batch, (1, 0, 2)))
# Create mask
rvals.append(self._create_mask(rval))
else:
rvals.append(rval)
# Reorder according to given data specs
if not self._return_tuple and len(rval) == 1:
rvals, = rvals
return tuple(rvals)
示例5: __call__
def __call__(self, model, X, Y = None, ** kwargs):
def wrapped_layer_cost(layer, coef):
try:
return layer.get_weight_decay(coeff)
except NotImplementedError:
if coef==0.:
return 0.
else:
raise NotImplementedError(str(type(layer))+" does not implement get_weight_decay.")
layer_costs = [ wrapped_layer_cost(layer, coeff)
for layer, coeff in safe_izip(model.layers, self.coeffs) ]
assert T.scalar() != 0. # make sure theano semantics do what I want
layer_costs = [ cost for cost in layer_costs if cost != 0.]
if len(layer_costs) == 0:
rval = T.as_tensor_variable(0.)
rval.name = '0_weight_decay'
return rval
else:
total_cost = reduce(lambda x, y: x + y, layer_costs)
total_cost.name = 'MLP_WeightDecay'
assert total_cost.ndim == 0
total_cost.name = 'weight_decay'
return total_cost
示例6: next
def next(self):
next_index = self._subset_iterator.next()
rvals = []
for space, source, data, fn in safe_izip(self._space, self._source,
self._raw_data,
self._convert):
rval = data[next_index]
if isinstance(space, SequenceDataSpace):
# Add padding
max_sequence_length = max(len(sample) for sample
in rval)
batch = np.zeros((len(rval), max_sequence_length) +
data[0].shape[1:], dtype=data[0].dtype)
for i, sample in enumerate(rval):
batch[i, :len(sample)] = sample
# Create mask
if source in self.mask_needed:
mask = self._create_mask(rval)
rval = np.swapaxes(batch, 0, 1)
if fn:
rval = fn(rval)
rvals.append(rval)
if source in self.mask_needed:
rvals.append(mask)
else:
if fn:
rval = fn(rval)
rvals.append(rval)
# Reorder according to given data specs
if not self._return_tuple and len(rval) == 1:
rvals, = rvals
return tuple(rvals)
示例7: get_monitoring_channels
def get_monitoring_channels(self, X=None, Y=None):
"""
Note: X and Y may both be None, in the case when this is
a layer of a bigger MLP.
"""
state = X
rval = OrderedDict()
for layer, scale in safe_izip(self.mlp.layers, self._params):
state = self.scale(state, layer, scale)
ch = layer.get_monitoring_channels()
for key in ch:
rval[layer.layer_name+'_'+key] = ch[key]
state = layer.fprop(state)
args = [state]
if layer is self.mlp.layers[-1]:
args.append(Y)
ch = layer.get_monitoring_channels_from_state(*args)
for key in ch:
rval[layer.layer_name+'_'+key] = ch[key]
for i in xrange(len(self._params)):
rval['scale_input_to_' + self.mlp.layers[i].layer_name + '_min'] = self._params[i].min()
rval['scale_input_to_' + self.mlp.layers[i].layer_name + '_min'] = self._params[i].mean()
rval['scale_input_to_' + self.mlp.layers[i].layer_name + '_min'] = self._params[i].max()
return rval
示例8: expr
def expr(self, model, data, ** kwargs):
"""
.. todo::
WRITEME
"""
self.get_data_specs(model)[0].validate(data)
layer_costs = [layer.get_weight_decay(coeff)
for layer, coeff in safe_izip(model.layers,
self.coeffs)]
assert T.scalar() != 0. # make sure theano semantics do what I want
layer_costs = [cost for cost in layer_costs if cost != 0.]
if len(layer_costs) == 0:
rval = T.as_tensor_variable(0.)
rval.name = '0_weight_decay'
return rval
else:
total_cost = reduce(lambda x, y: x + y, layer_costs)
total_cost.name = 'RNN_WeightDecay'
assert total_cost.ndim == 0
total_cost.name = 'weight_decay'
return total_cost
示例9: add_dataset
def add_dataset(self, dataset, mode="sequential", batch_size=None, num_batches=None, seed=None):
"""
Determines the data used to calculate the values of each channel.
Parameters
----------
dataset : object
A `pylearn2.datasets.Dataset` object.
mode : str or object, optional
Iteration mode; see the docstring of the `iterator` method
on `pylearn2.datasets.Dataset` for details.
batch_size : int, optional
The size of an individual batch. Optional if `mode` is
'sequential' and `num_batches` is specified (batch size
will be calculated based on full dataset size).
num_batches : int, optional
The total number of batches. Unnecessary if `mode` is
'sequential' and `batch_size` is specified (number of
batches will be calculated based on full dataset size).
"""
# The user can ommit using lists if only one dataset is set
if not isinstance(dataset, list):
dataset = [dataset]
if not isinstance(mode, list):
mode = [mode]
if not isinstance(batch_size, list):
batch_size = [batch_size]
if not isinstance(num_batches, list):
num_batches = [num_batches]
if seed is None:
seed = [None] * len(dataset)
if not isinstance(seed, list):
seed = [seed]
if any([len(l) != len(dataset) for l in [mode, batch_size, seed]]):
raise ValueError("make sure each dataset has its iteration " + "mode, batch size and number of batches.")
for (d, m, b, n, sd) in safe_izip(dataset, mode, batch_size, num_batches, seed):
try:
it = d.iterator(mode=m, batch_size=b, num_batches=n, topo=self.topo, targets=self.require_label, rng=sd)
except ValueError as exc:
raise ValueError("invalid iteration parameters in " "Monitor.add_dataset: " + str(exc))
if it.stochastic:
# must be a seed, not a random number generator
# if it were a random number generator, different iterators using
# it would update its state, so we would not get the same iterator
# each time
# Also, must not be None, because this makes the iterator pick
# a seed based on the clock
if not isinstance(sd, (list, tuple, int)):
raise TypeError(
"Monitor requires a seed (not a random number generator) when using stochastic iteration modes."
)
else:
assert sd is None # the iterator should catch this, but let's double-check
if not d in self._datasets:
self._datasets.append(d)
self._iteration_mode.append(m)
self._batch_size.append(b)
self._num_batches.append(n)
self._rng_seed.append(sd)
示例10: next
def next(self):
"""
Retrieves the next batch of examples.
Returns
-------
next_batch : object
An object representing a mini-batch of data, conforming
to the space specified in the `data_specs` constructor
argument to this iterator. Will be a tuple if more
than one data source was specified or if the constructor
parameter `return_tuple` was `True`.
Raises
------
StopIteration
When there are no more batches to return.
"""
next_index = self._subset_iterator.next()
# TODO: handle fancy-index copies by allocating a buffer and
# using np.take()
rval = tuple(
fn(data[next_index]) if fn else data[next_index]
for data, fn in safe_izip(self._raw_data, self._convert))
if not self._return_tuple and len(rval) == 1:
rval, = rval
return rval
示例11: getFeatures
def getFeatures(indexes):
"""
.. todo::
Write me
"""
if self._load_to_memory:
sequences = [self.samples_sequences[i] for i in indexes]
else:
sequences = [self.node[i] for i in indexes]
# Get random start point for ngram
# Get random source word index for "ngram"
source_i = [numpy.random.randint(self.frame_length/2 +1, len(s)-self.frame_length/2, 1)[0]
for s in sequences]
target_i = [min(abs(int(numpy.random.normal(s_i, self.frame_length/3.0))), len(s)-1)
for s_i, s in safe_izip(source_i, sequences)]
preX = [s[i] for i, s in safe_izip(source_i, sequences)]
X = []
def make_sequence(word):
string = self._inv_words[word]
#if len(string) < 1:
#print "Word index", word, "Returns empty word"
seq = map(lambda c: [self._char_labels.get(c, 0)], self._inv_words[word])
#if len(seq) < 1:
# print "Word index", word, "Returns empty sequence", string
seq.append([self._eow])
return numpy.asarray(seq)
for word in preX:
X.append(make_sequence(word))
X = numpy.asarray(X)
y = [numpy.asarray([s[i]]) for i, s in safe_izip(target_i, sequences)]
#y[y>=30000] = numpy.asarray([1])
y = numpy.asarray(y)
bad_is = numpy.where(y >= 30000)
y[bad_is] = numpy.asarray([1])
# Target Words mapped to integers greater than input max are set to
# 1 (unknown)
# Store the targets generated by these indices.
self.lastY = (y, indexes)
if self._use_words:
self.lastPreX = preX
return X
示例12: next
def next(self):
"""
Retrieves the next batch of examples.
Returns
-------
next_batch : object
An object representing a mini-batch of data, conforming
to the space specified in the `data_specs` constructor
argument to this iterator. Will be a tuple if more
than one data source was specified or if the constructor
parameter `return_tuple` was `True`.
Raises
------
StopIteration
When there are no more batches to return.
"""
next_index = self._subset_iterator.next()
next_index = self._dataset.support[ next_index ] # !!! added line to iterate over different index set !!!
spaces, sources = self._data_specs
output = []
for data, fn, source in safe_izip(self._raw_data, self._convert, sources):
if source=='targets':
if fn:
output.append( fn(data[next_index, :]) )
else:
output.append( data[next_index, :] )
else:
design_mat = []
for index in next_index:
X = np.abs(data[index:index+self._dataset.tframes, :])
design_mat.append( X.reshape((np.prod(X.shape),)) )
design_mat = np.vstack(design_mat)
if self._dataset.tframes > 1:
# ideally we'd standardize in a preprocessing layer
# (so that standardization is built-in to the model rather
# than the dataset) but i haven't quite figured out how to do
# this yet for images, due to a memory error associated with
# a really big diagonal scaling matrix
# (however, it works fine for vectors)
design_mat = self._dataset.standardize(design_mat)
if fn:
output.append( fn(design_mat) )
else:
output.append( design_mat )
rval = tuple(output)
if not self._return_tuple and len(rval) == 1:
rval, = rval
return rval
示例13: next
def next(self):
next_index = self._subset_iterator.next()
rval = tuple(
fn(batch) if fn else batch for batch, fn in
safe_izip(self._dataset.get(self._source, next_index),
self._convert)
)
if not self._return_tuple and len(rval) == 1:
rval, = rval
return rval
示例14: get_objs
def get_objs():
n = 0.
aves = [0. for model in models]
m = 0
for X, Y in train.iterator(batch_size = 5000, mode='sequential', targets=True):
objs = [func(X, Y) for func in funcs]
n += 1.
aves = [ave + (obj - ave) / n for ave, obj in safe_izip(aves, objs)]
m += X.shape[0]
if m != 10000:
raise AssertionError(str(m))
return aves
示例15: expr
def expr(self, model, data, ** kwargs):
"""Returns a theano expression for the cost function.
Parameters
----------
model : MLP
data : tuple
Should be a valid occupant of
CompositeSpace(model.get_input_space(),
model.get_output_space())
Returns
-------
total_cost : theano.gof.Variable
coeff * sum(abs(weights))
added up for each set of weights.
"""
assert T.scalar() != 0. # make sure theano semantics do what I want
self.get_data_specs(model)[0].validate(data)
if isinstance(self.coeffs, list):
warnings.warn("Coefficients should be given as a dictionary "
"with layer names as key. The support of "
"coefficients as list would be deprecated "
"from 03/06/2015")
layer_costs = [layer.get_l1_weight_decay(coeff)
for layer, coeff in safe_izip(model.layers,
self.coeffs)]
layer_costs = [cost for cost in layer_costs if cost != 0.]
else:
layer_costs = []
for layer in model.layers:
layer_name = layer.layer_name
if layer_name in self.coeffs:
cost = layer.get_l1_weight_decay(self.coeffs[layer_name])
if cost != 0.:
layer_costs.append(cost)
if len(layer_costs) == 0:
rval = T.as_tensor_variable(0.)
rval.name = '0_l1_penalty'
return rval
else:
total_cost = reduce(operator.add, layer_costs)
total_cost.name = 'MLP_L1Penalty'
assert total_cost.ndim == 0
total_cost.name = 'l1_penalty'
return total_cost