Python RandomStreams.permutation方法代码示例

# 需要导入模块: from theano.tensor.shared_randomstreams import RandomStreams [as 别名]
# 或者: from theano.tensor.shared_randomstreams.RandomStreams import permutation [as 别名]
def build_model(tparams, options):
    trng = RandomStreams(1234)
    use_noise = theano.shared(numpy.float32(0.))

    # description string: #words x #samples
    if options['use_target_as_input']:
        x = tensor.tensor3('x', dtype='float32')
    else:
        x = tensor.matrix('x', dtype='int64')
    mask = tensor.matrix('mask', dtype='float32')
    # context: #samples x dim
    ctx = tensor.matrix('ctx', dtype='float32')

    n_timesteps = x.shape[0]
    n_samples = x.shape[1]

    # word embedding
    if options['use_target_as_input']:
        emb = x
    else:
        emb = tparams['Wemb'][x.flatten()].reshape([n_timesteps, n_samples, options['dim_word']])
    # decoder
    if options.setdefault('feedforward', False):
        proj_h = tensor.dot(emb, tparams['Wff'])
        proj_h = (proj_h * mask[:,:,None]).sum(axis=0)
        proj_h = proj_h / mask.sum(axis=0)[:,None]
    elif options.setdefault('regress', False):
        proj_h = (emb * mask[:,:,None]).sum(axis=0)
        proj_h = tensor.dot(proj_h, tparams['Wff'])
        proj_h = proj_h / mask.sum(axis=0)[:,None]
    else:
        proj = get_layer('lstm')[1](tparams, emb, options, 
                                    prefix='encoder', 
                                    mask=mask)
        proj_h = proj[0]
        if options['use_mean']:
            proj_h = (proj_h * mask[:,:,None]).sum(axis=0)
            proj_h = proj_h / mask.sum(axis=0)[:,None]
        else:
            proj_h = proj_h[-1]

    if 'n_layers' in options:
        for lidx in xrange(1, options['n_layers']):
            proj_h = get_layer('ff')[1](tparams, proj_h, options, prefix='ff_out_%d'%lidx, activ='tanh')
    out = get_layer('ff')[1](tparams, proj_h, options, prefix='ff_out', activ='linear')

    # cost
    if options['loss_type'] == 'cosine':
        out = out / tensor.sqrt((out ** 2).sum(1))[:,None]
        cost = 1. - (out * ctx).sum(1)
    elif options['loss_type'] == 'ranking':
        out = out / tensor.sqrt((out ** 2).sum(1))[:,None]
        rndidx = trng.permutation(n=ctx.shape[0])
        ctx_rnd = ctx[rndidx]
        cost = tensor.maximum(0., 1 - (out * ctx).sum(1) + (out * ctx_rnd).sum(1))
    else:
        raise Exception('Unknown loss function')

    return trng, use_noise, x, mask, ctx, cost

# 需要导入模块: from theano.tensor.shared_randomstreams import RandomStreams [as 别名]
# 或者: from theano.tensor.shared_randomstreams.RandomStreams import permutation [as 别名]
    def add_negative(cls, var_x, x_tilde, type='samples'):

        if type is None:
            return 0

        random_stream = RandomStreams()
        if type == 'samples':
            n = var_x.shape[0]
            perm = random_stream.permutation(n=n)
            shuffled_var_x = var_x[perm, :]
            return Tensor.mean(((shuffled_var_x - x_tilde) ** 2).sum(axis=1))

        if type == 'features':
            n = var_x.shape[1]
            perm = random_stream.permutation(n=n)
            shuffled_var_x = var_x[:, perm]
            return Tensor.mean(((shuffled_var_x - x_tilde) ** 2).sum(axis=1))

# 需要导入模块: from theano.tensor.shared_randomstreams import RandomStreams [as 别名]
# 或者: from theano.tensor.shared_randomstreams.RandomStreams import permutation [as 别名]
    def shuffle_training_data(self):

        print "Shuffling training X and y data..."

        numRows = self.train_set_x.shape[0]

        srng = RandomStreams(seed=None)
        mask = srng.permutation(n=numRows, size=(1,)).reshape((numRows,))

        self.train_set_x = self.train_set_x[mask]
        self.train_set_y = self.train_set_y[mask]

# 需要导入模块: from theano.tensor.shared_randomstreams import RandomStreams [as 别名]
# 或者: from theano.tensor.shared_randomstreams.RandomStreams import permutation [as 别名]
class Visual(task.Task):

    def __init__(self, config):
        autoassign(locals())
        self.updater = util.Adam(max_norm=config['max_norm'], lr=config['lr'])
        self.Encode = Encoder(config['size_vocab'],
                              config['size_embed'], config['size'],
                              config['depth'],
                              activation=eval(config.get('activation',
                                                         'clipped_rectify')),
                              filter_length=config.get('filter_length', 6), 
                              filter_size=config.get('filter_size', 1024), 
                              stride=config.get('stride', 3),
                              residual=config.get('residual',False))
        self.Attn   = Attention(config['size'])
        self.ToImg  = Dense(config['size'], config['size_target'])
        self.inputs = [T.ftensor3()]
        self.target = T.fmatrix()
        self.config['margin'] = self.config.get('margin', False)
        if self.config['margin']:
            self.srng = RandomStreams(seed=234)
        
        
    def params(self):
        return params(self.Encode, self.Attn, self.ToImg)
    
    def __call__(self, input):
        return self.ToImg(self.Attn(self.Encode(input)))
    
    def cost(self, target, prediction):
        if self.config['margin']:
            return self.Margin(target, prediction, dist=CosineDistance, d=1)
        else:
            return CosineDistance(target, prediction)
    
    def Margin(self, U, V, dist=CosineDistance, d=1.0):
        V_ = (V[self.srng.permutation(n=T.shape(V)[0],
                                      size=(1,)),]).reshape(T.shape(V))
        # A bit silly making it nondet
        return T.maximum(0.0, dist(U, V) - dist(U, V_) + d)
    
    def args(self, item):
        return (item['audio'], item['target_v'])

    def _make_representation(self):
        with context.context(training=False):
            rep = self.Encode(*self.inputs)
        return theano.function(self.inputs, rep)

    def _make_pile(self):
        with context.context(training=False):
            rep = self.Encode.GRU.intermediate(*self.inputs)
        return theano.function(self.inputs, rep)

# 需要导入模块: from theano.tensor.shared_randomstreams import RandomStreams [as 别名]
# 或者: from theano.tensor.shared_randomstreams.RandomStreams import permutation [as 别名]
    def test_permutation(self):
        """Test that RandomStreams.permutation generates the same results as numpy"""
        # Check over two calls to see if the random state is correctly updated.
        random = RandomStreams(utt.fetch_seed())
        fn = function([], random.permutation((20,), 10), updates=random.updates())

        fn_val0 = fn()
        fn_val1 = fn()

        rng_seed = numpy.random.RandomState(utt.fetch_seed()).randint(2**30)
        rng = numpy.random.RandomState(int(rng_seed))  # int() is for 32bit

        # rng.permutation outputs one vector at a time, so we iterate.
        numpy_val0 = numpy.asarray([rng.permutation(10) for i in range(20)])
        numpy_val1 = numpy.asarray([rng.permutation(10) for i in range(20)])

        assert numpy.all(fn_val0 == numpy_val0)
        assert numpy.all(fn_val1 == numpy_val1)

# 需要导入模块: from theano.tensor.shared_randomstreams import RandomStreams [as 别名]
# 或者: from theano.tensor.shared_randomstreams.RandomStreams import permutation [as 别名]
    def train(self, X_train, y_train,
          X_valid, y_valid,
          n_epochs, batch_size,
          optimization_function,
          cost_function,
          random_order=True):

        unsupervised = (X_train is y_train)

        if not isinstance(X_train, (TensorVariable, SharedVariable)):
            N = X_train.shape[0]
        else:
            N = function([], X_train.shape[0])()

        n_batches = N // batch_size + (N % batch_size != 0)

        if not isinstance(X_train, (TensorVariable, SharedVariable)):
            X_train = shared(X_train.astype('float32'), name="X_train")

        if not isinstance(X_valid, (TensorVariable, SharedVariable)):
            X_valid = shared(X_valid.astype('float32'), name="X_valid")

        if not unsupervised and not isinstance(y_train, (TensorVariable, SharedVariable)):
            if self.classification:
                y_train = shared(y_train.astype('int32'), name="y_train")
            else:
                y_train = shared(y_train.astype('float32'), name="y_train")

        if not unsupervised and not isinstance(y_valid, (TensorVariable, SharedVariable)):
            if self.classification:
                y_valid = shared(y_valid.astype('int32'), name="y_valid")
            else:
                y_valid = shared(y_valid.astype('float32'), name="y_valid")

        if random_order:
            perm_rng = RandomStreams(1)
            perm = perm_rng.permutation(n=N)
            if unsupervised:
                self.manual_updates.append(function([], updates=[(X_train, X_train[perm])]))
            else:
                self.manual_updates.append(function([], updates=[(X_train, X_train[perm]),
                                                                 (y_train, y_train[perm])]))

        if unsupervised:
            y_train = X_train
            y_valid = X_valid

        cost = cost_function(self.yScaled, self.out, self.params)
        error = self.error()

        validate = function([], [cost, error],
                            givens=[(self.X, X_valid), (self.y, y_valid)]
                                   + self.turn_off_dropout,
                            no_default_updates=self.no_default_upd)

        index = T.iscalar()
        upd = optimization_function(self.params, cost)

        batch_begin = index * batch_size
        batch_end   = T.min(((index+1) * batch_size, N))

        optimize = function([index], [cost, error],
                            givens=[(self.X, X_train[batch_begin:batch_end]),
                                    (self.y, y_train[batch_begin:batch_end])],
                            updates=upd,
                            no_default_updates=self.no_default_upd)

        for epoch in range(n_epochs):
            print("Epoch", epoch)
            cost_sum, error_sum = 0, 0
            print("Running batches...")
            for i in range(n_batches):
                c, a = optimize(i)
                cost_sum += c
                error_sum += a

            print("Done!")
            print("training: cost", cost_sum / float(n_batches), ", error", error_sum / float(n_batches))
            c, a = validate()
            print("validation: cost", c, ", error", a)

            for man_upd in self.manual_updates:
                man_upd()

# 需要导入模块: from theano.tensor.shared_randomstreams import RandomStreams [as 别名]
# 或者: from theano.tensor.shared_randomstreams.RandomStreams import permutation [as 别名]
    def __init__(self, rng, input, n_in, n_batch, d_bucket, activation, activation_deriv,
                 w=None, index_permute=None, index_permute_reverse=None):
        srng = RandomStreams(seed=234)
        
        n_bucket = n_in / d_bucket + 1
        self.input = input

        # randomly permute input space
        if index_permute is None:
            index_permute = srng.permutation(n=n_in)#numpy.random.permutation(n_in)
            index_permute_reverse = T.argsort(index_permute)
            self.index_permute = index_permute
            self.index_permute_reverse = index_permute_reverse

        permuted_input = input[:, index_permute]
        self.permuted_input = permuted_input

        # initialize matrix parameters
        if w is None:
            bound = numpy.sqrt(3. / d_bucket)
            w_values = numpy.asarray(rng.uniform(low=-bound,
                                                 high=bound,
                                                 size=(n_bucket, d_bucket, d_bucket)),
                                     dtype=theano.config.floatX)
            w = theano.shared(value=w_values, name='w')
            
        self.w = w
        
        
        # compute outputs and Jacobians
        
        log_jacobian = T.alloc(0, n_batch)
        for b in xrange(n_bucket):
            bucket_size = d_bucket
            if b == n_bucket - 1:
                bucket_size = n_in - b * d_bucket
            
           
            if b>0:
                prev_input = x_b
                
                """here we warp the previous bucket of inputs and add to the new input"""            

            x_b = self.permuted_input[:, b*d_bucket:b*d_bucket + bucket_size]
            w_b = self.w[b, :bucket_size, :bucket_size]

            if b>0:
                x_b_plus = x_b + m_b
            else:
                x_b_plus = x_b

            Upper = T.triu(w_b)
            Lower = T.tril(w_b)
            Lower = T.extra_ops.fill_diagonal(Lower, 1.)
            log_det_Upper = T.log(T.abs_(T.nlinalg.ExtractDiag()(Upper))).sum() 

            W = T.dot(Upper, Lower)
            log_jacobian = log_jacobian + T.alloc(log_det_Upper, n_batch)

            lin_output_b = T.dot(x_b_plus, W)
            if b>0:
                lin_output = T.concatenate([lin_output, lin_output_b], axis=1)
            else:
                lin_output = lin_output_b
            if activation is not None:
                derivs = activation_deriv(lin_output_b)     
                #import pdb; pdb.set_trace()
                log_jacobian = log_jacobian + T.log(T.abs_(derivs)).sum(axis=1)                 
                    
        self.log_jacobian = log_jacobian        


        self.output = (
            lin_output[:, index_permute_reverse] if activation is None
            else activation(lin_output[:, index_permute_reverse])
        )

        self.params = [w]

# 需要导入模块: from theano.tensor.shared_randomstreams import RandomStreams [as 别名]
# 或者: from theano.tensor.shared_randomstreams.RandomStreams import permutation [as 别名]
class Visual(task.Task):

    def __init__(self, config):
        autoassign(locals())
        self.updater = util.Adam(max_norm=config['max_norm'], lr=config['lr'])
        self.Encode = Encoder(config['size_vocab'],
                              config['size_embed'], config['size'],
                              config['depth'],
                              activation=eval(config.get('activation',
                                                         'clipped_rectify')),
                              residual=config.get('residual',False))
        self.ToImg  = Dense(config['size'], config['size_target'])
        self.inputs = [T.imatrix()]
        self.target = T.fmatrix()
        self.config['margin'] = self.config.get('margin', False)
        if self.config['margin']:
            self.srng = RandomStreams(seed=234)
        
        
    def params(self):
        return params(self.Encode, self.ToImg)
    
    def __call__(self, input):
        return self.ToImg(last(self.Encode(input)))
    
    def cost(self, target, prediction):
        if self.config['margin']:
            return self.Margin(target, prediction, dist=CosineDistance, d=1)
        elif self.config.get('contrastive'):
            return self.contrastive(target, prediction, margin=0.2)
        else:
            return CosineDistance(target, prediction)
            
    def contrastive(self, i, s, margin=0.2): 
        # i: (fixed) image embedding, 
        # s: sentence embedding
        errors = - util.cosine_matrix(i, s)
        diagonal = errors.diagonal()
        # compare every diagonal score to scores in its column (all contrastive images for each sentence)
        cost_s = T.maximum(0, margin - errors + diagonal)  
        # all contrastive sentences for each image
        cost_i = T.maximum(0, margin - errors + diagonal.reshape((-1, 1)))  
        cost_tot = cost_s + cost_i
        # clear diagonals
        cost_tot = fill_diagonal(cost_tot, 0)

        return cost_tot.mean()
    
    def Margin(self, U, V, dist=CosineDistance, d=1.0):
        V_ = (V[self.srng.permutation(n=T.shape(V)[0],
                                      size=(1,)),]).reshape(T.shape(V))
        # A bit silly making it nondet
        return T.maximum(0.0, dist(U, V) - dist(U, V_) + d)
    
    def args(self, item):
        return (item['input'], item['target_v'])

    def _make_representation(self):
        with context.context(training=False):
            rep = self.Encode(*self.inputs)
        return theano.function(self.inputs, rep)

    def _make_pile(self):
        with context.context(training=False):
            rep = self.Encode.GRU.intermediate(self.Encode.Embed(*self.inputs))
        return theano.function(self.inputs, rep)

# 需要导入模块: from theano.tensor.shared_randomstreams import RandomStreams [as 别名]
# 或者: from theano.tensor.shared_randomstreams.RandomStreams import permutation [as 别名]

#.........这里部分代码省略.........
                    scale = scale
                )
    cost = (
            classifier.negative_log_likelihood(Y) 
            + L2_reg * classifier.L2_sqr # using the L2 regularization
        )

    gparams = [T.grad(cost, param) for param in classifier.params]

    
    # Random number generator for the gaussian noise
    # theano_rng = RandomStreams(int(np.random.rand()*100))
    train_model = theano.function(
                 inputs = [index, eta, noise],
                 outputs = cost,
                 updates = [(param, param - eta * gparam) 
                    for param, gparam in zip(classifier.params, gparams)],
                 givens = {
                         # train_dims[1] is the number of columns (features) in the training data
                         # apparently trainX gets first added to the random numbers before its sliced
                         # Hence we use 784 (features) random numbers and not 100 (batch_size) random numbers
                         # X : trainX[index * batch_size : (index + 1) * batch_size] + theano_rng.normal(size=(train_dims[1],))* n_scale,
                         X : trainX[index * batch_size : (index + 1) * batch_size] + noise,
                         Y : trainY[index * batch_size : (index + 1) * batch_size]
                     }
             )
    
    validate_model = theano.function(
                inputs = [index],
                outputs = classifier.errors(Y),
                givens = {
                         X : validX[index * batch_size : (index + 1) * batch_size],
                         Y : validY[index * batch_size : (index + 1) * batch_size]
                }
            )
    
    test_model = theano.function(
                inputs = [index],
                outputs = classifier.errors(Y),
                givens = {
                         X : testX[index * batch_size : (index + 1) * batch_size],
                         Y : testY[index * batch_size : (index + 1) * batch_size]
                }
            )


    train_error = []
    valid_error = []
    test_error  = []

    # Calculate the number of batches.
    n_train_batches = int(train_size / batch_size)
    n_val_batches = int(valid_size / batch_size)
    n_test_batches = int(test_size / batch_size)

    ANNEAL = 10*train_size # rate at which learning parameter "eta" is reduced as iterations increase ( momentum )
    print("Anneal = {}".format(ANNEAL))
    
    start_time = timeit.default_timer()
    learn_rate = etaVal

    # Initial Gaussian Noise
    gaussian_noise = 0
    for epoch in xrange(epochs):
        # shuffle data, reset the seed so that trainX and trainY are randomized
        # the same way
        theano_seed = int(np.random.rand()*100)
        theano_rng = RandomStreams(theano_seed)
        trainX = trainX[theano_rng.permutation(n=train_size, size=(1,)),]
        theano_rng = RandomStreams(theano_seed)
        trainY = trainY[theano_rng.permutation(n=train_size, size=(1,)),]
        
        cost = []
        val_cost = []
        

        # Add new gaussian noise
        # of size (batch_size, # of features)
        gaussian_noise = noise_scale * np.random.normal(size=(batch_size,train_dims[1])).astype(theano.config.floatX)

        for batch_idx in xrange(n_train_batches):
            cost.append(np.mean(np.asarray([train_model(batch_idx, learn_rate, gaussian_noise)])))

        # Delete the gaussian noise
        # trainX = trainX - gaussian_noise

        # Validation error checked in each epoch
        for val_batch_idx in xrange(n_val_batches): 
            val_cost.append(np.mean(np.asarray([validate_model(val_batch_idx)])))

        train_error.append(np.mean(cost))
        valid_error.append(np.mean(val_cost))

        time_check = timeit.default_timer()
        iteration = (epoch * batch_idx) + batch_idx
        print("epoch={}, mean train cost={}, mean_val_cost = {} time = {} eta={}".format(epoch, train_error[-1], valid_error[-1], (time_check - start_time)/60.0, learn_rate))
        # Search and then converge
        learn_rate = etaVal / ( 1.0 + (iteration*1.0 / ANNEAL))

    return train_error, valid_error

# 需要导入模块: from theano.tensor.shared_randomstreams import RandomStreams [as 别名]
# 或者: from theano.tensor.shared_randomstreams.RandomStreams import permutation [as 别名]
import theano as th

data = np.random.rand(10,3)




it = th.shared(0)
y = th.shared(data)




srng = RandomStreams(seed=234)

expectRvs   = srng.normal(size=(3,1))
expectRvs.name='expectRvs'
epochStream = srng.permutation(n=10)
currentBatch = epochStream.reshape((5,2))[:,it]
y_mini = y[ currentBatch, :]
L = th.tensor.sum(th.tensor.dot( y_mini, expectRvs ))
L_func = function([], L, no_default_updates=True)

padding = srng.choice(size=(3,), a=10, replace=False, p=None, ndim=None, dtype='int64')



f1 = function([], expectRvs, no_default_updates=True)
f2 = function([], expectRvs)

# 需要导入模块: from theano.tensor.shared_randomstreams import RandomStreams [as 别名]
# 或者: from theano.tensor.shared_randomstreams.RandomStreams import permutation [as 别名]
    def __init__(self, rng, input, n_in, n_batch, d_bucket, n_reflections, activation, activation_deriv,
                 w=None, index_permute=None, index_permute_reverse=None):
        srng = RandomStreams(seed=234)
        
        n_bucket = n_in / d_bucket + 1
        self.input = input

        # randomly permute input space
        if index_permute is None:
            index_permute = srng.permutation(n=n_in)#numpy.random.permutation(n_in)
            index_permute_reverse = T.argsort(index_permute)
            self.index_permute = index_permute
            self.index_permute_reverse = index_permute_reverse

        permuted_input = input[:, index_permute]
        self.permuted_input = permuted_input

        # initialize reflection parameters
        if w is None:
            w_values = numpy.asarray(rng.uniform(low=-1,
                                                 high=1,
                                                 size=(n_bucket, n_reflections, d_bucket)),
                                     dtype=theano.config.floatX)
            w = theano.shared(value=w_values, name='w')
            
        self.w = w
        
        
        # compute outputs and Jacobians
        
        log_jacobian = T.alloc(0, n_batch)
        for b in xrange(n_bucket):
            bucket_size = d_bucket
            if b == n_bucket - 1:
                #import pdb; pdb.set_trace()
                bucket_size = n_in - b * d_bucket
            
            x_b = self.permuted_input[:, b*d_bucket:b*d_bucket + bucket_size]
            for r in xrange(n_reflections):
                w_b_r = w[b, r, :bucket_size]
                if r>0:
                    Wtemp = T.eye(bucket_size) \
                        - 2 * T.outer(w_b_r, w_b_r) / ((w_b_r ** 2).sum())
                    W = T.dot(W, Wtemp)
#                    import pdb; pdb.set_trace()
                else:
                    W = T.eye(bucket_size) - 2 * T.outer(w_b_r, w_b_r) / ((w_b_r ** 2).sum())

            lin_output_b = T.dot(x_b, W)
            if b>0:
                lin_output = T.concatenate([lin_output, lin_output_b], axis=1)
            else:
                lin_output = lin_output_b
            if activation is not None:
                derivs = activation_deriv(lin_output_b)
                log_jacobian = log_jacobian + T.log(T.abs_(derivs)).sum(axis=1)
 
#                for n in xrange(n_batch):
#                    mat = T.tile(T.reshape(derivs[n], [1, bucket_size]), (bucket_size, 1))
#                    mat = mat * W
                    
#                    T.inc_subtensor(log_jacobian[n], T.log(T.abs_(T.nlinalg.Det()(mat))))
                    
        self.log_jacobian = log_jacobian        

        self.output = (
            lin_output if activation is None
            else activation(lin_output)
        )

        self.params = [w]

# 需要导入模块: from theano.tensor.shared_randomstreams import RandomStreams [as 别名]
# 或者: from theano.tensor.shared_randomstreams.RandomStreams import permutation [as 别名]
class UNET(object):

    def __init__(
        self,
        id,
        rng,
        batch_size, 
        patch_size=572, 
        patch_size_out=388,
        offline=False,
        path=None,
        train_time=5.0,
        learning_rate=0.01,
        momentum=0.95):
        self.id = id
        self.type = 'UNET'
        self.offline = offline
        self.done = False
        self.path = path    
        self.batchSize = batch_size
        self.patchSize = patch_size 
        self.patchSize_out = patch_size_out
        self.learning_rate = learning_rate
        self.momentum      = momentum

        self.best_validation_loss = numpy.inf
        self.trainTime = train_time
        self.resample = False       
        self.error = np.inf
        self.error_threshold = 0.06
        self.best_val_loss_so_far = 0
        self.patience_counter = 0
        self.patience = 100
        self.patience_reset = 100

        self.doBatchNormAll = False
        self.doFineTune = False

        self.weight_decay = 0.
        self.weight_class_1 = 1.
        self.initialization = 'glorot_uniform'

        self.model = None

        self.srng = RandomStreams(1234)

        self.initialize() 


    def initialize(self):
        print 'Unet.initialize'


    def trainiold(self, offline=False, data=None, mean=None, std=None):
        print 'UNET.train()'

        # need to define a custom loss, because all pre-implementations
        # seem to assume that scores over patch add up to one which
        # they clearly don't and shouldn't
        def unet_crossentropy_loss(y_true, y_pred):
            weight_class_1 = 1.
            epsilon = 1.0e-4
            y_pred_clipped = T.clip(y_pred, epsilon, 1.0-epsilon)
            loss_vector = -T.mean(weight_class_1*y_true * T.log(y_pred_clipped) + (1-y_true) * T.log(1-y_pred_clipped), axis=1)
            average_loss = T.mean(loss_vector)
            return average_loss

        def unet_crossentropy_loss_sampled(y_true, y_pred):
            print 'unet_crossentropy_loss_sampled'
            epsilon = 1.0e-4
            y_pred_clipped = T.flatten(T.clip(y_pred, epsilon, 1.0-epsilon))
            y_true = T.flatten(y_true)
            # this seems to work
            # it is super ugly though and I am sure there is a better way to do it
            # but I am struggling with theano to cooperate
            # filter the right indices
            indPos = T.nonzero(y_true)[0] # no idea why this is a tuple
            indNeg = T.nonzero(1-y_true)[0]
            # shuffle
            n = indPos.shape[0]
            indPos = indPos[self.srng.permutation(n=n)]
            n = indNeg.shape[0]
            indNeg = indNeg[self.srng.permutation(n=n)]
            # take equal number of samples depending on which class has less
            n_samples = T.cast(T.min([T.sum(y_true), T.sum(1-y_true)]), dtype='int64')

            indPos = indPos[:n_samples]
            indNeg = indNeg[:n_samples]
            loss_vector = -T.mean(T.log(y_pred_clipped[indPos])) - T.mean(T.log(1-y_pred_clipped[indNeg]))
            average_loss = T.mean(loss_vector)
            #return average_loss
            return T.mean(T.log(y_pred_clipped[indPos]))


        # input data should be large patches as prediction is also over large patches
        print
        print "=== building network ==="

        print "== BLOCK 1 =="
        input = Input(shape=(1, self.patchSize, self.patchSize))
#.........这里部分代码省略.........

# 需要导入模块: from theano.tensor.shared_randomstreams import RandomStreams [as 别名]
# 或者: from theano.tensor.shared_randomstreams.RandomStreams import permutation [as 别名]
    def __init__(self, rng, input, n_in, n_batch, d_bucket, activation, activation_deriv,
                 w=None, index_permute=None, index_permute_reverse=None):
        srng = RandomStreams(seed=234)
        
        n_bucket = n_in / d_bucket + 1
        self.input = input

        # randomly permute input space
        if index_permute is None:
            index_permute = srng.permutation(n=n_in)#numpy.random.permutation(n_in)
            index_permute_reverse = T.argsort(index_permute)
            self.index_permute = index_permute
            self.index_permute_reverse = index_permute_reverse

        permuted_input = input[:, index_permute]
        self.permuted_input = permuted_input

        # initialize reflection parameters
        if w is None:
            bound = numpy.sqrt(3. / d_bucket)
            w_values = numpy.asarray(rng.uniform(low=-bound,
                                                 high=bound,
                                                 size=(n_bucket, d_bucket, d_bucket)),
                                     dtype=theano.config.floatX)
            w = theano.shared(value=w_values, name='w')
            
        self.w = w
        
        
        # compute outputs and Jacobians
        
        log_jacobian = T.alloc(0, n_batch)
        for b in xrange(n_bucket):
            bucket_size = d_bucket
            if b == n_bucket - 1:
                bucket_size = n_in - b * d_bucket
            
            x_b = self.permuted_input[:, b*d_bucket:b*d_bucket + bucket_size]

            
            w_b = self.w[b, :bucket_size, :bucket_size]

#            W = T.slinalg.Expm()(w_b)
#            log_jacobian = log_jacobian + T.alloc(T.nlinalg.trace(w_b), n_batch)

            Upper = T.triu(w_b)
#            Upper = T.extra_ops.fill_diagonal(Upper, 1.)
            Lower = T.tril(w_b)
            Lower = T.extra_ops.fill_diagonal(Lower, 1.)
            log_det_Upper = T.log(T.abs_(T.nlinalg.ExtractDiag()(Upper))).sum() 
#            log_det_Lower = T.log(T.abs_(T.nlinalg.ExtractDiag()(Lower))).sum() 


            W = T.dot(Upper, Lower)
            log_jacobian = log_jacobian + T.alloc(log_det_Upper, n_batch)

            
#            W = T.dot(T.transpose(w_b), w_b) + 0.001*T.eye(bucket_size)
#            log_jacobian = log_jacobian + T.alloc(T.log(T.abs_(T.nlinalg.Det()(W))), n_batch)

#            diag = T.nlinalg.diag(W)
#            div = T.tile(T.reshape(T.sqrt(diag), [1, bucket_size]), (bucket_size, 1))
            
#            W = W / div / T.transpose(div)
            #import pdb; pdb.set_trace()

            lin_output_b = T.dot(x_b, W)
            if b>0:
                lin_output = T.concatenate([lin_output, lin_output_b], axis=1)
            else:
                lin_output = lin_output_b
            if activation is not None:
                derivs = activation_deriv(lin_output_b)     
                #import pdb; pdb.set_trace()
                log_jacobian = log_jacobian + T.log(T.abs_(derivs)).sum(axis=1)

                
#                for n in xrange(n_batch):                    
#                    mat = T.tile(T.reshape(derivs[n], [1, bucket_size]), (bucket_size, 1))
#                    mat = mat * W                   
#                    T.inc_subtensor(log_jacobian[n], T.log(T.abs_(T.nlinalg.Det()(mat))))
                    
        self.log_jacobian = log_jacobian        

        self.output = (
            lin_output if activation is None
            else activation(lin_output)
        )


        self.params = [w]

# 需要导入模块: from theano.tensor.shared_randomstreams import RandomStreams [as 别名]
# 或者: from theano.tensor.shared_randomstreams.RandomStreams import permutation [as 别名]
    def __init__(self, rng, input, n_in, n_batch, d_bucket, activation, activation_deriv,
                 w=None, index_permute=None, index_permute_reverse=None):
        srng = RandomStreams(seed=234)
        
        n_bucket = n_in / d_bucket + 1
        self.input = input

        # randomly permute input space
        if index_permute is None:
            index_permute = srng.permutation(n=n_in)#numpy.random.permutation(n_in)
            index_permute_reverse = T.argsort(index_permute)
            self.index_permute = index_permute
            self.index_permute_reverse = index_permute_reverse

        permuted_input = input[:, index_permute]
        self.permuted_input = permuted_input

        # initialize reflection parameters
        if w is None:
            bound = numpy.sqrt(3. / d_bucket)
            w_values = numpy.asarray(rng.uniform(low=-bound,
                                                 high=bound,
                                                 size=(n_bucket, d_bucket, d_bucket)),
                                     dtype=theano.config.floatX)
            w = theano.shared(value=w_values, name='w')
            
        self.w = w
        
        
        # compute outputs and Jacobians
        
        log_jacobian = T.alloc(0, n_batch)
        for b in xrange(n_bucket):
            bucket_size = d_bucket
            if b == n_bucket - 1:                
                bucket_size = n_in - b * d_bucket
            
            x_b = self.permuted_input[:, b*d_bucket:b*d_bucket + bucket_size]

            
            w_b = w[b, :bucket_size, :bucket_size]
            wTwinv = T.nlinalg.MatrixInverse()(T.dot(T.transpose(w_b), w_b) + 0.001*T.eye(bucket_size))
            L = T.slinalg.Cholesky()(wTwinv) 

            W = T.dot(w_b, L)

            #import pdb; pdb.set_trace()

            lin_output_b = T.dot(x_b, W)
            if b>0:
                lin_output = T.concatenate([lin_output, lin_output_b], axis=1)
            else:
                lin_output = lin_output_b
            if activation is not None:
                derivs = activation_deriv(x_b)             
                for n in xrange(n_batch):                    
                    mat = T.tile(T.reshape(derivs[n], [1, bucket_size]), (bucket_size, 1))
                    mat = mat * W                   
                    T.inc_subtensor(log_jacobian[n], T.log(T.abs_(T.nlinalg.Det()(mat))))
                    
        self.log_jacobian = log_jacobian        

        self.output = (
            lin_output if activation is None
            else activation(lin_output)
        )

        self.params = [w]

# 需要导入模块: from theano.tensor.shared_randomstreams import RandomStreams [as 别名]
# 或者: from theano.tensor.shared_randomstreams.RandomStreams import permutation [as 别名]
def do_gd(train_set, etaVal, epochs, layers, batch_size=100, scale=1):
    '''
    batch_size = 100
    '''
    SEED = 5318
    np.random.seed(SEED)
    X = T.matrix('X')
    Y = T.ivector('Y')
    index = T.lscalar('index')
    eta = T.fscalar('eta')
    
    n_in = layers[0]
    n_out = layers[-1]

    trainX, trainY = train_set
    dataset_size = trainX.get_value(borrow=True).shape[0]


    classifier = MLP(
                    rng = np.random.RandomState(SEED),
                    inpt = X,
                    layers = layers,
                    scale = scale
                )
    cost = classifier.negative_log_likelihood(Y)

    gparams = [T.grad(cost, param) for param in classifier.params]

    train_model = theano.function(
                 inputs = [index, eta],
                 outputs = cost,
                 updates = [(param, param - eta * gparam) 
                    for param, gparam in zip(classifier.params, gparams)],
                 givens = {
                         X : trainX[index * batch_size : (index + 1) * batch_size],
                         Y : trainY[index * batch_size : (index + 1) * batch_size]
                     }
             )

    # train_model = theano.function(
    #              inputs = [index, eta],
    #              outputs = cost,
    #              updates = [(param, param - eta * gparam) 
    #                  for param, gparam in zip(classifier.params, gparams)],
    #              givens = {
    #                      X : trainX[index],
    #                      Y : trainY[index]
    #                  }
    #          )
    
    # pydotprint(train_model,'./test.png')
    # d3v.d3viz(train_model,'./test.html')
    cost = []
    n_batches = int(dataset_size / batch_size)
    print dataset_size
    ANNEAL = 10*dataset_size # rate at which learning parameter "eta" is reduced as iterations increase ( momentum )
    print("Anneal = {}".format(ANNEAL))
    
    start_time = timeit.default_timer()
    learn_rate = etaVal
    for epoch in xrange(epochs):
        # shuffle data, reset the seed so that trainX and trainY are randomized
        # the same way
        theano_seed = int(np.random.rand()*100)
        theano_rng = RandomStreams(theano_seed)
        trainX = trainX[theano_rng.permutation(n=dataset_size, size=(1,)),]
        theano_rng = RandomStreams(theano_seed)
        trainY = trainY[theano_rng.permutation(n=dataset_size, size=(1,)),]

        for batch_idx in xrange(n_batches):
            cost.append(np.mean(np.asarray([train_model(batch_idx, learn_rate)])))

        time_check = timeit.default_timer()
        iteration = (epoch * batch_idx) + batch_idx
        print("epoch={}, mean cost={}, total_time(mins)={}, eta={}, iters={}".format(epoch, np.mean(cost[-n_batches:]), (time_check - start_time)/60.0, learn_rate, iteration))
        # Search and then converge
        learn_rate = etaVal / ( 1.0 + (iteration*1.0 / ANNEAL))

    print("Eta = {}, Cost Last= {} Mean last 10 Costs = {}".format(
            eta, cost[-1], np.mean(cost[-10:])) 
         )
    return np.mean(cost[-10:])