Python MLP.errors方法代碼示例

# 需要導入模塊: from mlp import MLP [as 別名]
# 或者: from mlp.MLP import errors [as 別名]
    rng = numpy.random.RandomState(1234)

    # construct the MLP class
    classifier = MLP(rng=rng, input=x, n_in=28 * 28, n_hidden=500, n_out=10)

    # load trained parameters
    params = numpy.load("mlp_mnist.npz")
    classifier.hiddenLayer.W.set_value(params['hidden_W'])
    classifier.hiddenLayer.b.set_value(params['hidden_b'])
    classifier.logRegressionLayer.W.set_value(params['logreg_W'])
    classifier.logRegressionLayer.b.set_value(params['logreg_b'])

    # test model functions
    train_loss = theano.function(inputs=[index],
            outputs=classifier.errors(y),
            givens={
                x: train_set_x[index * batch_size:(index + 1) * batch_size],
                y: train_set_y[index * batch_size:(index + 1) * batch_size]})

    test_model = theano.function(inputs=[index],
            outputs=classifier.errors(y),
            givens={
                x: test_set_x[index * batch_size:(index + 1) * batch_size],
                y: test_set_y[index * batch_size:(index + 1) * batch_size]})

    validate_model = theano.function(inputs=[index],
            outputs=classifier.errors(y),
            givens={
                x: valid_set_x[index * batch_size:(index + 1) * batch_size],
                y: valid_set_y[index * batch_size:(index + 1) * batch_size]})

# 需要導入模塊: from mlp import MLP [as 別名]
# 或者: from mlp.MLP import errors [as 別名]
def test_mlp(dataset, hyper):
    train_set_x, train_set_y = dataset.sharedTrain
    valid_set_x, valid_set_y = dataset.sharedValid
    test_set_x, test_set_y = dataset.sharedTest

    n_train_batches = train_set_x.get_value(borrow=True).shape[0] / hyper.batchSize
    n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] / hyper.batchSize
    n_test_batches = test_set_x.get_value(borrow=True).shape[0] / hyper.batchSize

    validationFrequency = min(n_train_batches, hyper.patience / 2)

    print '... building the model'

    # allocate symbolic variables for the data
    index = T.lscalar()  # index to a [mini]batch
    x = T.matrix('x')  # the data is presented as rasterized images
    y = T.ivector('y')  # the labels are presented as 1D vector of
                        # [int] labels

    rng = numpy.random.RandomState(1234)

    # construct the MLP class
    classifier = MLP(rng=rng, input=x, n_in=dataset.n_in,
                     n_hidden=hyper.nHidden1, n_out=dataset.n_out)

    # the cost we minimize during training is the negative log likelihood of
    # the model plus the regularization terms (L1 and L2); cost is expressed
    # here symbolically
    cost = classifier.negative_log_likelihood(y) \
         + hyper.L1Reg * classifier.L1 \
         + hyper.L2Reg * classifier.L2_sqr

    # compiling a Theano function that computes the mistakes that are made
    # by the model on a minibatch
    test_model = theano.function(inputs=[index],
            outputs=classifier.errors(y),
            givens={
                x: test_set_x[index * hyper.batchSize:(index + 1) * hyper.batchSize],
                y: test_set_y[index * hyper.batchSize:(index + 1) * hyper.batchSize]})

    validate_model = theano.function(inputs=[index],
            outputs=classifier.errors(y),
            givens={
                x: valid_set_x[index * hyper.batchSize:(index + 1) * hyper.batchSize],
                y: valid_set_y[index * hyper.batchSize:(index + 1) * hyper.batchSize]})

    # compute the gradient of cost with respect to theta (sotred in params)
    # the resulting gradients will be stored in a list gparams
    gparams = []
    for param in classifier.params:
        gparam = T.grad(cost, param)
        gparams.append(gparam)

    # specify how to update the parameters of the model as a list of
    # (variable, update expression) pairs
    updates = []
    # given two list the zip A = [a1, a2, a3, a4] and B = [b1, b2, b3, b4] of
    # same length, zip generates a list C of same size, where each element
    # is a pair formed from the two lists :
    #    C = [(a1, b1), (a2, b2), (a3, b3), (a4, b4)]
    for param, gparam in zip(classifier.params, gparams):
        updates.append((param, param - hyper.learningRate * gparam))

    # compiling a Theano function `train_model` that returns the cost, but
    # in the same time updates the parameter of the model based on the rules
    # defined in `updates`
    train_model = theano.function(inputs=[index], outputs=cost,
            updates=updates,
            givens={
                x: train_set_x[index * hyper.batchSize:(index + 1) * hyper.batchSize],
                y: train_set_y[index * hyper.batchSize:(index + 1) * hyper.batchSize]})

    ###############
    # TRAIN MODEL #
    ###############
    print '... training'

    best_params = None
    best_validation_loss = numpy.inf
    best_iter = 0
    test_score = 0.
    start_time = time.time()

    epoch = 0
    done_looping = False
    patience = hyper.patience

    while (epoch < hyper.numberEpochs) and (not done_looping):
        epoch = epoch + 1
        print('epoch %i, time %0.2fm' % (epoch, (time.clock() - start_time) / 60.0))
        for minibatch_index in xrange(n_train_batches):

            minibatch_avg_cost = train_model(minibatch_index)
            # iteration number
            iter = (epoch - 1) * n_train_batches + minibatch_index

            if (iter + 1) % validationFrequency == 0:
                # compute zero-one loss on validation set
                validation_losses = [validate_model(i) for i
                                     in xrange(n_valid_batches)]
#.........這裏部分代碼省略.........

# 需要導入模塊: from mlp import MLP [as 別名]
# 或者: from mlp.MLP import errors [as 別名]
        rng=rng,
        input=x,
        n_in=3000,
        n_hidden=n_hidden,
        n_out=2
    )

cost = (
        classifier.negative_log_likelihood(y)
        + L1_reg * classifier.L1
        + L2_reg * classifier.L2_sqr
    )

validate_model = theano.function(
        inputs=[x,y],
        outputs=classifier.errors(y)
    )

test_model = theano.function(
        inputs=[x],
        outputs=classifier.y_pred
    )

test_model_proba = theano.function(
        inputs=[x],
        outputs=classifier.p_y_given_x
    )

# start-snippet-5
# compute the gradient of cost with respect to theta (sotred in params)
# the resulting gradients will be stored in a list gparams

# 需要導入模塊: from mlp import MLP [as 別名]
# 或者: from mlp.MLP import errors [as 別名]
def sgd_optimization_mnist_mlp(learning_rate=0.01, L1_reg=0.0, L2_reg=0.0001,
                               n_epochs=1000, dataset='mnist.pkl.gz',
                               batch_size=20, n_hidden=500):
    datasets = load_data(dataset)

    train_set_x, train_set_y = datasets[0]
    valid_set_x, valid_set_y = datasets[1]
    test_set_x, test_set_y = datasets[2]

    # Notice that get_value is called with borrow
    # so that a deep copy of the input is not created
    n_train_batches = train_set_x.get_value(borrow=True).shape[0] // batch_size
    n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] // batch_size
    n_test_batches = test_set_x.get_value(borrow=True).shape[0] // batch_size

    print("... Building the model")

    index = T.lscalar()  # index to a mini-batch

    # Symbolic variables for input and output for a batch
    x = T.matrix('x')
    y = T.ivector('y')

    rng = numpy.random.RandomState(1234)
    # Build the logistic regression class
    # Images in MNIST are 28*28, there are 10 output classes
    classifier = MLP(
        rng=rng,
        input=x,
        n_in=28*28,
        n_hidden=n_hidden,
        n_out=10)

    # Cost to minimize
    cost = (
        classifier.loss(y)
        + L1_reg * classifier.L1
        + L2_reg * classifier.L2_sq
    )

    # Compile function that measures test performance wrt the 0-1 loss
    test_model = theano.function(
        inputs=[index],
        outputs=classifier.errors(y),
        givens=[
            (x, test_set_x[index * batch_size: (index + 1) * batch_size]),
            (y, test_set_y[index * batch_size: (index + 1) * batch_size])
        ]
    )
    validate_model = theano.function(
        inputs=[index],
        outputs=classifier.errors(y),
        givens=[
            (x, valid_set_x[index * batch_size: (index + 1) * batch_size]),
            (y, valid_set_y[index * batch_size: (index + 1) * batch_size])
        ]
    )

    # Stochastic Gradient descent
    updates = simple_sgd(cost, classifier.params, learning_rate)

    train_model = theano.function(
        inputs=[index],
        outputs=cost,
        updates=updates,
        givens=[
            (x, train_set_x[index * batch_size: (index + 1) * batch_size]),
            (y, train_set_y[index * batch_size: (index + 1) * batch_size])
        ]
    )

    ################
    # TRAIN MODEL  #
    ################
    print("... Training the model")
    # Early stopping parameters
    patience = 10000  # Look at these many parameters regardless
    # Increase patience by this quantity when a best score is achieved
    patience_increase = 2
    improvement_threshold = 0.995  # Minimum significant improvement
    validation_frequency = min(n_train_batches, patience // 2)
    best_validation_loss = numpy.inf
    test_score = 0.
    start_time = timeit.default_timer()

    done_looping = False
    epoch = 0
    while (epoch < n_epochs) and (not done_looping):
        epoch = epoch + 1
        for minibatch_index in range(n_train_batches):
            minibatch_avg_cost = train_model(minibatch_index)
            # Iteration number
            iter = (epoch - 1) * n_train_batches + minibatch_index
            # Check if validation needs to be performed
            if (iter + 1) % validation_frequency == 0:
                # Compute average 0-1 loss on validation set
                validation_losses = [validate_model(i)
                                     for i in range(n_valid_batches)]
                this_validation_loss = numpy.mean(validation_losses)

#.........這裏部分代碼省略.........

# 需要導入模塊: from mlp import MLP [as 別名]
# 或者: from mlp.MLP import errors [as 別名]
def do_gd(etaVal, epochs, layers, train_set, 
        valid_set=None, test_set=None, L2_reg=0, batch_size=100, scale=1, noise_scale=1):
    '''
    batch_size = 100
    0 L2 regularization (by default)
    function returns training error and validation error after each epoch
    '''
    SEED = 5318
    np.random.seed(SEED)
    X = T.matrix('X')
    Y = T.ivector('Y')
    index = T.lscalar('index')
    noise = T.matrix('noise')
    eta = T.fscalar('eta')
    n_scale = T.fscalar('noise_scale')
    
    n_in = layers[0]
    n_out = layers[-1]

    # Get the datasets
    trainX, trainY = train_set
    validX, validY = valid_set
    testX, testY   = test_set

    # Get the dataset sizes
    train_dims = trainX.get_value(borrow=True).shape
    train_size = trainX.get_value(borrow=True).shape[0]
    valid_size = validX.get_value(borrow=True).shape[0]
    test_size  = testX.get_value(borrow=True).shape[0]


    classifier = MLP(
                    rng = np.random.RandomState(SEED),
                    inpt = X,
                    layers = layers,
                    scale = scale
                )
    cost = (
            classifier.negative_log_likelihood(Y) 
            + L2_reg * classifier.L2_sqr # using the L2 regularization
        )

    gparams = [T.grad(cost, param) for param in classifier.params]

    
    # Random number generator for the gaussian noise
    # theano_rng = RandomStreams(int(np.random.rand()*100))
    train_model = theano.function(
                 inputs = [index, eta, noise],
                 outputs = cost,
                 updates = [(param, param - eta * gparam) 
                    for param, gparam in zip(classifier.params, gparams)],
                 givens = {
                         # train_dims[1] is the number of columns (features) in the training data
                         # apparently trainX gets first added to the random numbers before its sliced
                         # Hence we use 784 (features) random numbers and not 100 (batch_size) random numbers
                         # X : trainX[index * batch_size : (index + 1) * batch_size] + theano_rng.normal(size=(train_dims[1],))* n_scale,
                         X : trainX[index * batch_size : (index + 1) * batch_size] + noise,
                         Y : trainY[index * batch_size : (index + 1) * batch_size]
                     }
             )
    
    validate_model = theano.function(
                inputs = [index],
                outputs = classifier.errors(Y),
                givens = {
                         X : validX[index * batch_size : (index + 1) * batch_size],
                         Y : validY[index * batch_size : (index + 1) * batch_size]
                }
            )
    
    test_model = theano.function(
                inputs = [index],
                outputs = classifier.errors(Y),
                givens = {
                         X : testX[index * batch_size : (index + 1) * batch_size],
                         Y : testY[index * batch_size : (index + 1) * batch_size]
                }
            )


    train_error = []
    valid_error = []
    test_error  = []

    # Calculate the number of batches.
    n_train_batches = int(train_size / batch_size)
    n_val_batches = int(valid_size / batch_size)
    n_test_batches = int(test_size / batch_size)

    ANNEAL = 10*train_size # rate at which learning parameter "eta" is reduced as iterations increase ( momentum )
    print("Anneal = {}".format(ANNEAL))
    
    start_time = timeit.default_timer()
    learn_rate = etaVal

    # Initial Gaussian Noise
    gaussian_noise = 0
    for epoch in xrange(epochs):
        # shuffle data, reset the seed so that trainX and trainY are randomized
#.........這裏部分代碼省略.........

# 需要導入模塊: from mlp import MLP [as 別名]
# 或者: from mlp.MLP import errors [as 別名]
def fun_mlp(shared_args, private_args, this_queue, that_queue):
    '''
    shared_args 
    contains neural network parameters

    private_args
    contains parameters for process run on each gpu

    this_queue and that_queue are used for synchronization between processes.
    '''

    learning_rate = shared_args['learning_rate']
    n_epochs = shared_args['n_epochs']
    dataset = shared_args['dataset']
    batch_size = shared_args['batch_size']
    L1_reg = shared_args['L1_reg']
    L2_reg = shared_args['L2_reg']
    n_hidden = shared_args['n_hidden']

    ####
    # pycuda and zmq environment
    drv.init()
    dev = drv.Device(private_args['ind_gpu'])
    ctx = dev.make_context()
    sock = zmq.Context().socket(zmq.PAIR)

    if private_args['flag_client']:
        sock.connect('tcp://localhost:5000')
    else:
        sock.bind('tcp://*:5000')
    ####

    ####
    # import theano related
    import theano.sandbox.cuda
    theano.sandbox.cuda.use(private_args['gpu'])

    import theano
    import theano.tensor as T

    from logistic_sgd import load_data
    from mlp import MLP

    import theano.misc.pycuda_init
    import theano.misc.pycuda_utils

    ####


    datasets = load_data(dataset)

    train_set_x, train_set_y = datasets[0]
    valid_set_x, valid_set_y = datasets[1]
    test_set_x, test_set_y = datasets[2]

    # compute number of minibatches for training, validation and testing
    n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
    n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] / batch_size

    ######################
    # BUILD ACTUAL MODEL #
    ######################
    print '... building the model'

    # allocate symbolic variables for the data
    index = T.lscalar()  # index to a [mini]batch
    x = T.matrix('x')  # the data is presented as rasterized images
    y = T.ivector('y')  # the labels are presented as 1D vector of
    # [int] labels

    rng = np.random.RandomState(1234)

    classifier = MLP(rng=rng, input=x, n_in=28 * 28,
                     n_hidden=n_hidden, n_out=10)

    cost = (classifier.negative_log_likelihood(y)
            + L1_reg * classifier.L1
            + L2_reg * classifier.L2_sqr)

    validate_model = theano.function(
        inputs=[index],
        outputs=classifier.errors(y),
        givens={x: valid_set_x[index * batch_size:(index + 1) * batch_size],
                y: valid_set_y[index * batch_size:(index + 1) * batch_size]}
    )

    gparams = [T.grad(cost, param) for param in classifier.params]

    updates = [(param, param - learning_rate * gparam)
               for param, gparam in zip(classifier.params, gparams)]

    train_model = theano.function(
        inputs=[index],
        outputs=cost,
        updates=updates,
        givens={
            x: train_set_x[index * batch_size: (index + 1) * batch_size],
            y: train_set_y[index * batch_size: (index + 1) * batch_size]})
    ####
    # setting pycuda and
#.........這裏部分代碼省略.........

# 需要導入模塊: from mlp import MLP [as 別名]
# 或者: from mlp.MLP import errors [as 別名]
def test_mlp_admm(learning_rate=0.5, L1_reg=0.00, L2_reg=0.0001, n_epochs=30000,
                  dataset='mnist.pkl.gz', batch_size=1000, n_hidden=500, rho=0.05):
    """
    Demonstrate stochastic gradient descent optimization for a multilayer
    perceptron

    This is demonstrated on MNIST.

    :type learning_rate: float
    :param learning_rate: learning rate used (factor for the stochastic
    gradient

    :type L1_reg: float
    :param L1_reg: L1-norm's weight when added to the cost (see
    regularization)

    :type L2_reg: float
    :param L2_reg: L2-norm's weight when added to the cost (see
    regularization)

    :type n_epochs: int
    :param n_epochs: maximal number of epochs to run the optimizer

    :type dataset: string
    :param dataset: the path of the MNIST dataset file from
                 http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz


   """

    f = open('result.txt', 'w')
    datasets = load_data(dataset)

    train_set_x, train_set_y = datasets[0]
    valid_set_x, valid_set_y = datasets[1]
    test_set_x, test_set_y = datasets[2]

    # compute number of minibatches for training, validation and testing
    n_train_batches = train_set_x.get_value(borrow=True).shape[0] / batch_size
    n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] / batch_size
    n_test_batches = test_set_x.get_value(borrow=True).shape[0] / batch_size

    ######################
    # BUILD ACTUAL MODEL #
    ######################
    print '... building the model'

    # allocate symbolic variables for the data
    index = T.lscalar()  # index to a [mini]batch
    x = T.matrix('x')  # the data is presented as rasterized images
    y = T.ivector('y')  # the labels are presented as 1D vector of
    # [int] labels
    hw1 = T.matrix('hw1')
    hb1 = T.dvector('hb1')
    lw = T.matrix('lw')
    lb = T.dvector('hb')

    a1_hw1 = T.matrix('a1_hw1')
    a1_hb1 = T.dvector('a1_hb1')
    a1_lw = T.matrix('a1_lw')
    a1_lb = T.dvector('a1_hb')

    a2_hw1 = T.matrix('a2_hw1')
    a2_hb1 = T.dvector('a2_hb1')
    a2_lw = T.matrix('a2_lw')
    a2_lb = T.dvector('a2_hb')

    rng = numpy.random.RandomState(1234)

    # construct the MLP class
    classifier = MLP(
        rng=rng,
        input=x,
        n_in=28 * 28,
        n_hidden=n_hidden,
        n_out=10
    )

    update_model = theano.function(
        inputs=[hw1, hb1, lw, lb],
        updates=[(param, uparam)
                 for param, uparam in zip(classifier.params, [hw1, hb1, lw, lb])
                 ]
    )

    test_model = theano.function(
        inputs=[index],
        outputs=classifier.errors(y),
        givens={
            x: test_set_x[index * batch_size:(index + 1) * batch_size],
            y: test_set_y[index * batch_size:(index + 1) * batch_size]
        }
    )

    validate_model = theano.function(
        inputs=[index],
        outputs=classifier.errors(y),
        givens={
            x: valid_set_x[index * batch_size:(index + 1) * batch_size],
            y: valid_set_y[index * batch_size:(index + 1) * batch_size]
#.........這裏部分代碼省略.........

# 需要導入模塊: from mlp import MLP [as 別名]
# 或者: from mlp.MLP import errors [as 別名]
class MLPClassifier(object) :
    
    def __init__(self,input_size,output_size,n_hidden=500,learning_rate=0.01, 
            L1_reg=0.00, L2_reg=0.0001, 
            n_epochs=1000,batch_size=20):
        self.learning_rate = learning_rate
        self.L1_reg = L1_reg
        self.L2_reg = L2_reg
        self.n_epochs = n_epochs
        self.batch_size=batch_size
        self.n_hidden = n_hidden
        self.x = T.matrix('x')      
        self.mlp =  MLP(input = self.x, n_in = input_size, \
                     n_hidden = n_hidden, n_out = output_size)
        
    def fit(self,X,y):
         X_train, X_valid, y_train, y_valid = self.splitData(X,y)
         train_model(self.mlp,self.x, X_train,X_valid,y_train,
                     y_valid, self.L1_reg, 
                     self.L2_reg, self.learning_rate, 
                     self.n_epochs, self.batch_size)
#        
    def predict(self,X, y = None):        
        fit_model = theano.function(
            inputs=[],
            outputs=self.mlp.predict_class,
            givens={self.x : X}
            )
        output = fit_model()
        if(y != None):
            validate_model = theano.function(inputs=[],
            outputs=self.mlp.errors(y),
            givens={
                self.x: X,
                y: y})
            print((' validation error %f %%') % (validate_model() * 100.))
        return fit_model()
        
    def predict_proba(self,X, y = None):        
        fit_model = theano.function(
            inputs=[],
            outputs=self.mlp.predict_proba,
            givens={self.x : X}
            )
        output = fit_model()
        if(y != None):
            validate_model = theano.function(inputs=[],
            outputs=self.mlp.errors(y),
            givens={
                self.x: X,
                y: y})
            print((' validation error %f %%') % (validate_model() * 100.))
        return fit_model()
        
    def getSharedInstance(self,array):
            return theano.shared(np.asarray(array, dtype=theano.config.floatX))
        
    def splitData(self,X,y):    
        r = np.random.rand(X.shape[0])    
        
        X_train = self.getSharedInstance(X[r<0.9])
        X_valid = self.getSharedInstance(X[r>=0.9])
        
        y_train = T.as_tensor_variable(y[r<0.9])
        y_valid = T.as_tensor_variable(y[r>=0.9])
        
        # First 90% train, Lirst 10% validation
        return  X_train, \
                X_valid, \
                T.cast(y_train,'int32'), \
                T.cast(y_valid,'int32')

# 需要導入模塊: from mlp import MLP [as 別名]
# 或者: from mlp.MLP import errors [as 別名]
def mlp_mnist_train(learning_rate=0.01, L1_reg=0.00, L2_reg=0.0001, n_epochs=1000, dataset='mnist.pkl.gz', batch_size=20, n_hidden=500):

    datasets = load_data(dataset)

    train_set_x, train_set_y = datasets[0]
    valid_set_x, valid_set_y = datasets[1]
    test_set_x, test_set_y = datasets[2]

    n_train_batches = train_set_x.get_value(borrow=True).shape[0] // batch_size
    n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] // batch_size
    n_test_batches = test_set_x.get_value(borrow=True).shape[0] // batch_size

    print('... building the model')

    index = T.lscalar()  # index to a [mini]batch
    x = T.matrix('x')  # the data is presented as rasterized images
    y = T.ivector('y')  # the labels are presented as 1D vector of
                                                # [int] labels
    rng = np.random.RandomState(1234)

    classifier = MLP(
        rng=rng,
        input=x,
        n_in=28*28,
        n_hidden=n_hidden,
        n_out=10
    )

    cost = (
        classifier.negative_log_likelihood(y) + L1_reg*classifier.L1 + L2_reg*classifier.L2_sqr
    )


    test_model = theano.function(
        inputs=[index],
        outputs=classifier.errors(y),
        givens={
            x: test_set_x[index * batch_size:(index + 1) * batch_size],
            y: test_set_y[index * batch_size:(index + 1) * batch_size]
        }
    )

    validate_model = theano.function(
        inputs=[index],
        outputs=classifier.errors(y),
        givens={
            x: valid_set_x[index * batch_size:(index + 1) * batch_size],
            y: valid_set_y[index * batch_size:(index + 1) * batch_size]
        }
    )

    gparams = [T.grad(cost, param) for param in classifier.params]

    updates = [
        (param, param - learning_rate * gparam) for param, gparam in zip(classifier.params, gparams)
    ]

    train_model = theano.function(
        inputs=[index],
        outputs=cost,
        updates=updates,
        givens={
            x: train_set_x[index * batch_size: (index + 1) * batch_size],
            y: train_set_y[index * batch_size: (index + 1) * batch_size]
        }
    )


    print('... training')
    patience = 10000  # look as this many examples regardless
    patience_increase = 2  # wait this much longer when a new best is

    improvement_threshold = 0.995  # a relative improvement of this much is
    validation_frequency = min(n_train_batches, patience // 2)

    best_validation_loss = np.inf
    best_iter = 0
    test_score = 0.
    start_time = timeit.default_timer()

    epoch = 0
    done_looping = False

    while (epoch < n_epochs) and (not done_looping):
        epoch = epoch + 1
        for minibatch_index in range(n_train_batches):
            minibatch_avg_cost = train_model(minibatch_index)
            iter = (epoch - 1) * n_train_batches + minibatch_index

            if (iter + 1) % validation_frequency == 0:
                validation_losses = [validate_model(i) for i in range(n_valid_batches)]
                this_validation_loss = np.mean(validation_losses)
                print(
                    'epoch %i, minibatch %i/%i, validation error %f %%' % 
                    ( 
                        epoch, 
                        minibatch_index + 1,
                        n_train_batches,
                        this_validation_loss * 100.
                    )
#.........這裏部分代碼省略.........

# 需要導入模塊: from mlp import MLP [as 別名]
# 或者: from mlp.MLP import errors [as 別名]
def main(learning_rate=0.01, L1_reg=0.00, L2_reg=0.0001, n_epochs=1000,
             dataset='mnist.pkl.gz', batch_size=20, n_hidden=[500, 500]):
    """
    Demonstrate stochastic gradient descent optimization for a multilayer
    perceptron

    This is demonstrated on MNIST.

    :type learning_rate: float
    :param learning_rate: learning rate used (factor for the stochastic
    gradient

    :type L1_reg: float
    :param L1_reg: L1-norm's weight when added to the cost (see
    regularization)

    :type L2_reg: float
    :param L2_reg: L2-norm's weight when added to the cost (see
    regularization)

    :type n_epochs: int
    :param n_epochs: maximal number of epochs to run the optimizer

    :type dataset: string
    :param dataset: the path of the MNIST dataset file from
                 http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz


   """
    if add_noise==True:
        datasets = load_data(dataset, nb_classes=nb_classes, binarize=binarize,
                             noise_prop=noise_proportion)
    else:
        datasets = load_data(dataset, nb_classes=nb_classes, binarize=binarize)

    train_set_x, train_set_y = datasets[0]
    valid_set_x, valid_set_y = datasets[1]
    test_set_x, test_set_y = datasets[2]

    print('Showing data samples')
    if nb_classes == 2:
        labels = ['odd', 'even']
    else:
        labels = [0,1,2,3,4,5,6,7,8,9]
    imshow_samples(train_set_x.get_value(), train_set_y,
            valid_set_x.get_value(), valid_set_y, num_samples=4, labels=labels)
    plt.pause(0.0001)
    diary.save_figure(plt, filename='samples', extension='svg')

    # compute number of minibatches for training, validation and testing
    n_train_batches = train_set_x.get_value(borrow=True).shape[0] // batch_size
    n_valid_batches = valid_set_x.get_value(borrow=True).shape[0] // batch_size
    n_test_batches = test_set_x.get_value(borrow=True).shape[0] // batch_size

    ######################
    # BUILD ACTUAL MODEL #
    ######################
    print('... building the model')

    # allocate symbolic variables for the data
    index = T.lscalar()  # index to a [mini]batch
    x = T.matrix('x')  # the data is presented as rasterized images
    y = T.ivector('y')  # the labels are presented as 1D vector of
                        # [int] labels

    rng = numpy.random.RandomState(1234)

    # construct the MLP class
    classifier = MLP(
        rng=rng,
        input=x,
        n_in=28 * 28,
        n_hidden=n_hidden,
        n_out=nb_classes
    )

    # start-snippet-4
    # the cost we minimize during training is the negative log likelihood of
    # the model plus the regularization terms (L1 and L2); cost is expressed
    # here symbolically
    cost = (
        classifier.negative_log_likelihood(y)
        + L1_reg * classifier.L1
        + L2_reg * classifier.L2_sqr
    )
    # end-snippet-4

    # compiling a Theano function that computes the mistakes that are made
    # by the model on a minibatch
    test_model = theano.function(
        inputs=[index],
        outputs=classifier.errors(y),
        givens={
            x: test_set_x[index * batch_size:(index + 1) * batch_size],
            y: test_set_y[index * batch_size:(index + 1) * batch_size]
        }
    )

    validate_model = theano.function(
        inputs=[index],
#.........這裏部分代碼省略.........

# 需要導入模塊: from mlp import MLP [as 別名]
# 或者: from mlp.MLP import errors [as 別名]
def train_mlp(L1_reg = 0.0, L2_reg = 0.0000, num_batches_per_bunch = 512, batch_size = 1, num_bunches_queue = 5, offset = 0, path_name = '/afs/inf.ed.ac.uk/user/s12/s1264845/scratch/s1264845/data/'):
    

    voc_list = Vocabulary(path_name + 'train')
    voc_list.vocab_create()
    vocab = voc_list.vocab
    vocab_size = voc_list.vocab_size
    
    voc_list_valid = Vocabulary(path_name + 'valid')
    voc_list_valid.vocab_create()
    count = voc_list_valid.count

    voc_list_test = Vocabulary(path_name + 'test')
    voc_list_test.vocab_create()
    no_test_tokens = voc_list_test.count
    print 'The number of sentenses in test set:', no_test_tokens
 
    #print 'number of words in valid data:', count 
    dataprovider_train = DataProvider(path_name + 'train', vocab, vocab_size )
    dataprovider_valid = DataProvider(path_name + 'valid', vocab, vocab_size )
    dataprovider_test = DataProvider(path_name + 'test', vocab, vocab_size )

    #learn_list = [0.1, 0.1, 0.1, 0.75, 0.5, 0.25, 0.125, 0.0625, 0]
    exp_name = 'fine_tuning.hdf5'
    posterior_path = 'log_likelihoods'
    print '..building the model'

    #symbolic variables for input, target vector and batch index
    index = T.lscalar('index')
    x = T.fmatrix('x')
    y = T.ivector('y')
    learning_rate = T.fscalar('learning_rate') 

    #theano shares variables for train, valid and test
    train_set_x = theano.shared(numpy.empty((1,1), dtype='float32'), allow_downcast = True)
    train_set_y = theano.shared(numpy.empty((1), dtype = 'int32'), allow_downcast = True)
    
    valid_set_x = theano.shared(numpy.empty((1,1), dtype='float32'), allow_downcast = True)
    valid_set_y = theano.shared(numpy.empty((1), dtype = 'int32'), allow_downcast = True)
    
    test_set_x = theano.shared(numpy.empty((1,1), dtype='float32'), allow_downcast = True)
    test_set_y = theano.shared(numpy.empty((1), dtype = 'int32'), allow_downcast = True)
    
    rng = numpy.random.RandomState(1234) 
   
    classifier = MLP(rng = rng, input = x, n_in = vocab_size, n_hidden1 = 30, n_hidden2 = 60 , n_out = vocab_size)
    #classifier = MLP(rng = rng, input = x, n_in = vocab_size, n_hidden = 60, n_out = vocab_size)

    cost = classifier.negative_log_likelihood(y) + L1_reg * classifier.L1 + L2_reg * classifier.L2_sqr
    
    #constructor for learning rate class
    learnrate_schedular = LearningRateNewBob(start_rate=0.001, scale_by=.5, max_epochs=9999,\
                                    min_derror_ramp_start=.1, min_derror_stop=.1, init_error=100.)

    #learnrate_schedular = LearningRateList(learn_list)

    frame_error = classifier.errors(y)
    likelihood = classifier.sum(y)

    #test model
    test_model = theano.function(inputs = [index], outputs = likelihood,  \
                                 givens = {x: test_set_x[index * batch_size:(index + 1) * batch_size],
                                           y: test_set_y[index * batch_size:(index + 1) * batch_size]})
    #validation_model
    validate_model = theano.function(inputs = [index], outputs = [frame_error, likelihood], \
                                     givens = {x: valid_set_x[index * batch_size:(index + 1) * batch_size],
                                               y: valid_set_y[index * batch_size:(index + 1) * batch_size]})

    gradient_param = []
    #calculates the gradient of cost with respect to parameters 
    for param in classifier.params:
        gradient_param.append(T.cast(T.grad(cost, param), 'float32'))
        
    updates = []
    
    for param, gradient in zip(classifier.params, gradient_param):
        updates.append((param, param - learning_rate * gradient))
    
    #training_model
    train_model = theano.function(inputs = [index, theano.Param(learning_rate, default = 0.01)], outputs = cost, updates = updates, \
                                 givens = {x: train_set_x[index * batch_size:(index + 1) * batch_size],
                                           y: train_set_y[index * batch_size:(index + 1) * batch_size]})
   

    #theano.printing.pydotprint(train_model, outfile = "pics/train.png", var_with_name_simple = True) 
    #path_save = '/afs/inf.ed.ac.uk/user/s12/s1264845/scratch/s1264845/mlp/saved_weights/' 
    print '.....training'    
    best_valid_loss = numpy.inf    
    epoch = 1
    start_time = time.time()
    while(learnrate_schedular.get_rate() != 0):
	
	print 'learning_rate:', learnrate_schedular.get_rate()
        print 'epoch_number:', learnrate_schedular.epoch
        
        frames_showed, progress = 0, 0
        start_epoch_time = time.time()
        
        tqueue = TNetsCacheSimple.make_queue()
        cache = TNetsCacheSimple(tqueue, shuffle_frames = True, offset=0, \
#.........這裏部分代碼省略.........

# 需要導入模塊: from mlp import MLP [as 別名]
# 或者: from mlp.MLP import errors [as 別名]
def run_mlp(datasets, n_epochs, n_hidden, batch_size, L1_reg, L2_reg, learning_rate=None):
    
    train_set_x, train_set_y = datasets[0]
    valid_set_x, valid_set_y = datasets[1]
    
    del datasets
    
    n_out = len(train_set_y[0])
    n_in = len(train_set_x[0])
    
    sys.stderr.write("\nNumber of nodes:-\n  Input: {0}\n  Hidden: {1}\n  Output: {2}\n".format(n_in, n_hidden, n_out))
    sys.stderr.write("\nEpochs: {0}\nBatch Size: {1}\n".format(n_epochs, batch_size))
    sys.stderr.write("\nL1_reg: {0}\nL2_reg: {1}\n".format(L1_reg, L2_reg))
    
    n_train_batches = get_number_of_batches(train_set_x, batch_size)
    n_valid_batches = get_number_of_batches(valid_set_x, batch_size)
    
    train_set_x, train_set_y = shared_dataset(train_set_x, train_set_y)
    valid_set_x, valid_set_y = shared_dataset(valid_set_x, valid_set_y)
    
    sys.stderr.write('... building the model\n')

    #change here possibly
    index = T.lscalar()  # index to a [mini]batch
    x = T.matrix('x')    # the data is presented as rasterized images
    #y = T.ivector('y')  # the labels are presented as 1D vector of
    y = T.matrix('y')   # [int] labels
    
    rng = numpy.random.RandomState(1234)

    # construct the MLP class
    classifier = MLP(rng=rng, input=x, n_in=n_in, n_hidden=n_hidden, n_out=n_out)
    
    cost = classifier.errors(y) + L1_reg * classifier.L1 + L2_reg * classifier.L2_sqr

    # compiling a Theano function that computes the mistakes that are made
    # by the model on a minibatch
    validate_model = theano.function(inputs=[index],
            outputs=classifier.errors(y),
            givens={
                x: valid_set_x[index * batch_size:(index + 1) * batch_size],
                y: valid_set_y[index * batch_size:(index + 1) * batch_size]})

    # compute the gradient of cost with respect to theta (sotred in params)
    # the resulting gradients will be stored in a list gparams
    gparams = [T.grad(cost, param) for param in classifier.params]
    
    if learning_rate is not None:
        # SGD
        sys.stderr.write('\nUsing SGD with learning rate: {0}\n'.format(learning_rate))
        updates = [(param, param - learning_rate * gparam) for param, gparam in zip(classifier.params, gparams)]
    else:
        # RPROP+
        sys.stderr.write('\nUsing RPROP+...\n')
        updates = [(param, param_updated) for (param, param_updated) in rprop_plus_updates(classifier.params, gparams)]
    
    # compiling a Theano function `train_model` that returns the cost, but
    # in the same time updates the parameter of the model based on the rules
    # defined in `updates`
    train_model = theano.function(inputs=[index], outputs=cost,
            updates=updates,
            givens={
                x: train_set_x[index * batch_size:(index + 1) * batch_size],
                y: train_set_y[index * batch_size:(index + 1) * batch_size]})

    sys.stderr.write('... training\n')

    # early-stopping parameters
    patience = 100000  # look as this many examples regardless
    validation_frequency = min(n_train_batches, patience / 2)
    
    best_params = None
    best_validation_loss = numpy.inf
    best_iter = 0
    test_score = 0.
    start_time = time.clock()

    epoch = 0
    done_looping = False

    while (epoch < n_epochs) and (not done_looping):
        epoch = epoch + 1
        for minibatch_index in xrange(n_train_batches):

            minibatch_avg_cost = train_model(minibatch_index)
            # iteration number
            iter = (epoch - 1) * n_train_batches + minibatch_index

            if (iter + 1) % validation_frequency == 0:
                
                validation_losses = [validate_model(i) for i
                                     in xrange(n_valid_batches)]
                this_validation_loss = numpy.mean(validation_losses)

                print >> sys.stderr, ('epoch %i, minibatch %i/%i, validation error %f %%' %
                     (epoch, minibatch_index + 1, n_train_batches,
                      this_validation_loss * 100.))

    end_time = time.clock()
    print >> sys.stderr, (('Optimization complete. Best validation score of %f %% '
#.........這裏部分代碼省略.........

# 需要導入模塊: from mlp import MLP [as 別名]
# 或者: from mlp.MLP import errors [as 別名]
def run_mlp(datasets, n_epochs, n_hidden, batch_size, L1_reg, L2_reg, learning_rate=None):
    
    train_set_x, train_set_y = datasets[0]
    valid_set_x, valid_set_y = datasets[1]
    del datasets
    
    n_in = train_set_x.shape[1]
    n_out = train_set_y.shape[1]
    n_valid_batches = get_number_of_batches(valid_set_x.shape[0], batch_size)
    n_tr_load_batches = get_number_of_batches(train_set_x.shape[0],  TR_LOAD_SIZE)
    
    train_set_x = csr_matrix(train_set_x)
    train_set_y = csr_matrix(train_set_y)
    
    sys.stderr.write("\nNumber of nodes:-\n  Input: {0}\n  Hidden: {1}\n  Output: {2}\n".format(n_in, n_hidden, n_out))
    sys.stderr.write("\nEpochs: {0}\nBatch Size: {1}\nLoad size: {2}".format(n_epochs, batch_size, TR_LOAD_SIZE))
    sys.stderr.write("\nL1_reg: {0}\nL2_reg: {1}\n".format(L1_reg, L2_reg))
    
    valid_set_x, valid_set_y = shared_dataset(valid_set_x.todense(), valid_set_y.todense())
    
    sys.stderr.write('... building the model\n')

    # index to a [mini]batch
    index = T.lscalar()
    x = T.imatrix('x')
    y = T.imatrix('y')
    
    rng = numpy.random.RandomState(1234)

    # construct the MLP class
    classifier = MLP(rng=rng, input=x, n_in=n_in, n_hidden=n_hidden, n_out=n_out)
    
    cost = classifier.errors(y) + L1_reg * classifier.L1 + L2_reg * classifier.L2_sqr

    # compiling a Theano function that computes the mistakes that are made
    # by the model on a minibatch
    validate_model = theano.function(inputs=[index],
            outputs=classifier.errors(y),
            givens={
                x: valid_set_x[index * batch_size:(index + 1) * batch_size],
                y: valid_set_y[index * batch_size:(index + 1) * batch_size]
                })

    # compute the gradient of cost with respect to theta (sotred in params)
    # the resulting gradients will be stored in a list gparams
    gparams = [T.grad(cost, param) for param in classifier.params]
    
    if learning_rate is not None:
        # SGD
        sys.stderr.write('\nUsing SGD with learning rate: {0}\n'.format(learning_rate))
        updates = [(param, param - learning_rate * gparam) for param, gparam in zip(classifier.params, gparams)]
    else:
        # RPROP+
        sys.stderr.write('\nUsing RPROP+...\n')
        updates = [(param, param_updated) for (param, param_updated) in rprop_plus_updates(classifier.params, gparams)]
    
    sys.stderr.write('... training\n')
    
    # early-stopping parameters
    patience = 50000  # look as this many examples regardless
    patience_increase = 2  # wait this much longer when a new best is
                                      # found
    improvement_threshold = 0.995  # a relative improvement of this much is
                                      # considered significant
    validation_frequency = min(get_number_of_batches(TR_LOAD_SIZE, batch_size), patience / 2)
                                      # go through this many
                                      # minibatche before checking the network
                                      # on the validation set; in this case we
                                      # check every epoch
    
    best_params = None
    best_validation_loss = numpy.inf
    best_iter = 0
    start_time = time.clock()
    
    epoch = 0
    done_looping = False
    
    while (epoch < n_epochs) and (not done_looping):
        epoch = epoch + 1
        
        for load_tr_index in xrange(n_tr_load_batches):
            
            spliced_train_set_x, spliced_train_set_y, n_train_batches = sprase_to_normal_spliced_shared(\
                                                        train_set_x, train_set_y, load_tr_index, batch_size)
        
            train_model = theano.function(inputs=[index], outputs=cost, updates=updates,
                givens={
                    x: spliced_train_set_x[index*batch_size : (index+1)*batch_size],
                    y: spliced_train_set_y[index*batch_size : (index+1)*batch_size]
                })
        
            for minibatch_index in xrange(n_train_batches):
            
                minibatch_avg_cost = train_model(minibatch_index)
                iter = (epoch - 1) * n_train_batches + minibatch_index
            
                if (iter + 1) % validation_frequency == 0:
                
                    validation_losses = [validate_model(i) for i in xrange(n_valid_batches)]
#.........這裏部分代碼省略.........