Python auxiliary.GradientDescent类代码示例

 def __init__(self):
     # standard parameters
     self.epsilon = 2.0 #Initial value of sigmas
     self.baseline=0.0 #Moving average baseline, used for sigma adaption
     self.best=-1000000.0 #TODO ersetzen durch -inf
     self.symCount=1.0 #Switch for symetric sampling
     self.gd = GradientDescent()
     self.gdSig = GradientDescent()
     self.wDecay = 0.001 #lasso weight decay (0 to deactivate)

class FiniteDifferences(ContinuousOptimizer):
    """ Basic finite difference method. """

    epsilon = 1.0
    gamma = 0.999
    batchSize = 10

    # gradient descent parameters
    learningRate = 0.1
    learningRateDecay = None
    momentum = 0.0
    rprop = False

    

    def _setInitEvaluable(self, evaluable):
        ContinuousOptimizer._setInitEvaluable(self, evaluable)
        self.current = self._initEvaluable
        self.gd = GradientDescent()
        self.gd.alpha = self.learningRate
        if self.learningRateDecay is not None:
            self.gd.alphadecay = self.learningRateDecay
        self.gd.momentum = self.momentum
        self.gd.rprop = self.rprop
        self.gd.init(self._initEvaluable)

    def perturbation(self):
        """ produce a parameter perturbation """
        deltas = random.uniform(-self.epsilon, self.epsilon, self.numParameters)
        # reduce epsilon by factor gamma
        self.epsilon *= self.gamma
        return deltas

    def _learnStep(self):
        """ calls the gradient calculation function and executes a step in direction
            of the gradient, scaled with a small learning rate alpha. """

        # initialize matrix D and vector R
        D = ones((self.batchSize, self.numParameters))
        R = zeros((self.batchSize, 1))

        # calculate the gradient with pseudo inverse
        for i in range(self.batchSize):
            deltas = self.perturbation()
            x = self.current + deltas
            D[i, :] = deltas
            R[i, :] = self._oneEvaluation(x)
        beta = dot(pinv(D), R)
        gradient = ravel(beta)

        # update the weights
        self.current = self.gd(gradient)

class FDBasic(FDLearner):
    def __init__(self):
        # standard parameters
        self.epsilon = 1.0
        self.gamma = 0.999
        self.gd = GradientDescent()

    def setModule(self, module):
        FDLearner.setModule(self, module)        
        self.gd.init(self.original)

    def perturbate(self):
        """ perturb the parameters. """
        # perturb the parameters and store the deltas in dataset
        deltas = random.uniform(-self.epsilon, self.epsilon, self.module.paramdim)
        # reduce epsilon by factor gamma
        self.epsilon *= self.gamma
        self.ds.append('deltas', deltas)
        # change the parameters in module (params is a pointer!)
        params = self.module.params
        params[:] = self.original + deltas

    def learn(self):
        """ calls the gradient calculation function and executes a step in direction
            of the gradient, scaled with a small learning rate alpha. """
        assert self.ds != None
        assert self.module != None
        
        # get the deltas from the dataset
        deltas = self.ds.getField('deltas')
        
        # initialize matrix D and vector R
        D = ones((self.ds.getNumSequences(), self.module.paramdim + 1))
        R = zeros((self.ds.getNumSequences(), 1))
        
        # calculate the gradient with pseudo inverse
        for seq in range(self.ds.getNumSequences()):
            _state, _action, reward = self.ds.getSequence(seq)
            D[seq,:-1] = deltas[seq,:]
            R[seq,:] = mean(reward)
        
        beta = dot(pinv(D), R)        
        gradient = ravel(beta[:-1])
        
        # update the weights
        self.original = self.gd(gradient)       
        self.module._setParameters(self.original)
           
        self.module.reset()

 def __init__(self, module, dataset=None, learningrate=0.01, lrdecay=1.0,
              momentum=0., verbose=False, batchlearning=False,
              weightdecay=0.):
     """Create a BackpropTrainer to train the specified `module` on the
     specified `dataset`.
     The learning rate gives the ratio of which parameters are changed into
     the direction of the gradient. The learning rate decreases by `lrdecay`,
     which is used to to multiply the learning rate after each training
     step. The parameters are also adjusted with respect to `momentum`, which
     is the ratio by which the gradient of the last timestep is used.
     If `batchlearning` is set, the parameters are updated only at the end of
     each epoch. Default is False.
     `weightdecay` corresponds to the weightdecay rate, where 0 is no weight
     decay at all.
     """
     Trainer.__init__(self, module)
     self.setData(dataset)
     self.verbose = verbose
     self.batchlearning = batchlearning
     self.weightdecay = weightdecay
     self.epoch = 0
     self.totalepochs = 0
     # set up gradient descender
     self.descent = GradientDescent()
     self.descent.alpha = learningrate
     self.descent.momentum = momentum
     self.descent.alphadecay = lrdecay
     self.descent.init(module.params)

 def __init__(self):
     # standard parameters
     self.epsilon = 2.0 #Initial value of exploration size
     self.baseline=0.0 #Moving average baseline, used just for visualisation
     self.best=-1000000.0 #TODO ersetzen durch -inf
     self.symCount=1.0 #Switch for symetric sampling
     self.gd = GradientDescent()
     self.gamma=0.9995 #Exploration decay factor

 def _additionalInit(self):
     if self.sigmaLearningRate is None:
         self.sigmaLearningRate = self.learningRate    
     self.gdSig = GradientDescent()
     self.gdSig.alpha = self.sigmaLearningRate
     self.gdSig.rprop = self.rprop
     self.sigList = ones(self.numParameters) * self.epsilon #Stores the list of standard deviations (sigmas)
     self.gdSig.init(self.sigList)
     self.baseline = None

 def _setInitEvaluable(self, evaluable):
     ContinuousOptimizer._setInitEvaluable(self, evaluable)
     self.current = self._initEvaluable
     self.gd = GradientDescent()
     self.gd.alpha = self.learningRate
     if self.learningRateDecay is not None:
         self.gd.alphadecay = self.learningRateDecay
     self.gd.momentum = self.momentum
     self.gd.rprop = self.rprop
     self.gd.init(self._initEvaluable)

class PolicyGradientLearner(RLLearner):
    """ The PolicyGradientLearner takes a ReinforcementDataSet which has been extended with the log likelihood
        of each parameter for each time step. It additionally takes a Module which has been extended with
        a gaussian layer on top. It then changes the weights of both the gaussian layer and the rest of the
        module according to a specific gradient descent, which is a derivation of this base class.
    """
    def __init__(self):
        self.gd = GradientDescent()
        
    def setAlpha(self, alpha):
        """ pass the alpha value through to the gradient descent object """
        self.gd.alpha = alpha
        print "patching through to gd", alpha
    
    def getAlpha(self):
        return self.gd.alpha
    
    alpha = property(getAlpha, setAlpha)
        
    def setModule(self, module):
        RLLearner.setModule(self, module)
        self.gd.init(module.params)    
        
    def learn(self):
        """ calls the gradient calculation function and executes a step in direction
            of the gradient, scaled with a small learning rate alpha. """
        assert self.ds != None
        assert self.module != None
        
        # calculate the gradient with the specific function from subclass
        gradient = ravel(self.calculateGradient())

        # scale gradient if it has too large values
        if max(gradient) > 1000:
            gradient = gradient / max(gradient) * 1000
        
        # update the parameters
        p = self.gd(gradient)
        self.module._setParameters(p)
        self.module.reset()
        
    def calculateGradient(self):
        abstractMethod()

 def __init__(self):        
     # gradient descender
     self.gd = GradientDescent()
     
     # create default explorer
     self._explorer = None
     
     # loglh dataset
     self.loglh = None
     
     # network to tie module and explorer together
     self.network = None

 def __init__(self, module, dataset=None, learningrate=0.01, lrdecay=1.0,
              momentum=0., verbose=False, batchlearning=False,
              weightdecay=0.):
     Trainer.__init__(self, module)
     self.setData(dataset)
     self.verbose = verbose
     self.batchlearning = batchlearning
     self.weightdecay = weightdecay
     self.epoch = 0
     self.totalepochs = 0
     # set up gradient descender
     self.descent = GradientDescent()
     self.descent.alpha = learningrate
     self.descent.momentum = momentum
     self.descent.alphadecay = lrdecay
     self.descent.init(module.params)

    def __init__(self, module, dataset=None, learningrate=0.01, lrdecay=1.0,
                 momentum=0., verbose=False, batchlearning=False,
                 weightdecay=0., errfun=None):
        """Create a BackpropTrainer to train the specified `module` on the
        specified `dataset`.

        The learning rate gives the ratio of which parameters are changed into
        the direction of the gradient. The learning rate decreases by
        `lrdecay`, which is used to to multiply the learning rate after each
        training step. The parameters are also adjusted with respect to
        `momentum`, which is the ratio by which the gradient of the last
        timestep is used.

        If `batchlearning` is set, the parameters are updated only at the end
        of each epoch. Default is False.

        `weightdecay` corresponds to the weightdecay rate, where 0 is no weight
        decay at all.

        Arguments:
            errfun (func): Function that takes 2 positional arguments,
                the target (true) and predicted (estimated) output vectors, and
                returns an estimate of the signed distance to the target (true)
                output. default = lambda targ, est: (targ - est))
        """
        Trainer.__init__(self, module)
        self.setData(dataset)
        self.verbose = verbose
        self.batchlearning = batchlearning
        self.weightdecay = weightdecay
        self.epoch = 0
        self.totalepochs = 0
        # set up gradient descender
        self.descent = GradientDescent()
        self.descent.alpha = learningrate
        self.descent.momentum = momentum
        self.descent.alphadecay = lrdecay
        self.descent.init(module.params)
        self.errfun = errfun or abs_error

class TTrainer(BackpropTrainer):
    def __init__(self, module, dataset=None, learningrate=0.01, lrdecay=1.0,
                 momentum=0., verbose=False, batchlearning=False,
                 weightdecay=0.):
        super(BackpropTrainer, self).__init__(module)
        #self.setData(dataset)
        self.verbose = False
        self.batchlearning = batchlearning
        self.weightdecay = weightdecay
        self.epoch = 0
        self.totalepochs = 0
        # set up gradient descender
        self.descent = GradientDescent()
        self.descent.alpha = learningrate
        self.descent.momentum = momentum
        self.descent.alphadecay = lrdecay
        self.descent.init(module.params)


    def minibatch_training(self, dataset):

        for i in xrange(50):
            #ds = dataset.splitWithProportion(0.01)[0]
            ds = dataset.batches(1000)
            self.trainOnDataset(ds)
            ds.clear()

    def trainUntilConvergence(self, dataset=None, maxEpochs=None, verbose=None,
                              continueEpochs=10, validationProportion=0.25,FOLDER = os.curdir, modelfile = 'network.model', file = None):
        """Train the module on the dataset until it converges.

        Return the module with the parameters that gave the minimal validation
        error.

        If no dataset is given, the dataset passed during Trainer
        initialization is used. validationProportion is the ratio of the dataset
        that is used for the validation dataset.

        If maxEpochs is given, at most that many epochs
        are trained. Each time validation error hits a minimum, try for
        continueEpochs epochs to find a better one."""
        epochs = 0
        if dataset == None:
            dataset = self.ds
        if verbose == None:
            verbose = self.verbose
        # Split the dataset randomly: validationProportion of the samples for
        # validation.
        trainingData, validationData = (
            dataset.splitWithProportion(1 - validationProportion))
        if not (len(trainingData) > 0 and len(validationData)):
            raise ValueError("Provided dataset too small to be split into training " +
                             "and validation sets with proportion " + str(validationProportion))
        self.ds = trainingData
        bestweights = self.module.params.copy()
        bestverr = self.testOnData(validationData)
        trainingErrors = []
        validationErrors = [bestverr]
        while True:
            if maxEpochs != None:
                print float(epochs) / maxEpochs
            trainErr = self.train()
            validErr = self.testOnData(validationData)


            errs = [trainErr,validErr]
            print errs
            if file != None:
                with open (os.path.join(FOLDER, file), 'a') as neurons:
                    neurons.write(str(errs))
                    neurons.write('\n')

            trainingErrors.append(trainErr)
            validationErrors.append(validErr)

            if epochs == 0 or validationErrors[-1] < bestverr:
                # one update is always done
                bestverr = validationErrors[-1]
                bestweights = self.module.params.copy()
                NetworkWriter.writeToFile(self.module, os.path.join(FOLDER, modelfile))
                #with open (os.path.join(FOLDER, modelfile), 'wb') as fileObject:
                #    pickle.dump(self.module,fileObject)

            if maxEpochs != None and epochs >= maxEpochs:
                self.module.params[:] = bestweights
                break
            epochs += 1

            if len(validationErrors) >= continueEpochs * 2:
                # have the validation errors started going up again?
                # compare the average of the last few to the previous few
                old = validationErrors[-continueEpochs * 2:-continueEpochs]
                new = validationErrors[-continueEpochs:]
                if min(new) > max(old):
                    self.module.params[:] = bestweights
                    break
        trainingErrors.append(self.testOnData(trainingData))
        self.ds = dataset

        return trainingErrors, validationErrors

class ExtendedBackpropTrainer(Trainer):
    """Trainer that trains the parameters of a module according to a
    supervised dataset (potentially sequential) by backpropagating the errors
    (through time)."""

    def __init__(self, module, dataset=None, learningrate=0.01, lrdecay=1.0,
                 momentum=0., verbose=False, batchlearning=False,
                 weightdecay=0.):
        """Create a BackpropTrainer to train the specified `module` on the
        specified `dataset`.
        The learning rate gives the ratio of which parameters are changed into
        the direction of the gradient. The learning rate decreases by `lrdecay`,
        which is used to to multiply the learning rate after each training
        step. The parameters are also adjusted with respect to `momentum`, which
        is the ratio by which the gradient of the last timestep is used.
        If `batchlearning` is set, the parameters are updated only at the end of
        each epoch. Default is False.
        `weightdecay` corresponds to the weightdecay rate, where 0 is no weight
        decay at all.
        """
        Trainer.__init__(self, module)
        self.setData(dataset)
        self.verbose = verbose
        self.batchlearning = batchlearning
        self.weightdecay = weightdecay
        self.epoch = 0
        self.totalepochs = 0
        # set up gradient descender
        self.descent = GradientDescent()
        self.descent.alpha = learningrate
        self.descent.momentum = momentum
        self.descent.alphadecay = lrdecay
        self.descent.init(module.params)

    def train(self):
        """Train the associated module for one epoch."""
        assert len(self.ds) > 0, "Dataset cannot be empty."
        self.module.resetDerivatives()
        errors = 0
        ponderation = 0.
        shuffledSequences = []
        for seq in self.ds._provideSequences():
            shuffledSequences.append(seq)
        shuffle(shuffledSequences)
        for seq in shuffledSequences:
            e, p = self._calcDerivs(seq)
            errors += e
            ponderation += p
            if not self.batchlearning:
                gradient = self.module.derivs - self.weightdecay * self.module.params
                new = self.descent(gradient, errors)
                if new is not None:
                    self.module.params[:] = new
                self.module.resetDerivatives()

        if self.verbose:
            print("Total error: {z: .12g}".format(z=errors / ponderation))
        if self.batchlearning:
            self.module._setParameters(self.descent(self.module.derivs))
        self.epoch += 1
        self.totalepochs += 1
        return errors / ponderation


    def _calcDerivs(self, seq):
        """Calculate error function and backpropagate output errors to yield
        the gradient."""
        self.module.reset()
        for sample in seq:
            self.module.activate(sample[0])
        error = 0
        ponderation = 0.
        for offset, sample in reversed(list(enumerate(seq))):
            # need to make a distinction here between datasets containing
            # importance, and others
            target = sample[1]
            outerr = target - self.module.outputbuffer[offset]
            if len(sample) > 2:
                importance = sample[2]
                error += 0.5 * dot(importance, outerr ** 2)
                ponderation += sum(importance)
                self.module.backActivate(outerr * importance)
            else:
                error += 0.5 * sum(outerr ** 2)
                ponderation += len(target)
                # FIXME: the next line keeps arac from producing NaNs. I don't
                # know why that is, but somehow the __str__ method of the
                # ndarray class fixes something,
                str(outerr)
                self.module.backActivate(outerr)

        return error, ponderation

    def _checkGradient(self, dataset=None, silent=False):
        """Numeric check of the computed gradient for debugging purposes."""
        if dataset:
            self.setData(dataset)
        res = []
        for seq in self.ds._provideSequences():
            self.module.resetDerivatives()
#.........这里部分代码省略.........

 def __init__(self):
     # standard parameters
     self.epsilon = 1.0
     self.gamma = 0.999
     self.gd = GradientDescent()

class SPLA(FDLearner):
    def __init__(self):
        # standard parameters
        self.epsilon = 2.0 #Initial value of sigmas
        self.baseline=0.0 #Moving average baseline, used for sigma adaption
        self.best=-1000000.0 #TODO ersetzen durch -inf
        self.symCount=1.0 #Switch for symetric sampling
        self.gd = GradientDescent()
        self.gdSig = GradientDescent()
        self.wDecay = 0.001 #lasso weight decay (0 to deactivate)

    def setModule(self, module):
        """Sets and initializes all module settings"""
        FDLearner.setModule(self, module)
        self.original = zeros(self.module.params.shape) #Stores the parameter set
        self.sigList=ones(self.module.params.shape) #Stores the list of standard deviations (sigmas)
        self.initSigmas()
        self.deltas=zeros(self.module.params.shape) #the parameter difference vector for exploration
        self.module._setParameters(self.original) #initializes the module parameter set to zeros
        self.gd.init(self.original)
        self.numOParas=len(self.original)

    def initSigmas(self):
        self.sigList*=self.epsilon #initialize sigmas to epsilon
        self.gdSig.init(self.sigList)
        
    def genDifVect(self):
        # generates a difference vector with the given standard deviations
        self.deltas=random.normal(0.0, self.sigList)
        
    def perturbate(self):
        """ perturb the parameters. """
        self.symCount*=-1.0 #change sign of perturbation
        self.ds.append('deltas', self.symCount*self.deltas) #add perturbation to the data set
        self.module._setParameters(self.original + self.symCount*self.deltas) #set the actual perturbed parameters in module

    def learn(self):
        """ calculates the gradient and executes a step in the direction
            of the gradient, scaled with a learning rate alpha. """
        assert self.ds != None
        assert self.module != None
                       
        # calculate the gradient
        reward1=0.0 #reward of positive perturbation
        reward2=0.0 #reward of negative perturbation
        sym=1.0 #perturbation switch
        seqLen=self.ds.getNumSequences() #number of sequences done for learning
        for seq in range(seqLen):
            sym*=-1.0
            _, _, reward = self.ds.getSequence(seq)
            #add up the rewards of positive and negative perturbation role outs respectivly
            if sym==1.0: reward1+=sum(reward)
            else: reward2+=sum(reward)
        #normate rewards by seqLen 
        reward1/=float(seqLen)
        reward2/=float(seqLen)
        self.reward=(reward1+reward2)
        reward1*=2.0
        reward2*=2.0

        #check if reward is the best observed up to now
        if reward1 > self.best: self.best= reward1
        if reward2 > self.best: self.best= reward2

        #some checks at the first learnign sequence
        if self.baseline==0.0: 
            self.baseline=self.reward/2.0
            fakt=0.0
            fakt2=0.0
            if seqLen/2 != float(seqLen)/2.0: 
                print "ATTENTON!!! SPLA uses symetric sampling! Number of episodes per learning step must be even! (2 for deterministic settings, >2 for stochastic settings) A numer of episodes of ", seqLen, "is odd."
                while(True): sleep(1)
        else: 
            #calc the gradients
            if reward1!=reward2:
                #gradient estimate alla SPSA but with liklihood gradient and normalization
                fakt=(reward1-reward2)/(2.0*self.best-reward1-reward2) 
            else: fakt=0.0
            #normalized sigma gradient with moving average baseline
            fakt2=(self.reward-self.baseline)/(self.best-self.baseline) 
        self.baseline=0.9*self.baseline+0.1*self.reward #update baseline
            
        # update parameters and sigmas
        self.original = self.gd(fakt*self.deltas-self.original*self.sigList*self.wDecay)   
        print abs(self.original).sum()/self.numOParas            
        if fakt2> 0.0: #for sigma adaption alg. follows only positive gradients
            self.sigList=self.gdSig(fakt2*(self.deltas*self.deltas-self.sigList*self.sigList)/self.sigList) #apply sigma update
        self.module.reset() # reset the module 
        self.genDifVect() #generate a new perturbation vector