Python RandomStreams.multinomial方法代码示例

# 需要导入模块: from theano.tensor.shared_randomstreams import RandomStreams [as 别名]
# 或者: from theano.tensor.shared_randomstreams.RandomStreams import multinomial [as 别名]
    def test_multinomial_vector(self):
        random = RandomStreams(utt.fetch_seed())
        n = tensor.lvector()
        pvals = tensor.matrix()
        out = random.multinomial(n=n, pvals=pvals)
        assert out.ndim == 2
        f = function([n, pvals], out)

        n_val = [1, 2, 3]
        pvals_val = [[.1, .9], [.2, .8], [.3, .7]]
        pvals_val = numpy.asarray(pvals_val, dtype=config.floatX)
        seed_gen = numpy.random.RandomState(utt.fetch_seed())
        numpy_rng = numpy.random.RandomState(int(seed_gen.randint(2**30)))

        # Arguments of size (3,)
        val0 = f(n_val, pvals_val)
        numpy_val0 = numpy.asarray([numpy_rng.multinomial(n=nv, pvals=pv)
            for nv, pv in zip(n_val, pvals_val)])
        assert numpy.all(val0 == numpy_val0)

        # arguments of size (2,)
        val1 = f(n_val[:-1], pvals_val[:-1])
        numpy_val1 = numpy.asarray([numpy_rng.multinomial(n=nv, pvals=pv)
            for nv, pv in zip(n_val[:-1], pvals_val[:-1])])
        assert numpy.all(val1 == numpy_val1)

        # Specifying the size explicitly
        g = function([n, pvals], random.multinomial(n=n, pvals=pvals, size=(3,)))
        val2 = g(n_val, pvals_val)
        numpy_rng = numpy.random.RandomState(int(seed_gen.randint(2**30)))
        numpy_val2 = numpy.asarray([numpy_rng.multinomial(n=nv, pvals=pv)
            for nv, pv in zip(n_val, pvals_val)])
        assert numpy.all(val2 == numpy_val2)
        self.assertRaises(ValueError, g, n_val[:-1], pvals_val[:-1])

# 需要导入模块: from theano.tensor.shared_randomstreams import RandomStreams [as 别名]
# 或者: from theano.tensor.shared_randomstreams.RandomStreams import multinomial [as 别名]
    def prediction(self, h, bias):
        srng = RandomStreams(seed=42)

        prop, mean_x, mean_y, std_x, std_y, rho, bernoulli = \
            self.compute_parameters(h, bias)

        mode = T.argmax(srng.multinomial(pvals=prop, dtype=prop.dtype), axis=1)

        v = T.arange(0, mean_x.shape[0])
        m_x = mean_x[v, mode]
        m_y = mean_y[v, mode]
        s_x = std_x[v, mode]
        s_y = std_y[v, mode]
        r = rho[v, mode]
        # cov = r * (s_x * s_y)

        normal = srng.normal((h.shape[0], 2))
        x = normal[:, 0]
        y = normal[:, 1]

        # x_n = T.shape_padright(s_x * x + cov * y + m_x)
        # y_n = T.shape_padright(s_y * y + cov * x + m_y)

        x_n = T.shape_padright(m_x + s_x * x)
        y_n = T.shape_padright(m_y + s_y * (x * r + y * T.sqrt(1.-r**2)))

        uniform = srng.uniform((h.shape[0],))
        pin = T.shape_padright(T.cast(bernoulli > uniform, floatX))

        return T.concatenate([x_n, y_n, pin], axis=1)

# 需要导入模块: from theano.tensor.shared_randomstreams import RandomStreams [as 别名]
# 或者: from theano.tensor.shared_randomstreams.RandomStreams import multinomial [as 别名]
    def score(self, Y, Y_hat):
        # TODO fix me later when using IndexSpace

        assert hasattr(Y_hat, 'owner')
        owner = Y_hat.owner
        assert owner is not None
        op = owner.op
        if isinstance(op, Print):
            assert len(owner.inputs) == 1
            Y_hat, = owner.inputs
            owner = Y_hat.owner
            op = owner.op
        assert isinstance(op, T.nnet.Softmax)
        state_below, = owner.inputs
        assert state_below.ndim == 2

        # TODO make this more generic like above
        state_below = state_below.owner.inputs[0].owner.inputs[0]

        Y = T.argmax(Y, axis = 1)
        k = self.num_noise_samples

        if self.noise_prob is None:
            theano_rng = RandomStreams(seed = self.mlp.rng.randint(2 ** 15))
            noise = theano_rng.random_integers(size = (state_below.shape[0], self.num_noise_samples,), low=0, high = self.n_classes - 1)
            p_n = 1. / self.n_classes
            p_w = T.nnet.sigmoid((state_below * self.W[:, Y].T).sum(axis=1) + self.b[Y])
            p_x = T.nnet.sigmoid((T.concatenate([state_below] * k) * self.W[:, noise.flatten()].T).sum(axis=1) + self.b[noise.flatten()])
            # TODO is this reshape necessary?
            p_x = p_x.reshape((state_below.shape[0], k))

            #pos = k * p_n / (p_w + k * p_n) * T.log(p_w)
            #neg = (p_x / (p_x + k * p_n) * T.log(p_x)).sum(axis=1)
        else:
            #import ipdb
            #ipdb.set_trace()
            theano_rng = MRG_RandomStreams(max(self.mlp.rng.randint(2 ** 15), 1))
            assert self.mlp.batch_size is not None
            noise = theano_rng.multinomial(pvals = np.tile(self.noise_prob.get_value(), (k * self.mlp.batch_size, 1)))
            noise = T.argmax(noise, axis = 1)
            p_n = self.noise_prob
            p_w = T.nnet.sigmoid((state_below * self.W[:, Y].T).sum(axis=1) + self.b[Y])
            p_x = T.nnet.sigmoid((T.concatenate([state_below] * k) * self.W[:, noise.flatten()].T).sum(axis=1) + self.b[noise.flatten()])
            p_x = p_x.reshape((state_below.shape[0], k))

            pos = k * p_n[Y] / (p_w + k * p_n[Y]) * T.log(p_w)
            neg = (p_x / (p_x + k * p_n[noise].reshape(p_x.shape)) * T.log(p_x)).sum(axis=1)


        #return -(pos - neg).mean()
        return p_w, p_x

# 需要导入模块: from theano.tensor.shared_randomstreams import RandomStreams [as 别名]
# 或者: from theano.tensor.shared_randomstreams.RandomStreams import multinomial [as 别名]
    def test_multinomial(self):
        """Test that RandomStreams.multinomial generates the same results as numpy"""
        # Check over two calls to see if the random state is correctly updated.
        random = RandomStreams(utt.fetch_seed())
        fn = function([], random.multinomial((4, 4), 1, [0.1]*10), updates=random.updates())

        fn_val0 = fn()
        fn_val1 = fn()

        rng_seed = numpy.random.RandomState(utt.fetch_seed()).randint(2**30)
        rng = numpy.random.RandomState(int(rng_seed))  # int() is for 32bit
        numpy_val0 = rng.multinomial(1, [0.1]*10, size=(4, 4))
        numpy_val1 = rng.multinomial(1, [0.1]*10, size=(4, 4))

        assert numpy.all(fn_val0 == numpy_val0)
        assert numpy.all(fn_val1 == numpy_val1)

# 需要导入模块: from theano.tensor.shared_randomstreams import RandomStreams [as 别名]
# 或者: from theano.tensor.shared_randomstreams.RandomStreams import multinomial [as 别名]
    def test_default_shape(self):
        random = RandomStreams(utt.fetch_seed())
        f = function([], random.uniform())
        g = function([], random.multinomial())

        # seed_rng is generator for generating *seeds* for RandomStates
        seed_rng = numpy.random.RandomState(utt.fetch_seed())
        uniform_rng = numpy.random.RandomState(int(seed_rng.randint(2**30)))
        multinomial_rng = numpy.random.RandomState(int(seed_rng.randint(2**30)))

        val0 = f()
        val1 = f()
        numpy_val0 = uniform_rng.uniform()
        numpy_val1 = uniform_rng.uniform()
        assert numpy.allclose(val0, numpy_val0)
        assert numpy.allclose(val1, numpy_val1)

        for i in range(10):  # every test has 50% chance of passing even with non-matching random states
            val2 = g()
            numpy_val2 = multinomial_rng.multinomial(n=1, pvals=[.5, .5])
            assert numpy.all(val2 == numpy_val2)

# 需要导入模块: from theano.tensor.shared_randomstreams import RandomStreams [as 别名]
# 或者: from theano.tensor.shared_randomstreams.RandomStreams import multinomial [as 别名]

#.........这里部分代码省略.........
        if self.output_type == 'real':
            self._prediction = self._predictions[:, :, :self.numvis]
        elif self.output_type == 'binary':
            self._prediction = sigmoid(self._predictions[:, :, :self.numvis])
        elif self.output_type == 'softmax':
            # softmax doesn't support 3d tensors, reshape batch and time axis
            # together, apply softmax and reshape back to 3d tensor
            self._prediction = T.nnet.softmax(
                self._predictions[:, :, :self.numvis].reshape((
                    self._predictions.shape[0] * self._predictions.shape[1],
                    self.numvis
                ))
            ).reshape((
                self._predictions.shape[0],
                self._predictions.shape[1],
                self.numvis
            ))
        else:
            raise ValueError('unsupported output_type')

        # set cost
        self._prediction_for_training = self._prediction[:self.numframes-1]
        if self.output_type == 'real':
            self._cost = T.mean((
                self._prediction_for_training -
                self._input_frames[1:self.numframes]
            )**2)
            self._cost_varlen = T.mean((
                self._prediction -
                self._input_frames[1:]
            )**2)
        elif self.output_type == 'binary':
            self._cost = -T.mean(
                self._input_frames[1:self.numframes] *
                T.log(self._prediction_for_training) +
                (1.0 - self._input_frames[4:self.numframes]) * T.log(
                    1.0 - self._prediction))
            self._cost_varlen = -T.mean(
                self._input_frames[1:] *
                T.log(self._prediction_for_training) +
                (1.0 - self._input_frames[1:]) * T.log(
                    1.0 - self._prediction))
        elif self.output_type == 'softmax':
            self._cost = -T.mean(T.log(
                self._prediction_for_training) *
                self._input_frames[1:self.numframes])
            self._cost_varlen = -T.mean(T.log(
                self._prediction) *
                self._input_frames[1:])

        # set gradients
        self._grads = T.grad(self._cost, self.params)

        # theano function for computing cost and grad
        self.cost = theano.function([self.inputs], self._cost,
                                    updates=self.updates)
        self.grads = theano.function([self.inputs], self._grads,
                                     updates=self.updates)
        
        # another set of variables
        # give some time steps of characters and free the model to predict for all the rest.
        self.inputs_var = T.fmatrix('inputs_var')
        self.nsteps = T.lscalar('nsteps')
        givens = {}
        givens[self.inputs] = T.concatenate(
            (self.inputs_var[:, :self.numvis],
             T.zeros((self.inputs_var.shape[0], self.nsteps*self.numvis))
            ),
            axis=1)

        self.predict = theano.function(
            [self.inputs_var, theano.Param(self.nsteps, default=self.numframes-4)],
            self._prediction.transpose(1, 0, 2).reshape((
                self.inputs_var.shape[0], self.nsteps*self.numvis)),
            updates=self.updates,
            givens=givens)

    def grad(self, x):
        def get_cudandarray_value(x):
            if type(x) == theano.sandbox.cuda.CudaNdarray:
                return np.array(x.__array__()).flatten()
            else:
                return x.flatten()
        return np.concatenate([get_cudandarray_value(g) for g in self.grads(x)])

    def sample(self, numcases=1, numframes=10, temperature=1.0):
        assert self.output_type == 'softmax'
        next_prediction_and_state = theano.function(
            [self._input_frames, self.hids_0],
            [self.theano_rng.multinomial(pvals=T.nnet.softmax(self.x_pred_1/temperature)),
             self.hids_1]
        )
        preds = np.zeros((numcases, numframes, self.numvis), dtype="float32")
        preds[:, 0, :] = self.numpy_rng.multinomial(numcases, pvals=np.ones(self.numvis)/np.float(self.numvis))
        hids = np.zeros((numcases, self.numhid), dtype="float32")
        for t in range(1, numframes):
            nextpredandstate = next_prediction_and_state(preds[:,[t-1],:], hids)
            hids = nextpredandstate[1]
            preds[:,t,:] = nextpredandstate[0]
        return preds

# 需要导入模块: from theano.tensor.shared_randomstreams import RandomStreams [as 别名]
# 或者: from theano.tensor.shared_randomstreams.RandomStreams import multinomial [as 别名]

#.........这里部分代码省略.........
    
    def _output(self, x):
        softmax_x = T.log(T.exp(self._softminus(
            T.dot(x, self.W) + self.b)).sum(0))
        output = -T.nnet.softplus(self.U + softmax_x).sum(1)
        
        return T.argmax(T.nnet.softmax(output))
    
    def _softminus(self, x):
        return x - T.nnet.softplus(x)
    
    def _act(self, x, y):
        return self._softminus(self.b + T.dot(x, self.W)) + T.dot(y, self.U)
    
    def _mean_g(self, x, y):
        act = self._act(x, y)
        
        return T.exp(act) / (1. + T.exp(act).sum(0)), 1. / (1. + T.exp(act).sum(0))
    
    def _mean_h(self, g, x):
        return T.maximum(g, T.nnet.sigmoid(T.dot(x, self.W) + self.b))
    
    def _mean_x(self, h):
        return T.dot(h, self.W.T) + self.c
    
    def _mean_y(self, g):
        return T.nnet.softmax(T.dot(g, self.U.T).sum(0) + self.d)
    
    def _sample_g(self, x, y):
        g_mean, g_zeros = self._mean_g(x, y)
        
        g_mean = T.concatenate((g_zeros.dimshuffle('x', 0), g_mean))
        
        g_sample = self.theano_rng.multinomial(n=1, pvals=g_mean.T,
                dtype=theano.config.floatX).T[1:]
        
        return g_sample
    
    def _sample_h(self, g, x):
        h_mean = self._mean_h(g, x)
        
        h_sample = self.theano_rng.binomial(size=h_mean.shape, n=1, p=h_mean,
                dtype=theano.config.floatX)
        
        return h_sample
    
    def _sample_x(self, h):
        x_mean = self._mean_x(h)
        
        x_sample = self.theano_rng.binomial(size=x_mean.shape, n=1, p=x_mean,
                dtype = theano.config.floatX)
        
        return x_sample
    
    def _sample_y(self, g):
        y_mean = self._mean_y(g)
        
        y_sample = self.theano_rng.multinomial(n=1, pvals=y_mean,
                dtype = theano.config.floatX)
        
        return y_sample
    
    def __train(self):
        nx_samples = self.x
        ng_samples = self._sample_g(self.x, self.y)
        for _ in range(self.K):

# 需要导入模块: from theano.tensor.shared_randomstreams import RandomStreams [as 别名]
# 或者: from theano.tensor.shared_randomstreams.RandomStreams import multinomial [as 别名]

#.........这里部分代码省略.........
            self._prediction = self._predictions[:, :, :self.numvis]
        elif self.output_type == 'binary':
            self._prediction = sigmoid(self._predictions[:, :, :self.numvis])
        elif self.output_type == 'softmax':
            # softmax doesn't support 3d tensors, reshape batch and time axis
            # together, apply softmax and reshape back to 3d tensor
            self._prediction = T.nnet.softmax(
                self._predictions[:, :, :self.numvis].reshape((
                    self._predictions.shape[0] * self._predictions.shape[1],
                    self.numvis
                ))
            ).reshape((
                self._predictions.shape[0],
                self._predictions.shape[1],
                self.numvis
            ))
        else:
            raise ValueError('unsupported output_type')

        # set cost
        self._prediction_for_training = self._prediction[:self.numframes-1]
        if self.output_type == 'real':
            self._cost = T.mean((
                self._prediction_for_training -
                self._input_frames[1:self.numframes]
            )**2)
            self._cost_varlen = T.mean((
                self._prediction -
                self._input_frames[1:]
            )**2)
        elif self.output_type == 'binary':
            self._cost = -T.mean(
                self._input_frames[1:self.numframes] *
                T.log(self._prediction_for_training) +
                (1.0 - self._input_frames[4:self.numframes]) * T.log(
                    1.0 - self._prediction))
            self._cost_varlen = -T.mean(
                self._input_frames[1:] *
                T.log(self._prediction_for_training) +
                (1.0 - self._input_frames[1:]) * T.log(
                    1.0 - self._prediction))
        elif self.output_type == 'softmax':
            self._cost = -T.mean(T.log(
                self._prediction_for_training) *
                self._input_frames[1:self.numframes])
            self._cost_varlen = -T.mean(T.log(
                self._prediction) *
                self._input_frames[1:])

        # set gradients
        self._grads = T.grad(self._cost, self.params)

        # theano function for computing cost and grad
        self.cost = theano.function([self.inputs], self._cost,
                                    updates=self.updates)
        self.grads = theano.function([self.inputs], self._grads,
                                     updates=self.updates)
        
        # another set of variables
        # give some time steps of characters and free the model to predict for all the rest.
        self.inputs_var = T.fmatrix('inputs_var')
        self.nsteps = T.lscalar('nsteps')
        givens = {}
        givens[self.inputs] = T.concatenate(
            (self.inputs_var[:, :self.numvis],
             T.zeros((self.inputs_var.shape[0], self.nsteps*self.numvis))
            ),
            axis=1)

        self.predict = theano.function(
            [self.inputs_var, theano.Param(self.nsteps, default=self.numframes-4)],
            self._prediction.transpose(1, 0, 2).reshape((
                self.inputs_var.shape[0], self.nsteps*self.numvis)),
            updates=self.updates,
            givens=givens)

    def grad(self, x):
        def get_cudandarray_value(x):
            if type(x) == theano.sandbox.cuda.CudaNdarray:
                return numpy.array(x.__array__()).flatten()
            else:
                return x.flatten()
        return numpy.concatenate([get_cudandarray_value(g) for g in self.grads(x)])

    def sample(self, numcases=1, numframes=10, temperature=1.0):
        assert self.output_type == 'softmax'
        next_prediction_and_state = theano.function(
            [self._input_frames, T.concatenate(self.hids_t0)],
            [self.theano_rng.multinomial(pvals=T.nnet.softmax(self.x_pred_1/temperature)),
             T.concatenate(self.hids_t1)]
        )
        preds = numpy.zeros((numcases, numframes, self.numvis), dtype="float32")
        preds[:, 0, :] = self.numpy_rng.multinomial(
            numcases, pvals=numpy.ones(self.numvis)/numpy.float(self.numvis))
        hids = numpy.zeros((numcases, self.numhid), dtype="float32")
        for t in range(1, numframes):
            nextpredandstate = next_prediction_and_state(preds[:,[t-1],:], hids)
            hids = nextpredandstate[1]
            preds[:,t,:] = nextpredandstate[0]
        return preds

# 需要导入模块: from theano.tensor.shared_randomstreams import RandomStreams [as 别名]
# 或者: from theano.tensor.shared_randomstreams.RandomStreams import multinomial [as 别名]

#.........这里部分代码省略.........
        self.init_vbias()
        self.init_hbias()

    def prop_up(self, vis, D=None):
        if D == None:
            D = self.D
        pre_sigmoid_activation = T.dot(vis, self.W) + T.outer(D, self.hbias)
        return [pre_sigmoid_activation, T.nnet.sigmoid(pre_sigmoid_activation)]

    def prop_down(self, hid):
        pre_softmax_activation = T.dot(hid, self.W.T) + self.vbias
        return [pre_softmax_activation, T.nnet.softmax(pre_softmax_activation)]

    def free_energy(self, v_sample):
        D = T.sum(v_sample, axis=1)
        wx_b = T.dot(v_sample, self.W) + T.outer(D, self.hbias)
        vbias_term = T.dot(v_sample, self.vbias)
        hidden_term = T.sum(T.log(1 + T.exp(wx_b)), axis=1)
        return -hidden_term - vbias_term

    def sample_v_given_h(self, h_sample, D=None):
        if D == None:
            D = self.D
        pre_softmax_v, v_mean = self.prop_down(h_sample)

        v_mean = v_mean / T.sum(v_mean, axis=1).dimshuffle(0, "x")
        v_samples, updates = theano.scan(fn=self.multinom_sampler, non_sequences=[v_mean, D], n_steps=1)
        self.updates = updates
        # v_sample = T.mean(v_samples, axis=0)
        v_sample = v_samples[-1]
        return [pre_softmax_v, v_mean, v_sample]

    def multinom_sampler(self, probs, D):
        v_sample = self.theano_rng.multinomial(n=D, pvals=probs, dtype=theano.config.floatX)
        return v_sample

    def sample_v_given_h_mf(self, h_sample, D=None):
        if D == None:
            D = self.D
        pre_softmax_v, v_mean = self.prop_down(h_sample)
        v_sample = D.dimshuffle(0, "x") * v_mean
        return [pre_softmax_v, v_mean, v_sample]

    def sample_h_given_v(self, v_sample, D=None):
        if D == None:
            D = self.D
        pre_sigmoid_h, h_mean = self.prop_up(v_sample, D)
        h_sample = self.theano_rng.binomial(size=h_mean.shape, n=1, p=h_mean, dtype=theano.config.floatX)
        return [pre_sigmoid_h, h_mean, h_sample]

    def gibbs_hvh(self, h0_sample):
        pre_softmax_v1, v1_mean, v1_sample = self.sample_v_given_h(h0_sample)
        pre_sigmoid_h1, h1_mean, h1_sample = self.sample_h_given_v(v1_sample)
        return [pre_softmax_v1, v1_mean, v1_sample, pre_sigmoid_h1, h1_mean, h1_sample]

    def gibbs_hvh_mf(self, h0_sample):
        pre_softmax_v1, v1_mean, v1_sample = self.sample_v_given_h_mf(h0_sample)
        pre_sigmoid_h1, h1_mean, h1_sample = self.sample_h_given_v(v1_sample)
        return [pre_softmax_v1, v1_mean, v1_sample, pre_sigmoid_h1, h1_mean, h1_sample]

    def gibbs_vhv(self, v0_sample, D):
        pre_sigmoid_h1, h1_mean, h1_sample = self.sample_h_given_v(v0_sample, D)
        pre_softmax_v1, v1_mean, v1_sample = self.sample_v_given_h(h1_sample, D)
        return [pre_sigmoid_h1, h1_mean, h1_sample, pre_softmax_v1, v1_mean, v1_sample]

    def gibbs_vhv_mf(self, v0_sample, D):

# 需要导入模块: from theano.tensor.shared_randomstreams import RandomStreams [as 别名]
# 或者: from theano.tensor.shared_randomstreams.RandomStreams import multinomial [as 别名]

#.........这里部分代码省略.........
                y: valid_y[index * batch_size : (index + 1) * batch_size],
            },
        )

        # compute number of minibatches for training and validation
        n_train_batches = train_x.get_value(borrow=True).shape[0] / batch_size
        n_valid_batches = valid_x.get_value(borrow=True).shape[0] / batch_size

        validate_MSE = theano.function(
            inputs=[index],
            outputs=MSE(self.samples(), y=y),
            givens={
                self.input: valid_x[index * batch_size : (index + 1) * batch_size],
                y: valid_y[index * batch_size : (index + 1) * batch_size],
            },
        )

        print "training..."

        start_time = time.clock()
        epoch = 0

        total_training_costs = []
        total_validation_costs = []
        total_validation_MSE = []

        lr_time = 0
        lr_step = learning_rate / ((train_x.get_value().shape[0] * 1.0 / batch_size) * (n_epochs - 30))
        lr_val = learning_rate

        while epoch < n_epochs:
            epoch = epoch + 1
            epoch_training_costs = []
            # import pdb; pdb.set_trace()
            for minibatch_index in xrange(n_train_batches):

                # linear annealing after 40 epochs...
                if epoch > 40:
                    # lr_val = learning_rate / (1.0+lr_time)
                    # lr_time = lr_time + 1
                    lr_val = lr_val - lr_step
                else:
                    lr_val = learning_rate

                loss_value = train_model(minibatch_index, lr_val)
                epoch_training_costs.append(loss_value)

                if np.isnan(loss_value):
                    import pdb

                    pdb.set_trace()
                    print "got NaN in NLL"
                    sys.exit(1)

            this_training_cost = np.mean(epoch_training_costs)
            this_validation_cost = np.mean([validate_model(i) for i in xrange(n_valid_batches)])
            this_validation_MSE = np.mean([validate_MSE(i) for i in xrange(n_valid_batches)])

            total_training_costs.append(this_training_cost)
            total_validation_costs.append(this_validation_cost)
            total_validation_MSE.append(this_validation_MSE)

            print "epoch %i, training NLL %f, validation NLL %f, MSE %f" % (
                epoch,
                this_training_cost,
                this_validation_cost,
                this_validation_MSE,
            )

        end_time = time.clock()

        print "Training took %.2f minutes..." % ((end_time - start_time) / 60.0)

        # return losses and parameters..
        return total_training_costs, total_validation_costs, total_validation_MSE

    def samples(self):
        component = self.srng.multinomial(pvals=self.outputLayer.mixing)
        component_std = T.sum(self.outputLayer.sigma * component, axis=1, keepdims=True)

        samples = self.srng.normal(std=component_std)
        return samples

    def save_model(self, filename="MLP.save", output_folder="output_folder"):
        """
        This function pickles the paramaters in a file for later usage
        """
        storage_file = open(os.path.join(output_folder, filename), "wb")
        cPickle.dump(self, storage_file, protocol=cPickle.HIGHEST_PROTOCOL)
        storage_file.close()

    @staticmethod
    def load_model(filename="MLP.save", output_folder="output_folder"):
        """
        This function loads pickled paramaters from a file
        """
        storage_file = open(os.path.join(output_folder, filename), "rb")
        model = cPickle.load(storage_file)
        storage_file.close()
        return model

# 需要导入模块: from theano.tensor.shared_randomstreams import RandomStreams [as 别名]
# 或者: from theano.tensor.shared_randomstreams.RandomStreams import multinomial [as 别名]
										outputs_info=[None],
										non_sequences=[init_multi_samp,Tweights,nsteps],
										n_steps=ntest)
		
		sample_metropolis=theano.function([Tweights, nsteps],Tm_samps,
											allow_input_downcast=True)
		
		
		##setting up Theano sampling =======================================
		
		nummat=np.repeat(np.reshape(np.arange(npcl),(npcl,1)),npcl,axis=1)
		idx_mat=theano.shared(nummat.T)
		
		Tprobs=T.fvector()
		
		t_samp=rng.multinomial(size=Tprobs.shape,pvals=Tprobs)
		idxs=T.cast(T.sum(t_samp*idx_mat,axis=1),'int64')
		
		sample_theano=theano.function([Tprobs],idxs,allow_input_downcast=True)
		
		
		
		## Speed test
		
		weights=np.random.rand(npcl)
		probs=weights/np.sum(weights)
		
		
		
		m_samps=np.zeros((ntest,npcl))
		t_samps=np.zeros((ntest,npcl))

# 需要导入模块: from theano.tensor.shared_randomstreams import RandomStreams [as 别名]
# 或者: from theano.tensor.shared_randomstreams.RandomStreams import multinomial [as 别名]

#.........这里部分代码省略.........
            return x_pred_t, T.concatenate(hids_t, 1)

        def step_nodropout(x_gt_t, x_tm1, hids_tm1):
            hids_tm1 = [hids_tm1[:,k*self.numhid:(k+1)*self.numhid] for k in range(self.numlayers)]
            pre_hids_t = [T.dot(hids_tm1[0], self.whh[0]) + self.bhid[0] + T.dot(x_gt_t, self.wxh[0])]
            hids_t = [pre_hids_t[0] * (pre_hids_t[0] > 0)]
            for k in range(1, self.numlayers):
                pre_hids_t.append(T.dot(hids_tm1[k], self.whh[k]) + self.bhid[k] + T.dot(hids_t[k-1], self.wxh[k]))
                hids_t.append(pre_hids_t[k] * (pre_hids_t[k] > 0))
            x_pred_t = self.bx
            for k in range(self.numlayers):
                x_pred_t += T.dot(hids_t[k], self.whx[k]) 
            return x_pred_t, T.concatenate(hids_t, 1)

        if self.dropout == 0.0:
            (self._predictions, self.hids), self.updates = theano.scan(
                                                        fn=step_nodropout,
                                                        sequences=self._input_frames,
                                                        outputs_info=[self._input_frames[0], self.hids_0])
        else:
            self._dropoutmask = theano_rng.binomial(
                size=(self.inputs.shape[1] // self.numvis,
                      self._batchsize, self.numhid),
                n=1, p=self.dropout, dtype=theano.config.floatX
            )
            (self._predictions, self.hids), self.updates = theano.scan(
                                                        fn=step_dropout,
                                                        sequences=[self._input_frames, self._dropoutmask],
                                                        outputs_info=[self._input_frames[0], self.hids_0])

        if self.output_type == 'real':
            self._prediction = self._predictions[:, :, :self.numvis]  # dims: [time step, batch idx, numvis]
        elif self.output_type == 'binary':
            self._prediction = sigmoid(self._predictions[:, :, :self.numvis])
        elif self.output_type == 'softmax':
            # softmax doesn't support 3d tensors, reshape batch and time axis
            # together, apply softmax and reshape back to 3d tensor
            self._prediction = T.nnet.softmax(
                self._predictions[:, :, :self.numvis].reshape((
                    self._predictions.shape[0] * self._predictions.shape[1],
                    self.numvis
                ))
            ).reshape((
                self._predictions.shape[0],
                self._predictions.shape[1],
                self.numvis
            ))
        else:
            raise ValueError('unsupported output_type')

        self._prediction_for_training = self._prediction[:self.numframes-1]

        if self.output_type == 'real':
            self._cost = T.mean(( self._prediction_for_training - self._input_frames[1:self.numframes])**2)
            self._cost_varlen = T.mean(( self._prediction - self._input_frames[1:])**2)  # for various lengths
        elif self.output_type == 'binary':
            self._cost = -T.mean( self._input_frames[1:self.numframes] * T.log(self._prediction_for_training) + (1.0 - self._input_frames[1:self.numframes]) * T.log( 1.0 - self._prediction))
            self._cost_varlen = -T.mean( self._input_frames[1:] * T.log(self._prediction_for_training) + (1.0 - self._input_frames[1:]) * T.log( 1.0 - self._prediction))
        elif self.output_type == 'softmax':
            self._cost = -T.mean(T.log( self._prediction_for_training) * self._input_frames[1:self.numframes])
            self._cost_varlen = -T.mean(T.log( self._prediction) * self._input_frames[1:])

        self._grads = T.grad(self._cost, self.params)

        self.inputs_var = T.fmatrix('inputs_var')
        self.nsteps = T.lscalar('nsteps')
        givens = {}
        givens[self.inputs] = T.concatenate(
            ( self.inputs_var[:, :self.numvis],
              T.zeros((self.inputs_var.shape[0], self.nsteps*self.numvis))
            ),
            axis=1)
        
        # predict given the first letters. 
        self.predict = theano.function(
            [self.inputs_var, theano.Param(self.nsteps, default=self.numframes-4)],
            self._prediction.transpose(1, 0, 2).reshape((self.inputs_var.shape[0], self.nsteps*self.numvis)),
            updates=self.updates, givens=givens)
        self.cost = theano.function( [self.inputs], self._cost, updates=self.updates)
        self.grads = theano.function( [self.inputs], self._grads, updates=self.updates)

    def grad(self, x):
        def get_cudandarray_value(x):
            if type(x) == theano.sandbox.cuda.CudaNdarray:
                return np.array(x.__array__()).flatten()
            else:
                return x.flatten()
        return np.concatenate([get_cudandarray_value(g) for g in self.grads(x)])

    def sample(self, numcases=1, numframes=10, temperature=1.0):
        assert self.output_type == 'softmax'
        next_prediction_and_state = theano.function([self._input_frames, self.hids_0], [self.theano_rng.multinomial(pvals=T.nnet.softmax(self.x_pred_1/temperature)), self.hids_1])
        preds = np.zeros((numcases, numframes, self.numvis), dtype="float32")
        preds[:,0,:] = self.numpy_rng.multinomial(numcases, pvals=np.ones(self.numvis)/np.float(self.numvis))
        hids = np.zeros((numcases, self.numhid*self.numlayers), dtype="float32")
        for t in range(1, numframes):
            nextpredandstate = next_prediction_and_state(preds[:,[t-1],:], hids)
            hids = nextpredandstate[1]
            preds[:,t,:] = nextpredandstate[0]
        return preds

# 需要导入模块: from theano.tensor.shared_randomstreams import RandomStreams [as 别名]
# 或者: from theano.tensor.shared_randomstreams.RandomStreams import multinomial [as 别名]

#.........这里部分代码省略.........

        self.x_pred_1 = self.bx
        for k in range(2):
            self.x_pred_1 += T.dot(self.hids_t1[k], self.whx[k])
        # end of one-step prediction

        def step(x_tm1, hids_tm1):
            hids_tm1 = [hids_tm1[:, k * self.numhid : (k + 1) * self.numhid] for k in range(2)]
            hids_t = [ReLU(T.dot(hids_tm1[0], self.whh[0]) + T.dot(x_tm1, self.wxh[0]) + self.bhid[0])]
            hids_t.append(ReLU(T.dot(hids_tm1[1], self.whh[1]) + T.dot(hids_t[-1], self.wxh[1]) + self.bhid[1]))

            x_pred_t = self.bx
            for k in range(2):
                x_pred_t += T.dot(hids_t[k], self.whx[k])
            return x_pred_t, T.concatenate(hids_t, 1)

        (self._predictions, self.hids), self.updates = theano.scan(
            fn=step, sequences=self._input_frames, outputs_info=[None, T.concatenate(self.hids_t0, 1)]
        )

        # set up output prediction
        if self.output_type == "real":
            self._prediction = self._predictions[:, :, : self.numvis]
        elif self.output_type == "binary":
            self._prediction = sigmoid(self._predictions[:, :, : self.numvis])
        elif self.output_type == "softmax":
            # softmax doesn't support 3d tensors, reshape batch and time axis
            # together, apply softmax and reshape back to 3d tensor
            self._prediction = T.nnet.softmax(
                self._predictions[:, :, : self.numvis].reshape(
                    (self._predictions.shape[0] * self._predictions.shape[1], self.numvis)
                )
            ).reshape((self._predictions.shape[0], self._predictions.shape[1], self.numvis))
        else:
            raise ValueError("unsupported output_type")

        # set cost
        self._prediction_for_training = self._prediction[: self.numframes - 1]
        if self.output_type == "real":
            self._cost = T.mean((self._prediction_for_training - self._input_frames[1 : self.numframes]) ** 2)
            self._cost_varlen = T.mean((self._prediction - self._input_frames[1:]) ** 2)
        elif self.output_type == "binary":
            self._cost = -T.mean(
                self._input_frames[1 : self.numframes] * T.log(self._prediction_for_training)
                + (1.0 - self._input_frames[4 : self.numframes]) * T.log(1.0 - self._prediction)
            )
            self._cost_varlen = -T.mean(
                self._input_frames[1:] * T.log(self._prediction_for_training)
                + (1.0 - self._input_frames[1:]) * T.log(1.0 - self._prediction)
            )
        elif self.output_type == "softmax":
            self._cost = -T.mean(T.log(self._prediction_for_training) * self._input_frames[1 : self.numframes])
            self._cost_varlen = -T.mean(T.log(self._prediction) * self._input_frames[1:])

        # set gradients
        self._grads = T.grad(self._cost, self.params)

        # theano function for computing cost and grad
        self.cost = theano.function([self.inputs], self._cost, updates=self.updates)
        self.grads = theano.function([self.inputs], self._grads, updates=self.updates)

        # another set of variables
        # give some time steps of characters and free the model to predict for all the rest.
        self.inputs_var = T.fmatrix("inputs_var")
        self.nsteps = T.lscalar("nsteps")
        givens = {}
        givens[self.inputs] = T.concatenate(
            (self.inputs_var[:, : self.numvis], T.zeros((self.inputs_var.shape[0], self.nsteps * self.numvis))), axis=1
        )

        self.predict = theano.function(
            [self.inputs_var, theano.Param(self.nsteps, default=self.numframes - 4)],
            self._prediction.transpose(1, 0, 2).reshape((self.inputs_var.shape[0], self.nsteps * self.numvis)),
            updates=self.updates,
            givens=givens,
        )

    def grad(self, x):
        def get_cudandarray_value(x):
            if type(x) == theano.sandbox.cuda.CudaNdarray:
                return np.array(x.__array__()).flatten()
            else:
                return x.flatten()

        return np.concatenate([get_cudandarray_value(g) for g in self.grads(x)])

    def sample(self, numcases=1, numframes=10, temperature=1.0):
        assert self.output_type == "softmax"
        next_prediction_and_state = theano.function(
            [self._input_frames, self.hids_0],
            [self.theano_rng.multinomial(pvals=T.nnet.softmax(self.x_pred_1 / temperature)), self.hids_1],
        )
        preds = np.zeros((numcases, numframes, self.numvis), dtype="float32")
        preds[:, 0, :] = self.numpy_rng.multinomial(numcases, pvals=np.ones(self.numvis) / np.float(self.numvis))
        hids = np.zeros((numcases, self.numhid), dtype="float32")
        for t in range(1, numframes):
            nextpredandstate = next_prediction_and_state(preds[:, [t - 1], :], hids)
            hids = nextpredandstate[1]
            preds[:, t, :] = nextpredandstate[0]
        return preds

# 需要导入模块: from theano.tensor.shared_randomstreams import RandomStreams [as 别名]
# 或者: from theano.tensor.shared_randomstreams.RandomStreams import multinomial [as 别名]
class CharacterRNN(ParameterModel):

    def __init__(self, name, n_input, n_output, n_hidden=10, n_layers=2,
                 seed=None):
        super(CharacterRNN, self).__init__(name)
        self.n_hidden = n_hidden
        self.n_layers = n_layers

        self.n_input = n_input
        self.n_output = n_output

        self.lstm = MultilayerLSTM('%s-charrnn' % name, self.n_input,
                         n_hidden=self.n_hidden,
                         n_layers=self.n_layers,
                         )
        self.rng = RandomStreams(seed)

        self.output = Softmax('%s-softmax' % name, n_hidden, self.n_output)

    def save_parameters(self, location):
        state = {
            'n_hidden': self.n_hidden,
            'n_layers': self.n_layers,
            'lstm': self.lstm.state(),
            'output': self.output.state()
        }
        with open(location, 'wb') as fp:
            pickle.dump(state, fp)

    def load_parameters(self, location):
        with open(location, 'rb') as fp:
            state = pickle.load(fp)

        self.n_hidden = state['n_hidden']
        self.n_layers = state['n_layers']
        self.lstm.load(state['lstm'])
        self.output.load(state['output'])

    @theanify(T.tensor3('X'), T.tensor3('state'), T.tensor3('y'), returns_updates=True)
    def cost(self, X, state, y):
        (_, state, ypred), updates = self.forward(X, state)
        S, N, V = y.shape
        y = y.reshape((S * N, V))
        ypred = ypred.reshape((S * N, V))
        return (T.nnet.categorical_crossentropy(ypred, y).mean(), state), updates

    def forward(self, X, state):
        S, N, D = X.shape
        H = self.lstm.n_hidden
        L = self.lstm.n_layers
        O = self.output.n_output

        def step(input, previous_hidden, previous_state, previous_output):
            lstm_hidden, state = self.lstm.forward(input, previous_hidden, previous_state)
            final_output = self.output.forward(lstm_hidden[:, -1, :], 1.0)
            return lstm_hidden, state, final_output

        hidden = T.unbroadcast(T.alloc(np.array(0).astype(theano.config.floatX), N, L, H), 1)

        (encoder_output, encoder_state, softmax_output), updates = theano.scan(step,
                              sequences=[X],
                              outputs_info=[
                                            hidden,
                                            state,
                                            T.alloc(np.asarray(0).astype(theano.config.floatX),
                                                    N,
                                                    O),
                                           ],
                              n_steps=S)
        return (encoder_output, encoder_state, softmax_output), updates

    @theanify(T.vector('start_token'), T.iscalar('length'), T.scalar('temperature'), returns_updates=True)
    def generate(self, start_token, length, temperature):
        start_token = start_token[:, np.newaxis].T
        N = 1
        H = self.lstm.n_hidden
        L = self.lstm.n_layers

        def step(input, previous_hidden, previous_state, temperature):
            lstm_hidden, state = self.lstm.forward(input, previous_hidden, previous_state)
            final_output = self.output.forward(lstm_hidden[:, -1, :], temperature)
            sample = self.rng.multinomial(n=1, size=(1,), pvals=final_output, dtype=theano.config.floatX)
            return sample, lstm_hidden, state

        hidden = T.unbroadcast(T.alloc(np.array(0).astype(theano.config.floatX), N, L, H), 1)
        state = T.unbroadcast(T.alloc(np.array(0).astype(theano.config.floatX), N, L, H), 1)

        (softmax_output, _, _), updates = theano.scan(step,
                              outputs_info=[
                                            start_token,
                                            hidden,
                                            state,
                                           ],
                              non_sequences=[temperature],
                              n_steps=length)
        return softmax_output[:, 0, :], updates

    @theanify(T.fvector('start_token'), T.fvector('concat'), T.iscalar('length'), T.fscalar('temperature'), returns_updates=True)
    def generate_with_concat(self, start_token, concat, length, temperature):
        start_token = start_token[:, np.newaxis].T
#.........这里部分代码省略.........